From 86a8fe3ec5f622f4634cfaaffa0eb302c669e3eb Mon Sep 17 00:00:00 2001 From: iphydf Date: Sun, 7 Jan 2024 01:58:43 +0000 Subject: [PATCH] feat: Generate `${type}_Ptr` classes to wrap C pointers. --- src/Apigen/Language/PyDsl.hs | 16 +-- src/Apigen/Parser.hs | 29 +++--- tools/apigen.py | 75 +++++++------ tools/gen-python | 197 +++++++++++++++++++++++++---------- 4 files changed, 212 insertions(+), 105 deletions(-) diff --git a/src/Apigen/Language/PyDsl.hs b/src/Apigen/Language/PyDsl.hs index ae066a9..e9a22f4 100644 --- a/src/Apigen/Language/PyDsl.hs +++ b/src/Apigen/Language/PyDsl.hs @@ -12,7 +12,7 @@ import Apigen.Types (BitSize (..), BuiltinType (..), Module (..)) import Data.Text (Text) import qualified Data.Text as Text -import Language.Cimple (Lexeme (..), lexemeText) +import Language.Cimple (Lexeme (..), LexemeClass) import Prelude hiding ((<$>)) import Text.PrettyPrint.ANSI.Leijen @@ -88,7 +88,7 @@ ppConstness ConstThis = text "True" ppConstness MutableThis = text "False" ppGenerated :: Generated -> Doc -ppGenerated = ppCtor . show +ppGenerated = ppCtor . ("Generated." <>) . show ppBuiltinType :: BuiltinType -> Doc ppBuiltinType Void = ppCtor "Void" @@ -111,10 +111,14 @@ ppMaybe _ Nothing = text "None" ppMaybe f (Just x) = f x ppLexeme :: Lexeme Name -> Doc -ppLexeme = ppName . lexemeText - -ppName :: Name -> Doc -ppName (ns, name) = hgo "Name" [ppList (text . show) ns, ppList (text . show) name] +ppLexeme (L _ c s) = ppName c s + +ppName :: LexemeClass -> Name -> Doc +ppName c (ns, name) = hgo "Name" + [ ppCtor . ("LexemeClass." <>) . show $ c + , ppList (text . show) ns + , ppList (text . show) name + ] renderS :: Doc -> String renderS = flip displayS "" . renderSmart 1 120 diff --git a/src/Apigen/Parser.hs b/src/Apigen/Parser.hs index 08979ac..184f9ca 100644 --- a/src/Apigen/Parser.hs +++ b/src/Apigen/Parser.hs @@ -60,6 +60,9 @@ runSimplify decls = do go :: NodeF (Lexeme SId) [Sym] -> M a [Sym] -- {- go (PreprocInclude (L _ LitSysInclude _)) = return [] +go (TyPointer [ConstType (BuiltinType UInt{}) ]) = return [] +go (VarDecl [] _ []) = return [] +go (FunctionPrototype [] _ _) = return [] go (PreprocIfndef (L _ _ SYM_APIGEN_IGNORE) _ es) = return es @@ -74,10 +77,9 @@ go (Enumerator name _) = return [EnumMember name] go (EnumConsts (Just name) enums ) = mkEnum name enums go (EnumDecl name enums _) = mkEnum name enums -go (TyPointer [ BuiltinType ty@UInt{} ]) = return [ ArrayType ty] -go (TyPointer [ConstType (BuiltinType ty@UInt{}) ]) = return [ConstArrayType ty] -go (TyPointer [ConstType (BuiltinType Char )]) = return [BuiltinType String] -go (TyPointer [ConstType (Typename ty )]) = return [ConstPointerType ty] +go (TyPointer [ConstType (BuiltinType Char)]) = return [BuiltinType String] +go (TyPointer [ Typename ty ]) = return [ PointerType ty] +go (TyPointer [ConstType (Typename ty )]) = return [ConstPointerType ty] go (DeclSpecArray (Just [expr])) = return [SizedArrayType (BuiltinType Void) expr] go (DeclSpecArray Nothing) = return [ArrayType Void] @@ -93,7 +95,6 @@ go (FunctionCall [Ref (L _ _ SYM_max)] [[a], [b]]) = return [Max a b] go (TyConst [ty]) = return [ConstType ty] go (TyPointer [BuiltinType Void]) = return [BuiltinType VoidPtr] -go (TyPointer [Typename ty]) = return [PointerType ty] go (TyPointer ty@[CallbackType{}]) = return ty go (AggregateDecl ty@[TypeDecl _]) = return ty @@ -103,18 +104,18 @@ go (TyUserDefined ty) = return [Typename ty] go (TyFunc ty) = return [CallbackType ty] go (Typedef [Typename ty] _) = return [TypeDecl ty] -go (TyStd (L _ _ TY_void)) = return [BuiltinType Void] -go (TyStd (L _ _ TY_char)) = return [BuiltinType Char] -go (TyStd (L _ _ TY_bool)) = return [BuiltinType Bool] -go (TyStd (L _ _ TY_int8_t)) = return [BuiltinType (SInt B8)] -go (TyStd (L _ _ TY_uint8_t)) = return [BuiltinType (UInt B8)] -go (TyStd (L _ _ TY_int16_t)) = return [BuiltinType (SInt B16)] +go (TyStd (L _ _ TY_void )) = return [BuiltinType Void] +go (TyStd (L _ _ TY_char )) = return [BuiltinType Char] +go (TyStd (L _ _ TY_bool )) = return [BuiltinType Bool] +go (TyStd (L _ _ TY_int8_t )) = return [BuiltinType (SInt B8)] +go (TyStd (L _ _ TY_uint8_t )) = return [BuiltinType (UInt B8)] +go (TyStd (L _ _ TY_int16_t )) = return [BuiltinType (SInt B16)] go (TyStd (L _ _ TY_uint16_t)) = return [BuiltinType (UInt B16)] -go (TyStd (L _ _ TY_int32_t)) = return [BuiltinType (SInt B32)] +go (TyStd (L _ _ TY_int32_t )) = return [BuiltinType (SInt B32)] go (TyStd (L _ _ TY_uint32_t)) = return [BuiltinType (UInt B32)] -go (TyStd (L _ _ TY_int64_t)) = return [BuiltinType (SInt B64)] +go (TyStd (L _ _ TY_int64_t )) = return [BuiltinType (SInt B64)] go (TyStd (L _ _ TY_uint64_t)) = return [BuiltinType (UInt B64)] -go (TyStd (L _ _ TY_size_t)) = return [BuiltinType SizeT] +go (TyStd (L _ _ TY_size_t )) = return [BuiltinType SizeT] go (PreprocDefineConst name _) = return [Define name] go (VarDecl [ty] name []) = return [Var ty name] diff --git a/tools/apigen.py b/tools/apigen.py index 3b1f283..71d4bc4 100644 --- a/tools/apigen.py +++ b/tools/apigen.py @@ -3,6 +3,19 @@ from typing import Optional +class Generated(Enum): + GeneratedToString = 0 + GeneratedFromInt = 1 + + +class LexemeClass(Enum): + IdVar = 0 + IdConst = 1 + IdSueType = 2 + IdFuncType = 3 + LitInteger = 4 + + class CType: pass @@ -30,17 +43,9 @@ class UInt(CType): bitSize: int -@dataclass -class Generated: - function: str - - -GeneratedToString = Generated("to_string") -GeneratedFromInt = Generated("from_int") - - @dataclass class Name: + kind: LexemeClass ns: list[str] name: list[str] @@ -49,6 +54,10 @@ class Decl: pass +class Type(Decl): + pass + + @dataclass class Namespace(Decl): name: list[str] @@ -74,26 +83,28 @@ class Enumeration(Decl): @dataclass -class Property(Decl): - name: Name - prop: Decl - - -@dataclass -class ValueProp(Decl): +class Prop(Decl): type: Decl get: Optional[Decl] set: Optional[Decl] @dataclass -class ArrayProp(Decl): - type: Decl - get: Optional[Decl] - set: Optional[Decl] +class ValueProp(Prop): + pass + + +@dataclass +class ArrayProp(Prop): size: Optional[Decl] +@dataclass +class Property(Decl): + name: Name + prop: Prop + + @dataclass class Ref(Decl): name: Name @@ -154,54 +165,54 @@ class Define(Decl): @dataclass -class Typename(Decl): +class Typename(Type): name: Name @dataclass -class BuiltinType(Decl): +class BuiltinType(Type): type: CType @dataclass -class CallbackType(Decl): +class CallbackType(Type): type: Name @dataclass -class PointerType(Decl): +class PointerType(Type): type: Name @dataclass -class ConstPointerType(Decl): +class ConstPointerType(Type): type: Name @dataclass -class SizedArrayType(Decl): - type: Decl +class SizedArrayType(Type): + type: Type size: Decl @dataclass -class ArrayType(Decl): +class ArrayType(Type): type: CType @dataclass -class UserArrayType(Decl): +class UserArrayType(Type): type: Name @dataclass -class ConstArrayType(Decl): +class ConstArrayType(Type): type: CType @dataclass -class ConstType(Decl): - type: Decl +class ConstType(Type): + type: Type @dataclass diff --git a/tools/gen-python b/tools/gen-python index 030713c..6ff9cf3 100755 --- a/tools/gen-python +++ b/tools/gen-python @@ -14,6 +14,11 @@ import apigen # Set to True to type-check the generated model. MYPY = False +def _is_error_ptr(param: apigen.Var) -> bool: + if not (isinstance(param.type, apigen.PointerType) and param.type.type.name): + return False + return param.type.type.name[0] == "Err" + def _relocate_err(ns: tuple[str, ...], name: tuple[str, ...]): if ns and name and name[0].lower() == "err": @@ -40,22 +45,44 @@ def _scream(ns: tuple[str, ...], name: list[str]) -> str: @dataclass -class GenPython: - ns: tuple[str, ...] = tuple() +class CythonDeclarations: prelude: list[str] = field(default_factory=list) enums: list[str] = field(default_factory=list) typedefs: list[str] = field(default_factory=list) callbacks: list[str] = field(default_factory=list) funcs: list[str] = field(default_factory=list) - indent: int = 0 + classes: list[str] = field(default_factory=list) + + def finalize(self) -> list[str]: + return self.prelude + self.enums + self.typedefs + self.callbacks + self.funcs + self.classes + + +@dataclass +class CythonImplementation: + prelude: list[str] = field(default_factory=list) + classes: list[str] = field(default_factory=list) + + def finalize(self) -> list[str]: + return self.prelude + self.classes + + +@dataclass +class GenCython: + ns: tuple[str, ...] = tuple() + pxd: CythonDeclarations = field(default_factory=CythonDeclarations) + pyx: CythonImplementation = field(default_factory=CythonImplementation) in_prop: bool = False in_class: Optional[apigen.Name] = None def _add(self, lines: list[str], *added: str): - lines.extend(" " * (self.indent * 4) + line for line in added) + lines.extend(added) def make_typename(self, name: apigen.Name) -> str: - return _camel(tuple(name.ns), name.name) + if name.ns[0] != self.ns[0]: + prefix = f"{name.ns[0]}." + else: + prefix = "" + return prefix + _camel(tuple(name.ns), name.name) def make_identifier(self, name: apigen.Name) -> str: return _snake(tuple(name.ns), name.name) @@ -65,20 +92,9 @@ class GenPython: raise Exception("not in class") return f"{'const ' if const else ''}{self.make_typename(self.in_class)}* self" - def make_param(self, param: apigen.Var) -> str: - if (isinstance(param.type, apigen.SizedArrayType) - and isinstance(param.type.size, apigen.Ref) - and not param.type.size.name.ns): - return f"{self.make_type(param.type.type)}* {self.make_identifier(param.name)}, size_t {self.make_identifier(param.type.size.name)}" - else: - return f"{self.make_type(param.type)} {self.make_identifier(param.name)}" - - def make_params(self, params: list[apigen.Var]) -> list[str]: - return [self.make_param(param) for param in params] - def make_type(self, ty: apigen.Decl) -> str: if isinstance(ty, apigen.BuiltinType): - return self.make_c_type(ty.type) + return self.make_ctype(ty.type) if isinstance(ty, apigen.Typename): return self.make_typename(ty.name) if isinstance(ty, apigen.CallbackType): @@ -90,14 +106,14 @@ class GenPython: if isinstance(ty, apigen.ConstPointerType): return f"const {self.make_typename(ty.type)}*" if isinstance(ty, apigen.ArrayType): - return f"{self.make_c_type(ty.type)}*" + return f"{self.make_ctype(ty.type)}*" if isinstance(ty, apigen.ConstArrayType): - return f"const {self.make_c_type(ty.type)}*" + return f"const {self.make_ctype(ty.type)}*" if isinstance(ty, apigen.ConstType): return f"const {self.make_type(ty.type)}" raise Exception(f"unhandled type: {ty}") - def make_c_type(self, ty: apigen.CType) -> str: + def make_ctype(self, ty: apigen.CType) -> str: if ty == apigen.Void: return "void" if ty == apigen.VoidPtr: @@ -114,6 +130,70 @@ class GenPython: return f"uint{ty.bitSize}_t" raise Exception(f"unhandled type: {ty}") + def make_param(self, param: apigen.Var) -> str: + if (isinstance(param.type, apigen.SizedArrayType) + and isinstance(param.type.size, apigen.Ref) + and not param.type.size.name.ns): + return f"{self.make_type(param.type.type)}* {self.make_identifier(param.name)}, size_t {self.make_identifier(param.type.size.name)}" + else: + return f"{self.make_type(param.type)} {self.make_identifier(param.name)}" + + def make_params(self, params: list[apigen.Var]) -> list[str]: + return [self.make_param(param) for param in params] + + def make_cdef_param(self, param: apigen.Var) -> str: + return f"{self.make_cdef_type(param.type)} {self.make_identifier(param.name)}" + + def make_cdef_params(self, params: list[apigen.Var]) -> list[str]: + return [self.make_cdef_param(param) for param in params + if not _is_error_ptr(param)] + + def make_cdef_type(self, ty: apigen.Decl) -> str: + if isinstance(ty, apigen.BuiltinType): + return self.make_cdef_ctype(ty.type) + if isinstance(ty, apigen.Typename): + return self.make_typename(ty.name) + if isinstance(ty, apigen.SizedArrayType): + return self.make_cdef_array_type(ty.type) + if isinstance(ty, apigen.PointerType): + return f"{self.make_typename(ty.type)}_Ptr" + if isinstance(ty, apigen.ConstPointerType): + return f"{self.make_typename(ty.type)}_Ptr" + if isinstance(ty, apigen.ArrayType): + return self.make_cdef_array_ctype(ty.type) + if isinstance(ty, apigen.ConstArrayType): + return self.make_cdef_array_ctype(ty.type) + if isinstance(ty, apigen.ConstType): + return self.make_cdef_type(ty.type) + raise Exception(f"unhandled type: {ty}") + + def make_cdef_array_type(self, ty: apigen.Decl) -> str: + if isinstance(ty, apigen.BuiltinType): + return self.make_cdef_array_ctype(ty.type) + if isinstance(ty, apigen.ConstType): + return self.make_cdef_array_type(ty.type) + raise Exception(f"unhandled type: {ty}") + + def make_cdef_array_ctype(self, ty: apigen.CType) -> str: + if isinstance(ty, apigen.UInt) and ty.bitSize == 8: + return f"bytes" + raise Exception(f"unhandled type: {ty}") + + def make_cdef_ctype(self, ty: apigen.CType) -> str: + if ty == apigen.Void: + return "None" + if ty == apigen.Bool: + return "bool" + if ty == apigen.SizeT: + return "int" + if ty == apigen.String: + return "str" + if isinstance(ty, apigen.SInt): + return f"int" + if isinstance(ty, apigen.UInt): + return f"int" + raise Exception(f"unhandled type: {ty}") + def genModel(self, model: apigen.Model) -> dict[str, str]: if len(model.modules) == 1: basedir = os.path.dirname(model.modules[0].path) @@ -123,33 +203,28 @@ class GenPython: mods = {} for module in model.modules: - self.prelude = [ + self.pxd = CythonDeclarations(prelude=[ "# cython: language_level=3", "from libcpp cimport bool", "from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t, int16_t, int32_t", "from libc.stdlib cimport malloc, free", "", - ] - self.enums = [] - self.typedefs = [] - self.callbacks = [] - self.funcs = [] + ]) + self.pyx = CythonImplementation(prelude=[ + "# cython: language_level=3", + ]) basename = os.path.splitext(module.path[len(basedir) + 1:])[0] if basename == "toxav/toxav": - self.prelude.append("from pytox.toxcore.tox cimport Tox") + self.pxd.prelude.append("cimport pytox.toxcore.tox as tox") self.genModule(module) - mods[basename + ".pxd"] = ( - "\n".join(self.prelude + self.enums + self.typedefs + - self.callbacks + self.funcs) + "\n") + mods[basename + ".pxd"] = "\n".join(self.pxd.finalize()) + "\n" return mods def genModule(self, module: apigen.Module): header = os.path.join("tox", os.path.basename(module.path)) - self._add(self.prelude, f'cdef extern from "{header}":') - self.indent += 1 + self._add(self.pxd.prelude, f'cdef extern from "{header}":') for decl in module.decls: self.genDecl(decl) - self.indent -= 1 def genNamespace(self, decl: apigen.Namespace): ns = self.ns @@ -159,8 +234,7 @@ class GenPython: self.ns = ns def genEnumeration(self, decl: apigen.Enumeration): - self._add(self.enums, f"cpdef enum {self.make_typename(decl.name)}:") - self.indent += 1 + self._add(self.pxd.enums, f" cpdef enum {self.make_typename(decl.name)}:") ns = self.ns if decl.name.name[0] == "Err": self.ns = (tuple([self.ns[0], decl.name.name[0]]) + @@ -170,14 +244,25 @@ class GenPython: for mem in decl.mems: self.genEnumMember(mem) self.ns = ns - self.indent -= 1 def genEnumMember(self, decl: apigen.EnumMember): - self._add(self.enums, f"{_scream(self.ns, decl.name.name)}") + self._add(self.pxd.enums, f" {_scream(self.ns, decl.name.name)}") def genClassDecl(self, decl: apigen.ClassDecl): - self._add(self.typedefs, - f"ctypedef struct {self.make_typename(decl.name)}") + if decl.name.name == ["System"]: + return # skip for now + + cls = self.make_typename(decl.name) + self._add(self.pxd.typedefs, *( + f" ctypedef struct {cls}", + )) + self._add(self.pxd.classes, *( + "", + "", + f"cdef class {cls}_Ptr:", + f" cdef {cls}* _ptr", + f" cdef {cls}* _get(self) except *", + )) self.in_class = decl.name ns = self.ns @@ -209,30 +294,36 @@ class GenPython: def genMethod(self, decl: apigen.Method): name = _relocate_callback(tuple(decl.name.ns), tuple(decl.name.name)) self._add( - self.funcs, - f"cdef {self.make_type(decl.ret)} {'_'.join(name)}({', '.join([self.make_this(decl.const)] + self.make_params(decl.params))})", + self.pxd.funcs, + f" cdef {self.make_type(decl.ret)} {'_'.join(name)}({', '.join([self.make_this(decl.const)] + self.make_params(decl.params))})", ) def genConstructor(self, decl: apigen.Constructor): if not self.in_class: raise Exception("not in class") self._add( - self.funcs, - f"cdef {self.make_typename(self.in_class)}* {self.make_identifier(decl.name)}({', '.join(self.make_params(decl.params))})", + self.pxd.funcs, + f" cdef {self.make_typename(self.in_class)}* {self.make_identifier(decl.name)}({', '.join(self.make_params(decl.params))})", + ) + + self._add( + self.pxd.classes, + f" @staticmethod", + f" cdef {self.make_typename(self.in_class)}* _{'_'.join(decl.name.name)}({', '.join(self.make_cdef_params(decl.params))})", ) def genDestructor(self, decl: apigen.Destructor): if not self.in_class: raise Exception("not in class") self._add( - self.funcs, - f"cdef void {self.make_identifier(decl.name)}({', '.join([self.make_typename(self.in_class) + '* self'] + self.make_params(decl.params))})", + self.pxd.funcs, + f" cdef void {self.make_identifier(decl.name)}({', '.join([self.make_typename(self.in_class) + '* self'] + self.make_params(decl.params))})", ) def genFunction(self, decl: apigen.Function): self._add( - self.funcs, - f"cdef {self.make_type(decl.ret)} {self.make_identifier(decl.name)}({', '.join(self.make_params(decl.params))})", + self.pxd.funcs, + f" cdef {self.make_type(decl.ret)} {self.make_identifier(decl.name)}({', '.join(self.make_params(decl.params))})", ) def genDefine(self, decl: apigen.Define): @@ -240,13 +331,13 @@ class GenPython: def genCallbackTypeDecl(self, decl: apigen.CallbackTypeDecl): self._add( - self.callbacks, - f"ctypedef void {self.make_identifier(decl.name)}({', '.join(self.make_params(decl.params))})", + self.pxd.callbacks, + f" ctypedef void {self.make_identifier(decl.name)}({', '.join(self.make_params(decl.params))})", ) def genIdTypeDecl(self, decl: apigen.IdTypeDecl): - self._add(self.typedefs, - f"ctypedef uint32_t {self.make_typename(decl.name)}") + self._add(self.pxd.typedefs, + f" ctypedef uint32_t {self.make_typename(decl.name)}") def genDecl(self, decl: apigen.Decl): if isinstance(decl, apigen.Namespace): @@ -301,12 +392,12 @@ def main() -> None: tmp.write(b"import apigen\n") tmp.write(model_code.encode("utf-8")) subprocess.run( - ["mypy", "--config-file=/src/workspace/mypy.ini", tmp.name], + ["mypy", "--strict", "--config-file=/src/workspace/mypy.ini", tmp.name], check=True, ) model: apigen.Model = eval(model_code) - for file, content in GenPython().genModel(model).items(): + for file, content in GenCython().genModel(model).items(): with open(os.path.join(out, file), "w") as fh: fh.write(content)