From 94cb7c78499e0e4ee0460e2ed154b4b13faabbc6 Mon Sep 17 00:00:00 2001 From: William Grant Date: Wed, 8 Mar 2023 14:22:23 -0500 Subject: [PATCH] QoL improvements to python_to_native_converter and compiler_cache --- typed_python/compiler/compiler_cache.py | 376 ++--- .../compiler/expression_conversion_context.py | 8 +- .../compiler/function_conversion_context.py | 6 +- typed_python/compiler/llvm_compiler.py | 2 +- typed_python/compiler/llvm_compiler_test.py | 2 +- typed_python/compiler/native_ast_to_llvm.py | 24 +- .../compiler/python_to_native_converter.py | 1213 +++++++++-------- typed_python/compiler/runtime.py | 28 +- .../type_wrappers/alternative_wrapper.py | 8 +- .../compiler/type_wrappers/class_wrapper.py | 8 +- .../type_wrappers/const_dict_wrapper.py | 2 +- .../compiler/type_wrappers/dict_wrapper.py | 2 +- .../type_wrappers/held_class_wrapper.py | 4 +- .../compiler/type_wrappers/list_of_wrapper.py | 20 +- .../compiler/type_wrappers/one_of_wrapper.py | 2 +- .../python_typed_function_wrapper.py | 16 +- .../compiler/type_wrappers/set_wrapper.py | 2 +- .../type_wrappers/tuple_of_wrapper.py | 2 +- .../compiler/type_wrappers/tuple_wrapper.py | 4 +- .../type_wrappers/typed_cell_wrapper.py | 2 +- typed_python/compiler/typeof.py | 6 +- 21 files changed, 938 insertions(+), 799 deletions(-) diff --git a/typed_python/compiler/compiler_cache.py b/typed_python/compiler/compiler_cache.py index 90e542989..09f2d1f4a 100644 --- a/typed_python/compiler/compiler_cache.py +++ b/typed_python/compiler/compiler_cache.py @@ -17,26 +17,27 @@ import shutil import llvmlite.ir -from typing import Optional, List +from typing import List from typed_python.compiler.binary_shared_object import LoadedBinarySharedObject, BinarySharedObject from typed_python.compiler.directed_graph import DirectedGraph +from typed_python.compiler.native_function_pointer import NativeFunctionPointer from typed_python.compiler.typed_call_target import TypedCallTarget import typed_python.compiler.native_ast as native_ast from typed_python.SerializationContext import SerializationContext from typed_python import Dict, ListOf -def ensureDirExists(cacheDir): - if not os.path.exists(cacheDir): +def _ensure_dir_exists(cache_dir): + if not os.path.exists(cache_dir): try: - os.makedirs(cacheDir) + os.makedirs(cache_dir) except IOError: # this can happen because of race conditions with # other writers pass - if not os.path.exists(cacheDir): + if not os.path.exists(cache_dir): raise Exception("Failed to create the cache directory.") @@ -56,57 +57,71 @@ class CompilerCache: func_name - The identifier for the function, based on its identity hash. link_name - The identifier for the specific realization of that function, which lives in a specific cache module. + + Attributes: + cache_dir: the relative path to the cache directory. + loaded_binary_shared_objects: a mapping from a module hash to a LoadedBinarySharedObject. + name_to_module_hash: a mapping from link_name (defining a function) to the module hash + it's contained in. (assumes each function only defined in one module). + modules_marked_valid: a set of module hashes which have sucessfully been loaded + using _load_name_manifest_from_stored_module_by_hash. + modules_marked_invalid: a set of module hashes which have marked_invalid in their + associated directory. NB: this attribute is never referenced. + initialised: True if the initialise() method has been run, setting the cache dir and + loading all manifest files. + added_modules: a set of module hashes which have been added during this compiler_caches + lifetime, rather than loaded on initialisation. TODO fix this """ - def __init__(self, cacheDir): - self.cacheDir = cacheDir + def __init__(self, cache_dir): + self.cache_dir = cache_dir - ensureDirExists(cacheDir) + _ensure_dir_exists(cache_dir) - self.loadedBinarySharedObjects = Dict(str, LoadedBinarySharedObject)() + self.loaded_binary_shared_objects = Dict(str, LoadedBinarySharedObject)() self.link_name_to_module_hash = Dict(str, str)() - self.moduleManifestsLoaded = set() + self.module_manifests_loaded = set() # link_names with an associated module in loadedBinarySharedObjects - self.targetsLoaded: Dict[str, TypedCallTarget] = {} + self.targets_loaded: Dict[str, TypedCallTarget] = {} # the set of link_names for functions with linked and validated globals (i.e. ready to be run). - self.targetsValidated = set() + self.targets_validated = set() # the total number of instructions for each link_name - self.targetComplexity = Dict(str, int)() + self.target_complexity = Dict(str, int)() # link_name -> link_name self.function_dependency_graph = DirectedGraph() # dict from link_name to list of global names (should be llvm keys in serialisedGlobalDefinitions) self.global_dependencies = Dict(str, ListOf(str))() self.func_name_to_link_names = Dict(str, ListOf(str))() - for moduleHash in os.listdir(self.cacheDir): - if len(moduleHash) == 40: - self.loadNameManifestFromStoredModuleByHash(moduleHash) + for module_hash in os.listdir(self.cache_dir): + if len(module_hash) == 40: + self._load_name_manifest_from_stored_module_by_hash(module_hash) - def hasSymbol(self, func_name: str) -> bool: - """Returns true if there are any versions of `func_name` in the cache. + def has_symbol(self, func_name: str) -> bool: + """Returns True if there are any versions of `func_name` in the cache. There may be multiple copies in different modules with different link_names. """ return any(link_name in self.link_name_to_module_hash for link_name in self.func_name_to_link_names.get(func_name, [])) - def getTarget(self, func_name: str) -> TypedCallTarget: - if not self.hasSymbol(func_name): + def get_target(self, func_name: str) -> TypedCallTarget: + if not self.has_symbol(func_name): raise ValueError(f'symbol not found for func_name {func_name}') link_name = self._select_link_name(func_name) - self.loadForSymbol(link_name) - return self.targetsLoaded[link_name] + self._load_for_symbol(link_name) + return self.targets_loaded[link_name] - def getIR(self, func_name: str) -> llvmlite.ir.Function: - if not self.hasSymbol(func_name): + def get_IR(self, func_name: str) -> llvmlite.ir.Function: + if not self.has_symbol(func_name): raise ValueError(f'symbol not found for func_name {func_name}') link_name = self._select_link_name(func_name) module_hash = self.link_name_to_module_hash[link_name] - return self.loadedBinarySharedObjects[module_hash].binarySharedObject.functionIRs[func_name] + return self.loaded_binary_shared_objects[module_hash].binarySharedObject.functionIRs[func_name] - def getDefinition(self, func_name: str) -> native_ast.Function: - if not self.hasSymbol(func_name): + def get_definition(self, func_name: str) -> native_ast.Function: + if not self.has_symbol(func_name): raise ValueError(f'symbol not found for func_name {func_name}') link_name = self._select_link_name(func_name) module_hash = self.link_name_to_module_hash[link_name] - serialized_definition = self.loadedBinarySharedObjects[module_hash].binarySharedObject.serializedFunctionDefinitions[func_name] + serialized_definition = self.loaded_binary_shared_objects[module_hash].binarySharedObject.serializedFunctionDefinitions[func_name] return SerializationContext().deserialize(serialized_definition) def _generate_link_name(self, func_name: str, module_hash: str) -> str: @@ -121,97 +136,100 @@ def _select_link_name(self, func_name) -> str: link_name_candidates = self.func_name_to_link_names[func_name] return link_name_candidates[0] - def dependencies(self, link_name: str) -> Optional[List[str]]: - """Returns all the function names that `link_name` depends on""" + def dependencies(self, link_name: str) -> List[str]: + """Returns all the function names that `link_name` depends on, or an empty list.""" return list(self.function_dependency_graph.outgoing(link_name)) - def loadForSymbol(self, linkName: str) -> None: - """Loads the whole module, and any dependant modules, into LoadedBinarySharedObjects""" - moduleHash = self.link_name_to_module_hash[linkName] + def _load_for_symbol(self, link_name: str) -> None: + """Loads the whole module, and any dependant modules, into LoadedBinarySharedObjects. + + Link only the necessary globals for the link_name. + """ + module_hash = self.link_name_to_module_hash[link_name] - self.loadModuleByHash(moduleHash) + self._load_module_by_hash(module_hash) - if linkName not in self.targetsValidated: - self.targetsValidated.add(linkName) - for dependant_func in self.dependencies(linkName): - self.loadForSymbol(dependant_func) + if link_name not in self.targets_validated: + self.targets_validated.add(link_name) + for dependant_func in self.dependencies(link_name): + self._load_for_symbol(dependant_func) - globalsToLink = self.global_dependencies.get(linkName, []) - if globalsToLink: - definitionsToLink = {x: self.loadedBinarySharedObjects[moduleHash].serializedGlobalVariableDefinitions[x] - for x in globalsToLink - } - self.loadedBinarySharedObjects[moduleHash].linkGlobalVariables(definitionsToLink) - if not self.loadedBinarySharedObjects[moduleHash].validateGlobalVariables(definitionsToLink): - raise RuntimeError('failed to validate globals when loading:', linkName) + globals_to_link = self.global_dependencies.get(link_name, []) + if globals_to_link: + definitions_to_link = {x: self.loaded_binary_shared_objects[module_hash].serializedGlobalVariableDefinitions[x] + for x in globals_to_link + } + self.loaded_binary_shared_objects[module_hash].linkGlobalVariables(definitions_to_link) + if not self.loaded_binary_shared_objects[module_hash].validateGlobalVariables(definitions_to_link): + raise RuntimeError('failed to validate globals when loading:', link_name) - def complexityForSymbol(self, func_name: str) -> int: + def complexity_for_symbol(self, func_name: str) -> int: """Get the total number of LLVM instructions for a given symbol.""" try: link_name = self._select_link_name(func_name) - return self.targetComplexity[link_name] + return self.target_complexity[link_name] except KeyError as e: raise ValueError(f'No complexity value cached for {func_name}') from e - def loadModuleByHash(self, moduleHash: str) -> None: + def _load_module_by_hash(self, module_hash: str) -> None: """Load a module by name. Add the module contents to targetsLoaded, generate a LoadedBinarySharedObject, and update the function and global dependency graphs. """ - if moduleHash in self.loadedBinarySharedObjects: + if module_hash in self.loaded_binary_shared_objects: return - targetDir = os.path.join(self.cacheDir, moduleHash) + target_dir = os.path.join(self.cache_dir, module_hash) # TODO (Will) - store these names as module consts, use one .dat only - with open(os.path.join(targetDir, "type_manifest.dat"), "rb") as f: - callTargets = SerializationContext().deserialize(f.read()) - with open(os.path.join(targetDir, "globals_manifest.dat"), "rb") as f: - serializedGlobalVarDefs = SerializationContext().deserialize(f.read()) - with open(os.path.join(targetDir, "native_type_manifest.dat"), "rb") as f: - functionNameToNativeType = SerializationContext().deserialize(f.read()) - with open(os.path.join(targetDir, "submodules.dat"), "rb") as f: + with open(os.path.join(target_dir, "type_manifest.dat"), "rb") as f: + call_targets = SerializationContext().deserialize(f.read()) + with open(os.path.join(target_dir, "globals_manifest.dat"), "rb") as f: + serialized_global_var_defs = SerializationContext().deserialize(f.read()) + with open(os.path.join(target_dir, "native_type_manifest.dat"), "rb") as f: + function_name_to_native_type = SerializationContext().deserialize(f.read()) + with open(os.path.join(target_dir, "submodules.dat"), "rb") as f: submodules = SerializationContext().deserialize(f.read(), ListOf(str)) - with open(os.path.join(targetDir, "function_dependencies.dat"), "rb") as f: + with open(os.path.join(target_dir, "function_dependencies.dat"), "rb") as f: dependency_edgelist = SerializationContext().deserialize(f.read()) - with open(os.path.join(targetDir, "global_dependencies.dat"), "rb") as f: - globalDependencies = SerializationContext().deserialize(f.read()) - with open(os.path.join(targetDir, "function_complexities.dat"), "rb") as f: - functionComplexities = SerializationContext().deserialize(f.read()) - with open(os.path.join(targetDir, "function_irs.dat"), "rb") as f: - functionIRs = SerializationContext().deserialize(f.read()) - with open(os.path.join(targetDir, "function_definitions.dat"), "rb") as f: - functionDefinitions = SerializationContext().deserialize(f.read()) + with open(os.path.join(target_dir, "global_dependencies.dat"), "rb") as f: + global_dependencies = SerializationContext().deserialize(f.read()) + with open(os.path.join(target_dir, "function_complexities.dat"), "rb") as f: + function_complexities = SerializationContext().deserialize(f.read()) + with open(os.path.join(target_dir, "function_irs.dat"), "rb") as f: + function_IRs = SerializationContext().deserialize(f.read()) + with open(os.path.join(target_dir, "function_definitions.dat"), "rb") as f: + function_definitions = SerializationContext().deserialize(f.read()) # load the submodules first for submodule in submodules: - self.loadModuleByHash(submodule) + self._load_module_by_hash(submodule) - modulePath = os.path.join(targetDir, "module.so") + module_path = os.path.join(target_dir, "module.so") loaded = BinarySharedObject.fromDisk( - modulePath, - serializedGlobalVarDefs, - functionNameToNativeType, - globalDependencies, - functionComplexities, - functionIRs, - functionDefinitions - ).loadFromPath(modulePath) - - self.loadedBinarySharedObjects[moduleHash] = loaded - - for func_name, callTarget in callTargets.items(): - link_name = self._generate_link_name(func_name, moduleHash) - assert link_name not in self.targetsLoaded - self.targetsLoaded[link_name] = callTarget - - for func_name, complexity in functionComplexities.items(): - link_name = self._generate_link_name(func_name, moduleHash) - self.targetComplexity[link_name] = complexity - - link_name_global_dependencies = {self._generate_link_name(x, moduleHash): y for x, y in globalDependencies.items()} + module_path, + serialized_global_var_defs, + function_name_to_native_type, + global_dependencies, + function_complexities, + function_IRs, + function_definitions + ).loadFromPath(module_path) + + self.loaded_binary_shared_objects[module_hash] = loaded + + for func_name, call_target in call_targets.items(): + link_name = self._generate_link_name(func_name, module_hash) + assert link_name not in self.targets_loaded + self.targets_loaded[link_name] = call_target + + for func_name, complexity in function_complexities.items(): + link_name = self._generate_link_name(func_name, module_hash) + self.target_complexity[link_name] = complexity + + link_name_global_dependencies = {self._generate_link_name(x, module_hash): y for x, y in global_dependencies.items()} assert not any(key in self.global_dependencies for key in link_name_global_dependencies) self.global_dependencies.update(link_name_global_dependencies) @@ -219,66 +237,80 @@ def loadModuleByHash(self, moduleHash: str) -> None: for function_name, dependant_function_name in dependency_edgelist: self.function_dependency_graph.addEdge(source=function_name, dest=dependant_function_name) - def addModule(self, binarySharedObject, nameToTypedCallTarget, linkDependencies, dependencyEdgelist): + def add_module(self, binary_shared_object, name_to_typed_call_target, link_dependencies, dependency_edge_list): """Add new code to the compiler cache. + Generate the link_name to link_name dependency graph, and write the binary_shared_object to disk + along with its types, and dependency mappings. Then load that object back from disk into the set of + loaded binary_shared_objects, link & validate everything, and update the set of loaded symbols accordingly. Args: - binarySharedObject: a BinarySharedObject containing the actual assembler + binary_shared_object: a BinarySharedObject containing the actual assembler we've compiled. - nameToTypedCallTarget: a dict from func_name to TypedCallTarget telling us + name_to_typed_call_target: a dict from func_name to TypedCallTarget telling us the formal python types for all the objects. - linkDependencies: a set of func_names we depend on directly. (this becomes submodules) - dependencyEdgelist (list): a list of source, dest pairs giving the set of dependency graph for the + link_dependencies: a set of func_names we depend on directly. (this becomes submodules) + dependency_edge_list (list): a list of source, dest pairs giving the set of dependency graph for the module. TODO (Will): the notion of submodules/linkDependencies can be refactored out. """ - hashToUse = SerializationContext().sha_hash(str(uuid.uuid4())).hexdigest + hash_to_use = SerializationContext().sha_hash(str(uuid.uuid4())).hexdigest # the linkDependencies and dependencyEdgelist are in terms of func_name. - dependentHashes = set() - for name in linkDependencies: + dependent_hashes = set() + for name in link_dependencies: link_name = self._select_link_name(name) - dependentHashes.add(self.link_name_to_module_hash[link_name]) + dependent_hashes.add(self.link_name_to_module_hash[link_name]) link_name_dependency_edgelist = [] - for source, dest in dependencyEdgelist: - assert source in binarySharedObject.definedSymbols - source_link_name = self._generate_link_name(source, hashToUse) - if dest in binarySharedObject.definedSymbols: - dest_link_name = self._generate_link_name(dest, hashToUse) + for source, dest in dependency_edge_list: + assert source in binary_shared_object.definedSymbols + source_link_name = self._generate_link_name(source, hash_to_use) + if dest in binary_shared_object.definedSymbols: + dest_link_name = self._generate_link_name(dest, hash_to_use) else: dest_link_name = self._select_link_name(dest) link_name_dependency_edgelist.append([source_link_name, dest_link_name]) - path = self.writeModuleToDisk(binarySharedObject, hashToUse, nameToTypedCallTarget, dependentHashes, link_name_dependency_edgelist) + path = self._write_module_to_disk(binary_shared_object, + hash_to_use, + name_to_typed_call_target, + dependent_hashes, + link_name_dependency_edgelist) - for func_name, complexity in binarySharedObject.functionComplexities.items(): - link_name = self._generate_link_name(func_name, hashToUse) - self.targetComplexity[link_name] = complexity + for func_name, complexity in binary_shared_object.functionComplexities.items(): + link_name = self._generate_link_name(func_name, hash_to_use) + self.target_complexity[link_name] = complexity - self.loadedBinarySharedObjects[hashToUse] = ( - binarySharedObject.loadFromPath(os.path.join(path, "module.so")) + self.loaded_binary_shared_objects[hash_to_use] = ( + binary_shared_object.loadFromPath(os.path.join(path, "module.so")) ) - for func_name in binarySharedObject.definedSymbols: - link_name = self._generate_link_name(func_name, hashToUse) - self.link_name_to_module_hash[link_name] = hashToUse + for func_name in binary_shared_object.definedSymbols: + link_name = self._generate_link_name(func_name, hash_to_use) + self.link_name_to_module_hash[link_name] = hash_to_use self.func_name_to_link_names.setdefault(func_name, []).append(link_name) # link & validate all globals for the new module - self.loadedBinarySharedObjects[hashToUse].linkGlobalVariables() - if not self.loadedBinarySharedObjects[hashToUse].validateGlobalVariables( - self.loadedBinarySharedObjects[hashToUse].serializedGlobalVariableDefinitions): - raise RuntimeError('failed to validate globals in new module:', hashToUse) + self.loaded_binary_shared_objects[hash_to_use].linkGlobalVariables() + if not self.loaded_binary_shared_objects[hash_to_use].validateGlobalVariables( + self.loaded_binary_shared_objects[hash_to_use].serializedGlobalVariableDefinitions): + raise RuntimeError('failed to validate globals in new module:', hash_to_use) - def loadNameManifestFromStoredModuleByHash(self, moduleHash) -> None: - if moduleHash in self.moduleManifestsLoaded: + def _load_name_manifest_from_stored_module_by_hash(self, module_hash) -> None: + """ + Initialise the cache by reading the module from disk, populating the list of cached functions. + + Args: + module_hash: 40-character string representing a directory, with a name_manifest.dat + specifying the functions within. + """ + if module_hash in self.module_manifests_loaded: return - targetDir = os.path.join(self.cacheDir, moduleHash) + targetDir = os.path.join(self.cache_dir, module_hash) # TODO (Will) the name_manifest module_hash is the same throughout so this doesn't need to be a dict. with open(os.path.join(targetDir, "name_manifest.dat"), "rb") as f: @@ -289,9 +321,9 @@ def loadNameManifestFromStoredModuleByHash(self, moduleHash) -> None: self.func_name_to_link_names.setdefault(func_name, []).append(link_name) self.link_name_to_module_hash[link_name] = module_hash - self.moduleManifestsLoaded.add(moduleHash) + self.module_manifests_loaded.add(module_hash) - def writeModuleToDisk(self, binarySharedObject, hashToUse, nameToTypedCallTarget, submodules, dependencyEdgelist): + def _write_module_to_disk(self, binary_shared_object, hash_to_use, name_to_typed_call_target, submodules, dependency_edgelist): """Write out a disk representation of this module. This includes writing both the shared object, a manifest of the function names @@ -304,79 +336,97 @@ def writeModuleToDisk(self, binarySharedObject, hashToUse, nameToTypedCallTarget proper name in case we see conflicts. This allows multiple processes to interact with the compiler cache simultaneously without relying on individual file-level locking. + + Args: + binary_shared_object: The compiled module code. + hash_to_use: The unique hash for the module contents. + name_to_typed_call_target: A dict from linkname to TypedCallTarget telling us + the formal python types for all the objects. + submodules: The list of hashes of dependent modules. + dependency_edgelist: The function dependency graph for all functions in this module. + Returns: + The absolute path to the new module in the cache. """ - targetDir = os.path.join( - self.cacheDir, - hashToUse + target_dir = os.path.join( + self.cache_dir, + hash_to_use ) - assert not os.path.exists(targetDir) + assert not os.path.exists(target_dir) - tempTargetDir = targetDir + "_" + str(uuid.uuid4()) - ensureDirExists(tempTargetDir) + temp_target_dir = target_dir + "_" + str(uuid.uuid4()) + _ensure_dir_exists(temp_target_dir) # write the binary module - with open(os.path.join(tempTargetDir, "module.so"), "wb") as f: - f.write(binarySharedObject.binaryForm) + with open(os.path.join(temp_target_dir, "module.so"), "wb") as f: + f.write(binary_shared_object.binaryForm) # write the manifest. Every TP process using the cache will have to # load the manifest every time, so we try to use compiled code to load it manifest = Dict(str, str)() - for n in binarySharedObject.functionTypes: - manifest[n] = hashToUse + for n in binary_shared_object.functionTypes: + manifest[n] = hash_to_use - with open(os.path.join(tempTargetDir, "name_manifest.dat"), "wb") as f: + with open(os.path.join(temp_target_dir, "name_manifest.dat"), "wb") as f: f.write(SerializationContext().serialize(manifest, Dict(str, str))) - with open(os.path.join(tempTargetDir, "name_manifest.txt"), "w") as f: - for sourceName in manifest: - f.write(sourceName + "\n") + with open(os.path.join(temp_target_dir, "name_manifest.txt"), "w") as f: + for source_name in manifest: + f.write(source_name + "\n") - with open(os.path.join(tempTargetDir, "type_manifest.dat"), "wb") as f: - f.write(SerializationContext().serialize(nameToTypedCallTarget)) + with open(os.path.join(temp_target_dir, "type_manifest.dat"), "wb") as f: + f.write(SerializationContext().serialize(name_to_typed_call_target)) - with open(os.path.join(tempTargetDir, "native_type_manifest.dat"), "wb") as f: - f.write(SerializationContext().serialize(binarySharedObject.functionTypes)) + with open(os.path.join(temp_target_dir, "native_type_manifest.dat"), "wb") as f: + f.write(SerializationContext().serialize(binary_shared_object.functionTypes)) - with open(os.path.join(tempTargetDir, "globals_manifest.dat"), "wb") as f: - f.write(SerializationContext().serialize(binarySharedObject.serializedGlobalVariableDefinitions)) + with open(os.path.join(temp_target_dir, "globals_manifest.dat"), "wb") as f: + f.write(SerializationContext().serialize(binary_shared_object.serializedGlobalVariableDefinitions)) - with open(os.path.join(tempTargetDir, "submodules.dat"), "wb") as f: + with open(os.path.join(temp_target_dir, "submodules.dat"), "wb") as f: f.write(SerializationContext().serialize(ListOf(str)(submodules), ListOf(str))) - with open(os.path.join(tempTargetDir, "function_dependencies.dat"), "wb") as f: - f.write(SerializationContext().serialize(dependencyEdgelist)) + with open(os.path.join(temp_target_dir, "function_dependencies.dat"), "wb") as f: + f.write(SerializationContext().serialize(dependency_edgelist)) - with open(os.path.join(tempTargetDir, "global_dependencies.dat"), "wb") as f: - f.write(SerializationContext().serialize(binarySharedObject.globalDependencies)) + with open(os.path.join(temp_target_dir, "global_dependencies.dat"), "wb") as f: + f.write(SerializationContext().serialize(binary_shared_object.globalDependencies)) - with open(os.path.join(tempTargetDir, "function_complexities.dat"), "wb") as f: - f.write(SerializationContext().serialize(binarySharedObject.functionComplexities)) + with open(os.path.join(temp_target_dir, "function_complexities.dat"), "wb") as f: + f.write(SerializationContext().serialize(binary_shared_object.functionComplexities)) - with open(os.path.join(tempTargetDir, "function_irs.dat"), "wb") as f: - f.write(SerializationContext().serialize(binarySharedObject.functionIRs)) + with open(os.path.join(temp_target_dir, "function_irs.dat"), "wb") as f: + f.write(SerializationContext().serialize(binary_shared_object.functionIRs)) - with open(os.path.join(tempTargetDir, "function_definitions.dat"), "wb") as f: - f.write(SerializationContext().serialize(binarySharedObject.serializedFunctionDefinitions)) + with open(os.path.join(temp_target_dir, "function_definitions.dat"), "wb") as f: + f.write(SerializationContext().serialize(binary_shared_object.serializedFunctionDefinitions)) try: - os.rename(tempTargetDir, targetDir) + os.rename(temp_target_dir, target_dir) except IOError: - if not os.path.exists(targetDir): + if not os.path.exists(target_dir): raise else: - shutil.rmtree(tempTargetDir) + shutil.rmtree(temp_target_dir) - return targetDir + return target_dir - def function_pointer_by_name(self, func_name): + def function_pointer_by_name(self, func_name: str) -> NativeFunctionPointer: + """ + Find the module hash associated with an instance of . If not loaded, load the module, + and return the associated function pointer. + Args: + func_name: the symbol name, normally of the form .. + Returns: + The NativeFunctionPointer for that symbol. + """ linkName = self._select_link_name(func_name) - moduleHash = self.link_name_to_module_hash.get(linkName) - if moduleHash is None: + module_hash = self.link_name_to_module_hash.get(linkName) + if module_hash is None: raise Exception("Can't find a module for " + linkName) - if moduleHash not in self.loadedBinarySharedObjects: - self.loadForSymbol(linkName) + if module_hash not in self.loaded_binary_shared_objects: + self._load_for_symbol(linkName) - return self.loadedBinarySharedObjects[moduleHash].functionPointers[func_name] + return self.loaded_binary_shared_objects[module_hash].functionPointers[func_name] diff --git a/typed_python/compiler/expression_conversion_context.py b/typed_python/compiler/expression_conversion_context.py index 926315960..af0756c2c 100644 --- a/typed_python/compiler/expression_conversion_context.py +++ b/typed_python/compiler/expression_conversion_context.py @@ -119,7 +119,7 @@ def allocateClassMethodDispatchSlot(self, clsType, methodName, retType, argTuple # the first argument indicates whether this is an instance or type-level dispatch assert argTupleType.ElementTypes[0].Value in ('type', 'instance') - identHash = self.converter.hashObjectToIdentity( + identHash = self.converter.hash_object_to_identity( (clsType, methodName, retType, argTupleType, kwargTupleType) ) @@ -1141,7 +1141,7 @@ def call_py_function(self, f, args, kwargs, returnTypeOverload=None): funcGlobals[f.__code__.co_freevars[i]] = f.__closure__[i].cell_contents globalsInCells.append(f.__code__.co_freevars[i]) - call_target = self.functionContext.converter.convert( + call_target = self.functionContext.converter.convert_python_to_native( f.__name__, f.__code__, funcGlobals, @@ -1185,7 +1185,7 @@ def call_overload(self, overload, funcObj, args, kwargs, returnTypeOverload=None else: returnType = typeWrapper(overload.returnType) if overload.returnType is not None else None - call_target = self.functionContext.converter.convert( + call_target = self.functionContext.converter.convert_python_to_native( overload.name, overload.functionCode, overload.realizedGlobals, @@ -1474,7 +1474,7 @@ def namedVariableLookup(self, name): return None def expressionAsFunctionCall(self, name, args, generatingFunction, identity, outputType=None, alwaysRaises=False): - callTarget = self.converter.defineNonPythonFunction( + callTarget = self.converter.define_non_python_function( name, ("expression", identity), typed_python.compiler.function_conversion_context.ExpressionFunctionConversionContext( diff --git a/typed_python/compiler/function_conversion_context.py b/typed_python/compiler/function_conversion_context.py index abf858853..0dc8e7a5b 100644 --- a/typed_python/compiler/function_conversion_context.py +++ b/typed_python/compiler/function_conversion_context.py @@ -621,9 +621,9 @@ def __iter__(self) -> generatorBaseclass: returnSlot.convert_copy_initialize(output) - context.converter.triggerVirtualDestructor(generatorSubclass) + context.converter.trigger_virtual_destructor(generatorSubclass) - context.converter.triggerVirtualMethodInstantiation( + context.converter.trigger_virtual_method_instantiation( generatorSubclass, "__fastnext__", PointerTo(T), @@ -631,7 +631,7 @@ def __iter__(self) -> generatorBaseclass: NamedTuple() ) - context.converter.triggerVirtualMethodInstantiation( + context.converter.trigger_virtual_method_instantiation( generatorSubclass, "__next__", T, diff --git a/typed_python/compiler/llvm_compiler.py b/typed_python/compiler/llvm_compiler.py index b9e31bd6d..100d579e0 100644 --- a/typed_python/compiler/llvm_compiler.py +++ b/typed_python/compiler/llvm_compiler.py @@ -124,7 +124,7 @@ def buildSharedObject(self, functions): serializedGlobalVariableDefinitions, module.functionNameToType, module.globalDependencies, - {name: self.converter.totalFunctionComplexity(name) for name in functions}, + {name: self.converter.total_function_complexity(name) for name in functions}, {name: self.converter._functions_by_name[name] for name in functions}, {name: SerializationContext().serialize(self.converter._function_definitions[name]) for name in functions}, ) diff --git a/typed_python/compiler/llvm_compiler_test.py b/typed_python/compiler/llvm_compiler_test.py index d914bae4f..f8ee572a3 100644 --- a/typed_python/compiler/llvm_compiler_test.py +++ b/typed_python/compiler/llvm_compiler_test.py @@ -154,7 +154,7 @@ def test_loaded_modules_persist(): " return f(x) * 100", "g(1000)", "def get_loaded_modules():", - " return len(Runtime.singleton().converter.loadedUncachedModules)" + " return len(Runtime.singleton().converter.loaded_uncached_modules)" ]) VERSION1 = {'x.py': xmodule} assert evaluateExprInFreshProcess(VERSION1, 'x.get_loaded_modules()') == 1 diff --git a/typed_python/compiler/native_ast_to_llvm.py b/typed_python/compiler/native_ast_to_llvm.py index 310881dee..98ff49f2d 100644 --- a/typed_python/compiler/native_ast_to_llvm.py +++ b/typed_python/compiler/native_ast_to_llvm.py @@ -516,7 +516,7 @@ def __init__(self, # a list of the global LLVM names that the function depends on. self.global_names = [] self.module = module - self.converter = converter + self.converter: Converter = converter self.builder = builder self.arg_assignments = arg_assignments self.output_type = output_type @@ -665,9 +665,9 @@ def namedCallTargetToLLVM(self, target: native_ast.NamedCallTarget) -> TypedLLVM if func.module is not self.module: # first, see if we'd like to inline this module if ( - self.converter.totalFunctionComplexity(target.name) < CROSS_MODULE_INLINE_COMPLEXITY + self.converter.total_function_complexity(target.name) < CROSS_MODULE_INLINE_COMPLEXITY ): - func = self.converter.repeatFunctionInModule(target.name, self.module) + func = self.converter.repeat_function_in_module(target.name, self.module) else: if target.name not in self.external_function_references: self.external_function_references[target.name] = \ @@ -675,14 +675,14 @@ def namedCallTargetToLLVM(self, target: native_ast.NamedCallTarget) -> TypedLLVM func = self.external_function_references[target.name] else: - assert self.compilerCache is not None and self.compilerCache.hasSymbol(target.name) + assert self.compilerCache is not None and self.compilerCache.has_symbol(target.name) # this function is defined in a shared object that we've loaded from a prior # invocation. Again, first make an inlining decision. if ( - self.compilerCache.complexityForSymbol(target.name) < CROSS_MODULE_INLINE_COMPLEXITY + self.compilerCache.complexity_for_symbol(target.name) < CROSS_MODULE_INLINE_COMPLEXITY ): - self.converter.generateDefinition(target.name) - func = self.converter.repeatFunctionInModule(target.name, self.module) + self.converter.generate_definition(target.name) + func = self.converter.repeat_function_in_module(target.name, self.module) else: if target.name not in self.external_function_references: func_type = llvmlite.ir.FunctionType( @@ -1518,7 +1518,7 @@ def __init__(self, compilerCache=None): self.compilerCache = compilerCache - def totalFunctionComplexity(self, name): + def total_function_complexity(self, name): """Return the total number of instructions contained in a function. The function must already have been defined in a prior pass. We use this @@ -1535,19 +1535,19 @@ def totalFunctionComplexity(self, name): return res - def generateDefinition(self, name: str) -> None: + def generate_definition(self, name: str) -> None: """Pull the TypedCallTarget matching `name` from the cache, and use to rebuild the function definition. Add to _function_definitions and _functions_by_name. """ assert self.compilerCache is not None - definition = self.compilerCache.getDefinition(name) - llvm_func = self.compilerCache.getIR(name) + definition = self.compilerCache.get_definition(name) + llvm_func = self.compilerCache.get_IR(name) self._functions_by_name[name] = llvm_func self._function_definitions[name] = definition - def repeatFunctionInModule(self, name, module): + def repeat_function_in_module(self, name, module): """Request that the function given by 'name' be inlined into 'module'. It must already have been defined in another module. diff --git a/typed_python/compiler/python_to_native_converter.py b/typed_python/compiler/python_to_native_converter.py index 131ece7db..afd897d03 100644 --- a/typed_python/compiler/python_to_native_converter.py +++ b/typed_python/compiler/python_to_native_converter.py @@ -12,39 +12,50 @@ # See the License for the specific language governing permissions and # limitations under the License. -import types import logging - -from typed_python.hash import Hash +import types from types import ModuleType -from typing import Dict, Optional -from typed_python import Class -import typed_python.python_ast as python_ast +from typing import Dict, List, Optional, Tuple, Type + +from sortedcontainers import SortedSet + import typed_python._types as _types import typed_python.compiler import typed_python.compiler.native_ast as native_ast -from typed_python.compiler.native_function_pointer import NativeFunctionPointer -from sortedcontainers import SortedSet +import typed_python.python_ast as python_ast +from typed_python import Class +from typed_python.compiler.compiler_cache import CompilerCache from typed_python.compiler.directed_graph import DirectedGraph -from typed_python.compiler.type_wrappers.wrapper import Wrapper -from typed_python.compiler.type_wrappers.class_wrapper import ClassWrapper -from typed_python.compiler.python_object_representation import typedPythonTypeToTypeWrapper -from typed_python.compiler.function_conversion_context import FunctionConversionContext, FunctionOutput, FunctionYield +from typed_python.compiler.function_conversion_context import (FunctionConversionContext, FunctionOutput, FunctionYield) +from typed_python.compiler.llvm_compiler import Compiler from typed_python.compiler.native_function_conversion_context import NativeFunctionConversionContext +from typed_python.compiler.native_function_pointer import NativeFunctionPointer +from typed_python.compiler.python_object_representation import typedPythonTypeToTypeWrapper +from typed_python.compiler.type_wrappers.class_wrapper import ClassWrapper from typed_python.compiler.type_wrappers.python_typed_function_wrapper import ( - PythonTypedFunctionWrapper, CannotBeDetermined, NoReturnTypeSpecified -) + CannotBeDetermined, NoReturnTypeSpecified, PythonTypedFunctionWrapper) +from typed_python.compiler.type_wrappers.wrapper import Wrapper from typed_python.compiler.typed_call_target import TypedCallTarget +from typed_python.hash import Hash +from typed_python.internals import ClassMetaclass + -typeWrapper = lambda t: typed_python.compiler.python_object_representation.typedPythonTypeToTypeWrapper(t) +__all__ = ['PythonToNativeConverter'] + +type_wrapper = lambda t: typed_python.compiler.python_object_representation.typedPythonTypeToTypeWrapper(t) VALIDATE_FUNCTION_DEFINITIONS_STABLE = False class FunctionDependencyGraph: + """ + A wrapper for DirectedGraph with identity levels and the ability to tag nodes for + recomputation. The nodes of the graph are function hashes, and the edges represent + dependency. + """ def __init__(self): - self._dependencies = DirectedGraph() + self.dependency_graph = DirectedGraph() # the search depth in the dependency to find 'identity' # the _first_ time we ever saw it. We prefer to update @@ -58,13 +69,13 @@ def __init__(self): # (priority, node) pairs that need to recompute self._dirty_inflight_functions_with_order = SortedSet(key=lambda pair: pair[0]) - def dropNode(self, node): - self._dependencies.dropNode(node, False) + def drop_node(self, node): + self.dependency_graph.dropNode(node, False) if node in self._identity_levels: del self._identity_levels[node] self._dirty_inflight_functions.discard(node) - def getNextDirtyNode(self): + def get_next_dirty_node(self): while self._dirty_inflight_functions_with_order: priority, identity = self._dirty_inflight_functions_with_order.pop() @@ -73,13 +84,13 @@ def getNextDirtyNode(self): return identity - def addRoot(self, identity, dirty=True): + def add_root(self, identity, dirty=True): if identity not in self._identity_levels: self._identity_levels[identity] = 0 if dirty: - self.markDirty(identity) + self.mark_dirty(identity) - def addEdge(self, caller, callee, dirty=True): + def add_edge(self, caller, callee, dirty=True): if caller not in self._identity_levels: raise Exception(f"unknown identity {caller} found in the graph") @@ -87,21 +98,21 @@ def addEdge(self, caller, callee, dirty=True): self._identity_levels[callee] = self._identity_levels[caller] + 1 if dirty: - self.markDirty(callee, isNew=True) + self.mark_dirty(callee, isNew=True) - self._dependencies.addEdge(caller, callee) + self.dependency_graph.addEdge(caller, callee) - def getNamesDependedOn(self, caller): - return self._dependencies.outgoing(caller) + def get_names_depended_on(self, caller): + return self.dependency_graph.outgoing(caller) - def markDirtyWithLowPriority(self, callee): + def mark_dirty_with_low_priority(self, callee): # mark this dirty, but call it back after new functions. self._dirty_inflight_functions.add(callee) level = self._identity_levels[callee] self._dirty_inflight_functions_with_order.add((-1000000 + level, callee)) - def markDirty(self, callee, isNew=False): + def mark_dirty(self, callee, isNew=False): self._dirty_inflight_functions.add(callee) if isNew: @@ -113,32 +124,77 @@ def markDirty(self, callee, isNew=False): self._dirty_inflight_functions_with_order.add((level, callee)) - def functionReturnSignatureChanged(self, identity): - for caller in self._dependencies.incoming(identity): - self.markDirty(caller) + def function_return_signature_changed(self, identity): + for caller in self.dependency_graph.incoming(identity): + self.mark_dirty(caller) class PythonToNativeConverter: - def __init__(self, llvmCompiler, compilerCache): - object.__init__(self) + """ + What do you do? + - init + + Parses Python functions into an intermediate representation of the AST. These representations + are held in self._definitions, and added how?! + + 'installed' but new (and hence not fully converted) definitions are stored in + _new_native_functions, and are sent downstream when build_and_link_new_module is called. + + + Holds optional + + + Usage is effectively, in compileFunctionOverload, we do convert_typed_function_call, + then demasquerade_call_target_output, then generate_call_converter, then + build_and_link_new_module, + then function_pointer_by_name. + + + x = understanding + + 'identity' is the hexdigest of the function hash + 'link_name' is the symbol name, prefix + func_name + hash - self.llvmCompiler = llvmCompiler - self.compilerCache = compilerCache + Attributes: + llvm_compiler: The WHAT? + compiler_cache: The WHAT? + generate_debug_checks: ? + _all_defined_names: ? + _all_cached_names: + _link_name_for_identity: ? + _identity_for_link_name: ? + _definitions: definition blablabla + _targets: ? + _inflight_definitions: ? + _inflight_function_conversions: ? + _identifier_to_pyfunc: ? + _times_calculated: ? + _new_native_functions: + _visitors: A list of context managers which run on each new function installed with + _install_inflight_functions, giving info on e.g. compilation counts. + _currently_converting: + _dependencies: A directed graph in which nodes are function hashes and edges are a dependence. + """ + + def __init__(self, llvm_compiler: Compiler, compiler_cache: CompilerCache): + self.llvm_compiler = llvm_compiler + self.compiler_cache = compiler_cache # all LoadedModule objects that we have created. We need to keep them alive so # that any python metadata objects the've created stay alive as well. Ultimately, this # may not be the place we put these objects (for instance, you could imagine a # 'dummy' compiler cache or something). But for now, we need to keep them alive. - self.loadedUncachedModules = [] + self.loaded_uncached_modules = [] # if True, then insert additional code to check for undefined behavior. - self.generateDebugChecks = False + self.generate_debug_checks = False - self._link_name_for_identity = {} - self._identity_for_link_name = {} + self._link_name_for_identity: Dict[str, str] = {} + self._identity_for_link_name: Dict[str, str] = {} self._definitions: Dict[str, native_ast.Function] = {} self._targets: Dict[str, TypedCallTarget] = {} - self._inflight_definitions = {} + # NB i don't think the Wrapper is ever actually used. + self._inflight_definitions: Dict[str, Tuple[native_ast.Function, Type[Wrapper]]] = {} self._inflight_function_conversions: Dict[str, FunctionConversionContext] = {} self._identifier_to_pyfunc = {} self._times_calculated = {} @@ -151,302 +207,227 @@ def __init__(self, llvmCompiler, compilerCache): # the identity of the function we're currently evaluating. # we use this to track which functions need to get rebuilt when # other functions change types. - self._currentlyConverting = None + self._currently_converting = None # tuple of (baseClass, childClass, slotIndex) containing # virtual methods that need to get instantiated during the # current compilation unit. self._delayedVMIs = [] - self._delayedDestructors = [] + self._delayed_destructors = [] self._installedVMIs = set() - self._installedDestructors = set() + self._installed_destructors = set() self._dependencies = FunctionDependencyGraph() - def isCurrentlyConverting(self): - return len(self._inflight_function_conversions) > 0 - - def getDefinitionCount(self): - return len(self._definitions) - - def addVisitor(self, visitor): - self._visitors.append(visitor) - - def removeVisitor(self, visitor): - self._visitors.remove(visitor) - - def identityToName(self, identity): - """Convert a function identity to the link-time name for the function. - + def convert_typed_function_call(self, + function_type, + overload_index: int, + input_wrappers: List[Type[Wrapper]], + assert_is_root=False) -> Optional[TypedCallTarget]: + """Does what? Args: - identity - an identity tuple that uniquely identifies the function - + function_type: Normally a subclass of typed_python._types.Function? sometimes not a class at all? + overload_index: The index of function_type.overloads to access, corresponding to a specific set of input argument types + input_wrappers: The input wrappers? TODO not always a Wrapper, sometimes just a type?! + assert_is_root: If True, then assert that no other functions are using + the converter right now. + Trash! TODO fix Returns: - name - the linker name of the native function this represents, or None - if the identity is unknown + bla """ - return self._link_name_for_identity.get(identity) - - def buildAndLinkNewModule(self): - definitions = self.extract_new_function_definitions() - - if not definitions: - return + overload = function_type.overloads[overload_index] - if self.compilerCache is None: - loadedModule = self.llvmCompiler.buildModule(definitions) - loadedModule.linkGlobalVariables() - self.loadedUncachedModules.append(loadedModule) - return + realized_input_wrappers = [] - # get a set of function names that we depend on - externallyUsed = set() - dependency_edgelist = [] + closure_type = function_type.ClosureType - for funcName in definitions: - ident = self._identity_for_link_name.get(funcName) - if ident is not None: - for dep in self._dependencies.getNamesDependedOn(ident): - depLN = self._link_name_for_identity.get(dep) - dependency_edgelist.append([funcName, depLN]) - if depLN not in definitions: - externallyUsed.add(depLN) + for closure_var_name, closure_var_path in overload.closureVarLookups.items(): + realized_input_wrappers.append( + type_wrapper( + PythonTypedFunctionWrapper.closurePathToCellType(closure_var_path, closure_type) + ) + ) - binary = self.llvmCompiler.buildSharedObject(definitions) + realized_input_wrappers.extend(input_wrappers) - self.compilerCache.addModule( - binary, - {name: self.getTarget(name) for name in definitions if self.hasTarget(name)}, - externallyUsed, - dependency_edgelist + return_type = PythonTypedFunctionWrapper.computeFunctionOverloadReturnType( + overload, + input_wrappers, + {} ) - def extract_new_function_definitions(self): - """Return a list of all new function definitions from the last conversion.""" - res = {} - - for u in self._new_native_functions: - res[u] = self._definitions[u] - - self._new_native_functions = set() - - return res + if return_type is CannotBeDetermined: + return_type = object - def identityHashToFunctionName(self, name, identityHash, prefix="tp."): - assert isinstance(name, str) - assert isinstance(identityHash, str) - assert isinstance(prefix, str) + if return_type is NoReturnTypeSpecified: + return_type = None - return prefix + name + "." + identityHash + return self.convert_python_to_native( + overload.name, + overload.functionCode, + overload.realizedGlobals, + overload.functionGlobals, + overload.funcGlobalsInCells, + list(overload.closureVarLookups), + realized_input_wrappers, + return_type, + assert_is_root=assert_is_root + ) - def createConversionContext( + def convert_python_to_native( self, - identity, - funcName, - funcCode, - funcGlobals, - funcGlobalsRaw, - closureVars, + func_name: str, + func_code: types.CodeType, + func_globals: Dict, + func_glocals_raw, + func_globals_from_cells, + closure_vars, input_types, output_type, - conversionType + assert_is_root=False, + conversion_type=None ): - ConverterType = conversionType or FunctionConversionContext - - pyast = self._code_to_ast(funcCode) - - return ConverterType( - self, - funcName, - identity, - input_types, - output_type, - closureVars, - funcGlobals, - funcGlobalsRaw, - pyast.args, - pyast, - ) + """Convert a single pure python function using args of 'input_types'. - def defineLinkName(self, identity, linkName): - if identity in self._link_name_for_identity: - if self._link_name_for_identity[identity] != linkName: - raise Exception( - f"For identity {identity}:\n\n" - f"{self._link_name_for_identity[identity]}\n\n!=\n\n{linkName}" - ) - assert self._identity_for_link_name[linkName] == identity - else: - self._link_name_for_identity[identity] = linkName - self._identity_for_link_name[linkName] = identity + It will return no more than 'output_type'. if output_type is None we produce + the tightest output type possible. - def hasTarget(self, linkName): - return self.getTarget(linkName) is not None + Args: + func_name: the name of the function + func_code: a Code object representing the code to compile + func_globals: the globals object from the relevant function + func_globals_raw: the original globals object (with no merging or filtering done) + which we use to figure out the location of global variables that are not in cells. + func_globals_from_cells: a list of the names that are globals that are actually accessed + as cells. + closure_vars: a list of the names of the variables that are accessed as free variables. TODO check + input_types - a type for each free variable in the function closure, and + then again for each input argument + output_type - the output type of the function, if known. if this is None, + then we use type inference to produce the tightest type we can. + If not None, then we will produce this type or throw an exception. + assert_is_root - if True, then assert that no other functions are using + the converter right now. + conversion_type - if None, this is a normal function conversion. Otherwise, + this must be a subclass of FunctionConversionContext + """ + assert isinstance(func_name, str) + assert isinstance(func_code, types.CodeType) + assert isinstance(func_globals, dict) - def deleteTarget(self, linkName): - self._targets.pop(linkName) + input_types = tuple([typedPythonTypeToTypeWrapper(i) for i in input_types]) - def setTarget(self, linkName, target): - assert (isinstance(target, TypedCallTarget)) - self._targets[linkName] = target + identity_hash = ( + Hash.from_integer(1) + + self.hash_object_to_identity(( + func_code, + func_name, + input_types, + output_type, + closure_vars, + conversion_type + )) + + self._hash_globals(func_globals, func_code, func_globals_from_cells) + ) - def getTarget(self, linkName) -> Optional[TypedCallTarget]: - if linkName in self._targets: - return self._targets[linkName] + assert not identity_hash.isPoison() - if self.compilerCache is not None and self.compilerCache.hasSymbol(linkName): - return self.compilerCache.getTarget(linkName) + identity = identity_hash.hexdigest - return None + name = self._identity_hash_to_function_name(func_name, identity) - def defineNonPythonFunction(self, name, identityTuple, context): - """Define a non-python generating function (if we haven't defined it before already) + self._define_link_name(identity, name) - name - the name to actually give the function. - identityTuple - a unique (sha)hashable tuple - context - a FunctionConversionContext lookalike + if identity not in self._identifier_to_pyfunc: + self._identifier_to_pyfunc[identity] = ( + func_name, func_code, func_globals, closure_vars, input_types, output_type, conversion_type + ) - returns a TypedCallTarget, or None if it's not known yet - """ - identity = self.hashObjectToIdentity(identityTuple).hexdigest - linkName = self.identityHashToFunctionName(name, identity, "runtime.") + is_root = len(self._inflight_function_conversions) == 0 - self.defineLinkName(identity, linkName) + if assert_is_root: + assert is_root - target = self.getTarget(linkName) + target = self._get_target(name) - if self._currentlyConverting is not None: - self._dependencies.addEdge(self._currentlyConverting, identity, dirty=(target is None)) + if self._currently_converting is not None: + self._dependencies.add_edge(self._currently_converting, identity, dirty=(target is None)) else: - self._dependencies.addRoot(identity, dirty=(target is None)) + self._dependencies.add_root(identity, dirty=(target is None)) if target is not None: return target - self._inflight_function_conversions[identity] = context - - if context.knownOutputType() is not None or context.alwaysRaises(): - self.setTarget( - linkName, - self.getTypedCallTarget( - name, - context.getInputTypes(), - context.knownOutputType(), - alwaysRaises=context.alwaysRaises(), - functionMetadata=context.functionMetadata, - ) + if identity not in self._inflight_function_conversions: + function_converter = self._create_conversion_context( + identity, + func_name, + func_code, + func_globals, + func_glocals_raw, + closure_vars, + input_types, + output_type, + conversion_type ) + self._inflight_function_conversions[identity] = function_converter - if self._currentlyConverting is None: - # force the function to resolve immediately - self._resolveAllInflightFunctions() - self._installInflightFunctions() - self._inflight_function_conversions.clear() - - return self.getTarget(linkName) - - def defineNativeFunction(self, name, identity, input_types, output_type, generatingFunction): - """Define a native function if we haven't defined it before already. - - name - the name to actually give the function. - identity - a tuple consisting of strings, ints, type wrappers, and tuples of same - input_types - list of Wrapper objects for the incoming types - output_type - Wrapper object for the output type, or None if the function doesn't - ever return - generatingFunction - a function producing a native_function_definition. - It should accept an expression_conversion_context, an expression for the output - if it's not pass-by-value (or None if it is), and a bunch of TypedExpressions - and produce code that always ends in a terminal expression, (or if it's pass by value, - flows off the end of the function) - - returns a TypedCallTarget. 'generatingFunction' may call this recursively if it wants. - """ - if output_type is not None: - output_type = typeWrapper(output_type) - - input_types = [typeWrapper(x) for x in input_types] - - identity = ( - Hash.from_integer(2) + - self.hashObjectToIdentity(identity) + - self.hashObjectToIdentity(output_type) + - self.hashObjectToIdentity(input_types) - ).hexdigest - - return self.defineNonPythonFunction( - name, - identity, - NativeFunctionConversionContext( - self, input_types, output_type, generatingFunction, identity - ) - ) + if is_root: + try: + self._resolve_all_inflight_functions() + self._install_inflight_functions() + return self._get_target(name) + finally: + self._inflight_function_conversions.clear() - def getTypedCallTarget(self, name, input_types, output_type, alwaysRaises=False, functionMetadata=None): - native_input_types = [a.getNativePassingType() for a in input_types if not a.is_empty] - if output_type is None: - native_output_type = native_ast.Type.Void() - elif output_type.is_pass_by_ref: - native_input_types = [output_type.getNativePassingType()] + native_input_types - native_output_type = native_ast.Type.Void() else: - native_output_type = output_type.getNativeLayoutType() - - res = TypedCallTarget( - native_ast.NamedCallTarget( - name=name, - arg_types=native_input_types, - output_type=native_output_type, - external=False, - varargs=False, - intrinsic=False, - can_throw=True - ), - input_types, - output_type, - alwaysRaises=alwaysRaises, - functionMetadata=functionMetadata - ) - - return res - - def _code_to_ast(self, f): - return python_ast.convertFunctionToAlgebraicPyAst(f) + # above us on the stack, we are walking a set of function conversions. + # if we have ever calculated this function before, we'll have a call + # target with an output type and we can return that. Otherwise we have to + # return None, which will cause callers to replace this with a throw + # until we have had a chance to do a full pass of conversion. + if self._get_target(name) is not None: + raise RuntimeError(f"Unexpected conversion error for {name}") + return None - def demasqueradeCallTargetOutput(self, callTarget: TypedCallTarget): + def demasquerade_call_target_output(self, call_target: TypedCallTarget) -> Optional[TypedCallTarget]: """Ensure we are returning the correct 'interpreterType' from callTarget. In some cases, we may return a 'masquerade' type in compiled code. This is fine for other compiled code, but the interpreter needs us to transform the result back to the right interpreter type. For instance, we may be returning a *args tuple. + + Args: + call_target: the input TypedCallTarget to demasquerade (or return unchanged) Returns: a new TypedCallTarget where the output type has the right return type. """ - if callTarget.output_type is None: - return callTarget + if call_target.output_type is None: + return call_target - if callTarget.output_type.interpreterTypeRepresentation == callTarget.output_type.typeRepresentation: - return callTarget + if call_target.output_type.interpreterTypeRepresentation == call_target.output_type.typeRepresentation: + return call_target def generator(context, out, *args): assert out is not None, "we should have an output because no masquerade types are pass-by-value" - res = context.call_typed_call_target(callTarget, args) + res = context.call_typed_call_target(call_target, args) out.convert_copy_initialize(res.convert_masquerade_to_untyped()) - res = self.defineNativeFunction( - "demasquerade_" + callTarget.name, - ("demasquerade", callTarget.name), - callTarget.input_types, - typeWrapper(callTarget.output_type.interpreterTypeRepresentation), + res = self.define_native_function( + "demasquerade_" + call_target.name, + ("demasquerade", call_target.name), + call_target.input_types, + type_wrapper(call_target.output_type.interpreterTypeRepresentation), generator ) return res - def generateCallConverter(self, callTarget: TypedCallTarget): + def generate_call_converter(self, call_target: TypedCallTarget) -> str: """Given a call target that's optimized for llvm-level dispatch (with individual arguments packed into registers), produce a (native) call-target that we can dispatch to from our C extension, where arguments are packed into @@ -459,50 +440,46 @@ def generateCallConverter(self, callTarget: TypedCallTarget): where X is the union of A1, A2, etc. Args: - callTarget - a TypedCallTarget giving the function we need + call_target: a TypedCallTarget giving the function we need to generate an alternative entrypoint for Returns: the linker name of the defined native function """ - identifier = "call_converter_" + callTarget.name - linkName = callTarget.name + ".dispatch" + identifier = "call_converter_" + call_target.name + link_name = call_target.name + ".dispatch" - # # we already made a definition for this in this process so don't do it again - if linkName in self._definitions: - return linkName + # we already made a definition for this in this process so don't do it again + if link_name in self._definitions: + return link_name - # # we already defined it in another process so don't do it again - if self.compilerCache is not None and self.compilerCache.hasSymbol(linkName): - return linkName + # we already defined it in another process so don't do it again + if self.compiler_cache is not None and self.compiler_cache.has_symbol(link_name): + return link_name # N.B. there aren't targets for call converters. We make the definition directly. - - # if self.getTarget(linkName): - # return linkName - args = [] - for i in range(len(callTarget.input_types)): - if not callTarget.input_types[i].is_empty: - argtype = callTarget.input_types[i].getNativeLayoutType() + for i in range(len(call_target.input_types)): + if not call_target.input_types[i].is_empty: + argtype = call_target.input_types[i].getNativeLayoutType() - untypedPtr = native_ast.var('input').ElementPtrIntegers(i).load() + untyped_ptr = native_ast.var('input').ElementPtrIntegers(i).load() - if callTarget.input_types[i].is_pass_by_ref: + if call_target.input_types[i].is_pass_by_ref: # we've been handed a pointer, and it's already a pointer - args.append(untypedPtr.cast(argtype.pointer())) + args.append(untyped_ptr.cast(argtype.pointer())) else: - args.append(untypedPtr.cast(argtype.pointer()).load()) + args.append(untyped_ptr.cast(argtype.pointer()).load()) - if callTarget.output_type is not None and callTarget.output_type.is_pass_by_ref: - body = callTarget.call( - native_ast.var('return').cast(callTarget.output_type.getNativeLayoutType().pointer()), + if call_target.output_type is not None and call_target.output_type.is_pass_by_ref: + body = call_target.call( + native_ast.var('return').cast(call_target.output_type.getNativeLayoutType().pointer()), *args ) else: - body = callTarget.call(*args) + body = call_target.call(*args) - if not (callTarget.output_type is None or callTarget.output_type.is_empty): - body = native_ast.var('return').cast(callTarget.output_type.getNativeLayoutType().pointer()).store(body) + if not (call_target.output_type is None or call_target.output_type.is_empty): + body = native_ast.var('return').cast(call_target.output_type.getNativeLayoutType().pointer()).store(body) body = native_ast.FunctionBody.Internal(body=body) @@ -515,175 +492,198 @@ def generateCallConverter(self, callTarget: TypedCallTarget): output_type=native_ast.Type.Void() ) - self._link_name_for_identity[identifier] = linkName - self._identity_for_link_name[linkName] = identifier + self._link_name_for_identity[identifier] = link_name + self._identity_for_link_name[link_name] = identifier - self._definitions[linkName] = definition - self._new_native_functions.add(linkName) + self._definitions[link_name] = definition + self._new_native_functions.add(link_name) - return linkName + return link_name - def _resolveAllInflightFunctions(self): - while True: - identity = self._dependencies.getNextDirtyNode() - if not identity: - return + def build_and_link_new_module(self) -> None: + """Pull all in-flight conversions and compile as a module. + This grabs all new functions, where the python->native conversion has been completed, + and collates these + their dependencies and global variables into a module, performing the + native -> llvm conversion. The list of in-flight functions is cleared once parsed to avoid + double-compilation. + """ - functionConverter = self._inflight_function_conversions[identity] + definitions = self._extract_new_function_definitions() - hasDefinitionBeforeConversion = identity in self._inflight_definitions + if not definitions: + return - try: - self._currentlyConverting = identity + if self.compiler_cache is None: + loaded_module = self.llvm_compiler.buildModule(definitions) # TODO handle None + loaded_module.linkGlobalVariables() + self.loaded_uncached_modules.append(loaded_module) + return - self._times_calculated[identity] = self._times_calculated.get(identity, 0) + 1 + # get a set of function names that we depend on + externally_used = set() + dependency_edgelist = [] - # this calls back into convert with dependencies - # they get registered as dirty - nativeFunction, actual_output_type = functionConverter.convertToNativeFunction() + for func_name in definitions: + ident = self._identity_for_link_name.get(func_name) + if ident is not None: + for dep in self._dependencies.get_names_depended_on(ident): + depLN = self._link_name_for_identity.get(dep) + dependency_edgelist.append([func_name, depLN]) + if depLN not in definitions: + externally_used.add(depLN) - if nativeFunction is not None: - self._inflight_definitions[identity] = (nativeFunction, actual_output_type) - except Exception: - for i in self._inflight_function_conversions: - if i in self._link_name_for_identity: - name = self._link_name_for_identity[i] - if self.hasTarget(name): - self.deleteTarget(name) - ln = self._link_name_for_identity.pop(i) - self._identity_for_link_name.pop(ln) + binary = self.llvm_compiler.buildSharedObject(definitions) - self._dependencies.dropNode(i) + self.compiler_cache.add_module( + binary, + {name: self._get_target(name) for name in definitions if self._has_target(name)}, + externally_used, + dependency_edgelist + ) - self._inflight_function_conversions.clear() - self._inflight_definitions.clear() - raise - finally: - self._currentlyConverting = None + def define_non_python_function(self, name: str, identity_tuple: Tuple, context) -> Optional[TypedCallTarget]: + """Define a non-python generating function (if we haven't defined it before already) - dirtyUpstream = False + name: the name to actually give the function. + identity_tuple: a unique (sha)hashable tuple + context: a FunctionConversionContext lookalike - # figure out whether we ought to recalculate all the upstream nodes of this - # node. we do that if we get a definition and we didn't have one before, or if - # our type stability changed - if nativeFunction is not None: - if not hasDefinitionBeforeConversion: - dirtyUpstream = True + Returns: + A TypedCallTarget, or None if it's not known yet + """ + identity = self.hash_object_to_identity(identity_tuple).hexdigest + linkName = self._identity_hash_to_function_name(name, identity, "runtime.") - if functionConverter.typesAreUnstable(): - functionConverter.resetTypeInstabilityFlag() - self._dependencies.markDirtyWithLowPriority(identity) - dirtyUpstream = True + self._define_link_name(identity, linkName) - name = self._link_name_for_identity[identity] + target = self._get_target(linkName) + + if self._currently_converting is not None: + self._dependencies.add_edge(self._currently_converting, identity, dirty=(target is None)) + else: + self._dependencies.add_root(identity, dirty=(target is None)) - self.setTarget( + if target is not None: + return target + + self._inflight_function_conversions[identity] = context + + if context.knownOutputType() is not None or context.alwaysRaises(): + self._set_target( + linkName, + self._get_typed_call_target( name, - self.getTypedCallTarget( - name, - functionConverter._input_types, - actual_output_type, - alwaysRaises=functionConverter.alwaysRaises(), - functionMetadata=functionConverter.functionMetadata, - ), + context.getInputTypes(), + context.knownOutputType(), + always_raises=context.alwaysRaises(), + function_metadata=context.functionMetadata, ) + ) - if dirtyUpstream: - self._dependencies.functionReturnSignatureChanged(identity) - - def triggerVirtualDestructor(self, instanceType): - self._delayedDestructors.append(instanceType) + if self._currently_converting is None: + # force the function to resolve immediately + self._resolve_all_inflight_functions() + self._install_inflight_functions() + self._inflight_function_conversions.clear() - def triggerVirtualMethodInstantiation(self, instanceType, methodName, returnType, argTupleType, kwargTupleType): - """Instantiate a virtual method as part of this batch of compilation. + return self._get_target(linkName) - Normally, compiling 'virtual' methods (method instantiations on subclasses - that are known as a base class to the compiler) happens lazily. In some cases - (for instance, generators) we want to force compilation of specific methods - when we define the class, since otherwise we end up with irregular performance - because we're lazily triggering an expensive operation. + def define_native_function(self, + name: str, + identity: Tuple, + input_types, + output_type, + generatingFunction) -> Optional[TypedCallTarget]: + """Define a native function if we haven't defined it before already. - This method forces the current compilation operation to compile and link - instantiating 'methodName' on instances of 'instanceType'. - """ - for baseClass in instanceType.__mro__: - if issubclass(baseClass, Class) and baseClass is not Class: - slot = _types.allocateClassMethodDispatch( - baseClass, - methodName, - returnType, - argTupleType, - kwargTupleType - ) - self._delayedVMIs.append( - (baseClass, instanceType, slot) - ) + Args: + name: the name to actually give the function. + identity: a tuple consisting of strings, ints, type wrappers, and tuples of same + input_types: list of Wrapper objects for the incoming types + output_type: Wrapper object for the output type, or None if the function doesn't + ever return + generatingFunction: a function producing a native_function_definition. + It should accept an expression_conversion_context, an expression for the output + if it's not pass-by-value (or None if it is), and a bunch of TypedExpressions + and produce code that always ends in a terminal expression, (or if it's pass by value, + flows off the end of the function) - def flushDelayedVMIs(self): - while self._delayedVMIs or self._delayedDestructors: - vmis = self._delayedVMIs - self._delayedVMIs = [] + Returns: + A TypedCallTarget. 'generatingFunction' may call this recursively if it wants. + """ + if output_type is not None: + output_type = type_wrapper(output_type) - for baseClass, instanceClass, dispatchSlot in vmis: - self.compileSingleClassDispatch(baseClass, instanceClass, dispatchSlot) + input_types = [type_wrapper(x) for x in input_types] - delayedDestructors = self._delayedDestructors - self._delayedDestructors = [] + identity = ( + Hash.from_integer(2) + + self.hash_object_to_identity(identity) + + self.hash_object_to_identity(output_type) + + self.hash_object_to_identity(input_types) + ).hexdigest - for T in delayedDestructors: - self.compileClassDestructor(T) + return self.define_non_python_function( + name, + identity, + NativeFunctionConversionContext( + self, input_types, output_type, generatingFunction, identity + ) + ) - def compileSingleClassDispatch(self, interfaceClass, implementingClass, slotIndex): - if (interfaceClass, implementingClass, slotIndex) in self._installedVMIs: + def compile_single_class_dispatch(self, interface_class: ClassMetaclass, implementing_class: ClassMetaclass, slot_index: int): + if (interface_class, implementing_class, slot_index) in self._installedVMIs: return - name, retType, argTypeTuple, kwargTypeTuple = _types.getClassMethodDispatchSignature(interfaceClass, implementingClass, slotIndex) + name, ret_type, arg_type_tuple, kwarg_type_tuple = _types.getClassMethodDispatchSignature(interface_class, + implementing_class, + slot_index) # we are compiling the function 'name' in 'implementingClass' to be installed when # viewing an instance of 'implementingClass' as 'interfaceClass' that's function # 'name' called with signature '(*argTypeTuple, **kwargTypeTuple) -> retType' - typedCallTarget = ClassWrapper.compileVirtualMethodInstantiation( + typed_call_target = ClassWrapper.compileVirtualMethodInstantiation( self, - interfaceClass, - implementingClass, + interface_class, + implementing_class, name, - retType, - argTypeTuple, - kwargTypeTuple + ret_type, + arg_type_tuple, + kwarg_type_tuple ) - assert typedCallTarget is not None + assert typed_call_target is not None - self.buildAndLinkNewModule() + self.build_and_link_new_module() - fp = self.functionPointerByName(typedCallTarget.name) + fp = self.function_pointer_by_name(typed_call_target.name) if fp is None: - raise Exception(f"Couldn't find a function pointer for {typedCallTarget.name}") + raise Exception(f"Couldn't find a function pointer for {typed_call_target.name}") - _types.installClassMethodDispatch(interfaceClass, implementingClass, slotIndex, fp.fp) + _types.installClassMethodDispatch(interface_class, implementing_class, slot_index, fp.fp) self._installedVMIs.add( - (interfaceClass, implementingClass, slotIndex) + (interface_class, implementing_class, slot_index) ) - def compileClassDestructor(self, cls): - if cls in self._installedDestructors: + def compile_class_destructor(self, cls): + if cls in self._installed_destructors: return - typedCallTarget = typeWrapper(cls).compileDestructor(self) + typed_call_target = type_wrapper(cls).compileDestructor(self) - assert typedCallTarget is not None + assert typed_call_target is not None - self.buildAndLinkNewModule() + self.build_and_link_new_module() - fp = self.functionPointerByName(typedCallTarget.name) + fp = self.function_pointer_by_name(typed_call_target.name) _types.installClassDestructor(cls, fp.fp) - self._installedDestructors.add(cls) + self._installed_destructors.add(cls) - def functionPointerByName(self, func_name) -> NativeFunctionPointer: + def function_pointer_by_name(self, func_name) -> NativeFunctionPointer: """Find a NativeFunctionPointer for a given link-time name. Args: @@ -692,55 +692,15 @@ def functionPointerByName(self, func_name) -> NativeFunctionPointer: Returns: a NativeFunctionPointer or None """ - if self.compilerCache is None: + if self.compiler_cache is None: # the llvm compiler holds it all - return self.llvmCompiler.function_pointer_by_name(func_name) + return self.llvm_compiler.function_pointer_by_name(func_name) else: # the llvm compiler is just building shared objects, but the # compiler cache has all the pointers. - return self.compilerCache.function_pointer_by_name(func_name) - - def convertTypedFunctionCall(self, functionType, overloadIx, inputWrappers, assertIsRoot=False): - overload = functionType.overloads[overloadIx] - - realizedInputWrappers = [] - - closureType = functionType.ClosureType - - for closureVarName, closureVarPath in overload.closureVarLookups.items(): - realizedInputWrappers.append( - typeWrapper( - PythonTypedFunctionWrapper.closurePathToCellType(closureVarPath, closureType) - ) - ) - - realizedInputWrappers.extend(inputWrappers) - - returnType = PythonTypedFunctionWrapper.computeFunctionOverloadReturnType( - overload, - inputWrappers, - {} - ) - - if returnType is CannotBeDetermined: - returnType = object + return self.compiler_cache.function_pointer_by_name(func_name) - if returnType is NoReturnTypeSpecified: - returnType = None - - return self.convert( - overload.name, - overload.functionCode, - overload.realizedGlobals, - overload.functionGlobals, - overload.funcGlobalsInCells, - list(overload.closureVarLookups), - realizedInputWrappers, - returnType, - assertIsRoot=assertIsRoot - ) - - def hashObjectToIdentity(self, hashable, isModuleVal=False): + def hash_object_to_identity(self, hashable, is_module_val=False): if isinstance(hashable, Hash): return hashable @@ -753,14 +713,14 @@ def hashObjectToIdentity(self, hashable, isModuleVal=False): if hashable is None: return Hash.from_integer(1) + Hash.from_integer(0) - if isinstance(hashable, (dict, list)) and isModuleVal: + if isinstance(hashable, (dict, list)) and is_module_val: # don't look into dicts and lists at module level return Hash.from_integer(2) if isinstance(hashable, (tuple, list)): res = Hash.from_integer(len(hashable)) for t in hashable: - res += self.hashObjectToIdentity(t, isModuleVal) + res += self.hash_object_to_identity(t, is_module_val) return res if isinstance(hashable, Wrapper): @@ -768,172 +728,301 @@ def hashObjectToIdentity(self, hashable, isModuleVal=False): return Hash(_types.identityHash(hashable)) - def hashGlobals(self, funcGlobals, code, funcGlobalsFromCells): - """Hash a given piece of code's accesses to funcGlobals. + def is_currently_converting(self): + return len(self._inflight_function_conversions) > 0 - We're trying to make sure that if we have a reference to module 'x' - in our globals, but we only ever use 'x' by writing 'x.f' or 'x.g', then - we shouldn't depend on the entirety of the definition of 'x'. + def get_definition_count(self): + return len(self._definitions) + + def add_visitor(self, visitor): + self._visitors.append(visitor) + + def remove_visitor(self, visitor): + self._visitors.remove(visitor) + + def identity_to_name(self, identity): + """Convert a function identity to the link-time name for the function. + + Args: + identity - an identity tuple that uniquely identifies the function + + Returns: + name - the linker name of the native function this represents, or None + if the identity is unknown """ + return self._link_name_for_identity.get(identity) - res = Hash.from_integer(0) + def trigger_virtual_destructor(self, instance_type): + self._delayed_destructors.append(instance_type) - for dotSeq in _types.getCodeGlobalDotAccesses(code): - res += self.hashDotSeq(dotSeq, funcGlobals) + def trigger_virtual_method_instantiation(self, instance_type, method_name, return_type, arg_tuple_type, kwarg_tuple_type): + """Instantiate a virtual method as part of this batch of compilation. - for globalName in funcGlobalsFromCells: - res += self.hashDotSeq([globalName], funcGlobals) + Normally, compiling 'virtual' methods (method instantiations on subclasses + that are known as a base class to the compiler) happens lazily. In some cases + (for instance, generators) we want to force compilation of specific methods + when we define the class, since otherwise we end up with irregular performance + because we're lazily triggering an expensive operation. - return res + This method forces the current compilation operation to compile and link + instantiating 'methodName' on instances of 'instanceType'. + """ + for baseClass in instance_type.__mro__: + if issubclass(baseClass, Class) and baseClass is not Class: + slot = _types.allocateClassMethodDispatch( + baseClass, + method_name, + return_type, + arg_tuple_type, + kwarg_tuple_type + ) - def hashDotSeq(self, dotSeq, funcGlobals): - if not dotSeq or dotSeq[0] not in funcGlobals: - return Hash.from_integer(0) + self._delayedVMIs.append( + (baseClass, instance_type, slot) + ) - item = funcGlobals[dotSeq[0]] + def flush_delayed_VMIs(self): + while self._delayedVMIs or self._delayed_destructors: + vmis = self._delayedVMIs + self._delayedVMIs = [] - if not isinstance(item, ModuleType) or len(dotSeq) == 1: - return Hash.from_string(dotSeq[0]) + self.hashObjectToIdentity(item, True) + for baseClass, instanceClass, dispatchSlot in vmis: + self.compile_single_class_dispatch(baseClass, instanceClass, dispatchSlot) - if not hasattr(item, dotSeq[1]): - return Hash.from_integer(0) + delayedDestructors = self._delayed_destructors + self._delayed_destructors = [] - return Hash.from_string(dotSeq[0] + "." + dotSeq[1]) + self.hashObjectToIdentity(getattr(item, dotSeq[1]), True) + for T in delayedDestructors: + self.compile_class_destructor(T) - def convert( + def _extract_new_function_definitions(self): + """Return a list of all new function definitions from the last conversion.""" + res = {} + + for u in self._new_native_functions: + res[u] = self._definitions[u] + + self._new_native_functions = set() + + return res + + def _identity_hash_to_function_name(self, name, identity_hash, prefix="tp."): + assert isinstance(name, str) + assert isinstance(identity_hash, str) + assert isinstance(prefix, str) + + return prefix + name + "." + identity_hash + + def _create_conversion_context( self, + identity, funcName, funcCode, funcGlobals, funcGlobalsRaw, - funcGlobalsFromCells, closureVars, input_types, output_type, - assertIsRoot=False, - conversionType=None + conversionType ): - """Convert a single pure python function using args of 'input_types'. + ConverterType = conversionType or FunctionConversionContext - It will return no more than 'output_type'. if output_type is None we produce - the tightest output type possible. + pyast = self._code_to_ast(funcCode) - Args: - funcName - the name of the function - funcCode - a Code object representing the code to compile - funcGlobals - the globals object from the relevant function - funcGlobalsRaw - the original globals object (with no merging or filtering done) - which we use to figure out the location of global variables that are not in cells. - funcGlobalsFromCells - a list of the names that are globals that are actually accessed - as cells. - input_types - a type for each free variable in the function closure, and - then again for each input argument - output_type - the output type of the function, if known. if this is None, - then we use type inference to produce the tightest type we can. - If not None, then we will produce this type or throw an exception. - assertIsRoot - if True, then assert that no other functions are using - the converter right now. - conversionType - if None, this is a normal function conversion. Otherwise, - this must be a subclass of FunctionConversionContext - """ - assert isinstance(funcName, str) - assert isinstance(funcCode, types.CodeType) - assert isinstance(funcGlobals, dict) + return ConverterType( + self, + funcName, + identity, + input_types, + output_type, + closureVars, + funcGlobals, + funcGlobalsRaw, + pyast.args, + pyast, + ) - input_types = tuple([typedPythonTypeToTypeWrapper(i) for i in input_types]) + def _define_link_name(self, identity, link_name): + if identity in self._link_name_for_identity: + if self._link_name_for_identity[identity] != link_name: + raise Exception( + f"For identity {identity}:\n\n" + f"{self._link_name_for_identity[identity]}\n\n!=\n\n{link_name}" + ) + assert self._identity_for_link_name[link_name] == identity + else: + self._link_name_for_identity[identity] = link_name + self._identity_for_link_name[link_name] = identity - identityHash = ( - Hash.from_integer(1) - + self.hashObjectToIdentity(( - funcCode, - funcName, - input_types, - output_type, - closureVars, - conversionType - )) + - self.hashGlobals(funcGlobals, funcCode, funcGlobalsFromCells) - ) + def _has_target(self, link_name): + return self._get_target(link_name) is not None - assert not identityHash.isPoison() + def _delete_target(self, link_name): + self._targets.pop(link_name) - identity = identityHash.hexdigest + def _set_target(self, link_name, target): + assert (isinstance(target, TypedCallTarget)) + self._targets[link_name] = target - name = self.identityHashToFunctionName(funcName, identity) + def _get_target(self, link_name) -> Optional[TypedCallTarget]: + if link_name in self._targets: + return self._targets[link_name] - self.defineLinkName(identity, name) + if self.compiler_cache is not None and self.compiler_cache.has_symbol(link_name): + return self.compiler_cache.get_target(link_name) - if identity not in self._identifier_to_pyfunc: - self._identifier_to_pyfunc[identity] = ( - funcName, funcCode, funcGlobals, closureVars, input_types, output_type, conversionType - ) + return None - isRoot = len(self._inflight_function_conversions) == 0 + def _get_typed_call_target(self, name, input_types, output_type, always_raises=False, function_metadata=None) -> TypedCallTarget: + native_input_types = [a.getNativePassingType() for a in input_types if not a.is_empty] + if output_type is None: + native_output_type = native_ast.Type.Void() + elif output_type.is_pass_by_ref: + native_input_types = [output_type.getNativePassingType()] + native_input_types + native_output_type = native_ast.Type.Void() + else: + native_output_type = output_type.getNativeLayoutType() - if assertIsRoot: - assert isRoot + res = TypedCallTarget( + native_ast.NamedCallTarget( + name=name, + arg_types=native_input_types, + output_type=native_output_type, + external=False, + varargs=False, + intrinsic=False, + can_throw=True + ), + input_types, + output_type, + alwaysRaises=always_raises, + functionMetadata=function_metadata + ) - target = self.getTarget(name) + return res - if self._currentlyConverting is not None: - self._dependencies.addEdge(self._currentlyConverting, identity, dirty=(target is None)) - else: - self._dependencies.addRoot(identity, dirty=(target is None)) + def _code_to_ast(self, f): + return python_ast.convertFunctionToAlgebraicPyAst(f) - if target is not None: - return target + def _resolve_all_inflight_functions(self): + while True: + identity = self._dependencies.get_next_dirty_node() + if not identity: + return - if identity not in self._inflight_function_conversions: - functionConverter = self.createConversionContext( - identity, - funcName, - funcCode, - funcGlobals, - funcGlobalsRaw, - closureVars, - input_types, - output_type, - conversionType - ) - self._inflight_function_conversions[identity] = functionConverter + functionConverter = self._inflight_function_conversions[identity] + + hasDefinitionBeforeConversion = identity in self._inflight_definitions - if isRoot: try: - self._resolveAllInflightFunctions() - self._installInflightFunctions() - return self.getTarget(name) - finally: + self._currently_converting = identity + + self._times_calculated[identity] = self._times_calculated.get(identity, 0) + 1 + + # this calls back into convert with dependencies + # they get registered as dirty + nativeFunction, actual_output_type = functionConverter.convertToNativeFunction() + + if nativeFunction is not None: + self._inflight_definitions[identity] = (nativeFunction, actual_output_type) + except Exception: + for i in self._inflight_function_conversions: + if i in self._link_name_for_identity: + name = self._link_name_for_identity[i] + if self._has_target(name): + self._delete_target(name) + ln = self._link_name_for_identity.pop(i) + self._identity_for_link_name.pop(ln) + + self._dependencies.drop_node(i) + self._inflight_function_conversions.clear() + self._inflight_definitions.clear() + raise + finally: + self._currently_converting = None - else: - # above us on the stack, we are walking a set of function conversions. - # if we have ever calculated this function before, we'll have a call - # target with an output type and we can return that. Otherwise we have to - # return None, which will cause callers to replace this with a throw - # until we have had a chance to do a full pass of conversion. - if self.getTarget(name) is not None: - raise RuntimeError(f"Unexpected conversion error for {name}") - return None + dirtyUpstream = False + + # figure out whether we ought to recalculate all the upstream nodes of this + # node. we do that if we get a definition and we didn't have one before, or if + # our type stability changed + if nativeFunction is not None: + if not hasDefinitionBeforeConversion: + dirtyUpstream = True + + if functionConverter.typesAreUnstable(): + functionConverter.resetTypeInstabilityFlag() + self._dependencies.mark_dirty_with_low_priority(identity) + dirtyUpstream = True + + name = self._link_name_for_identity[identity] + + self._set_target( + name, + self._get_typed_call_target( + name, + functionConverter._input_types, + actual_output_type, + always_raises=functionConverter.alwaysRaises(), + function_metadata=functionConverter.functionMetadata, + ), + ) + + if dirtyUpstream: + self._dependencies.function_return_signature_changed(identity) + + def _hash_globals(self, funcGlobals, code, funcGlobalsFromCells): + """Hash a given piece of code's accesses to funcGlobals. + + We're trying to make sure that if we have a reference to module 'x' + in our globals, but we only ever use 'x' by writing 'x.f' or 'x.g', then + we shouldn't depend on the entirety of the definition of 'x'. + """ + + res = Hash.from_integer(0) + + for dotSeq in _types.getCodeGlobalDotAccesses(code): + res += self._hash_dot_seq(dotSeq, funcGlobals) + + for globalName in funcGlobalsFromCells: + res += self._hash_dot_seq([globalName], funcGlobals) + + return res + + def _hash_dot_seq(self, dotSeq, funcGlobals): + if not dotSeq or dotSeq[0] not in funcGlobals: + return Hash.from_integer(0) + + item = funcGlobals[dotSeq[0]] + + if not isinstance(item, ModuleType) or len(dotSeq) == 1: + return Hash.from_string(dotSeq[0]) + self.hash_object_to_identity(item, True) + + if not hasattr(item, dotSeq[1]): + return Hash.from_integer(0) + + return Hash.from_string(dotSeq[0] + "." + dotSeq[1]) + self.hash_object_to_identity(getattr(item, dotSeq[1]), True) - def _installInflightFunctions(self): + def _install_inflight_functions(self): """Add all function definitions corresponding to keys in inflight_function_conversions to the relevant dictionaries.""" if VALIDATE_FUNCTION_DEFINITIONS_STABLE: # this should always be true, but its expensive so we have it off by default for identifier, functionConverter in self._inflight_function_conversions.items(): try: - self._currentlyConverting = identifier + self._currently_converting = identifier nativeFunction, actual_output_type = functionConverter.convertToNativeFunction() assert nativeFunction == self._inflight_definitions[identifier] finally: - self._currentlyConverting = None + self._currently_converting = None for identifier, functionConverter in self._inflight_function_conversions.items(): outboundTargets = [] - for outboundFuncId in self._dependencies.getNamesDependedOn(identifier): + for outboundFuncId in self._dependencies.get_names_depended_on(identifier): name = self._link_name_for_identity[outboundFuncId] - target = self.getTarget(name) + target = self._get_target(name) if target is not None: outboundTargets.append(target) else: @@ -978,7 +1067,7 @@ def _installInflightFunctions(self): if identifier not in self._inflight_definitions: raise Exception( f"Expected a definition for {identifier} depended on by:\n" - + "\n".join(" " + str(i) for i in self._dependencies._dependencies.incoming(identifier)) + + "\n".join(" " + str(i) for i in self._dependencies.dependency_graph.incoming(identifier)) ) nativeFunction, actual_output_type = self._inflight_definitions.get(identifier) diff --git a/typed_python/compiler/runtime.py b/typed_python/compiler/runtime.py index a339c2be1..47cc082df 100644 --- a/typed_python/compiler/runtime.py +++ b/typed_python/compiler/runtime.py @@ -231,10 +231,10 @@ def __init__(self): self.verbosityLevel = 0 def addEventVisitor(self, visitor: RuntimeEventVisitor): - self.converter.addVisitor(visitor) + self.converter.add_visitor(visitor) def removeEventVisitor(self, visitor: RuntimeEventVisitor): - self.converter.removeVisitor(visitor) + self.converter.remove_visitor(visitor) @staticmethod def passingTypeForValue(arg): @@ -313,7 +313,7 @@ def compileFunctionOverload(self, functionType, overloadIx, arguments, arguments t0 = time.time() t1 = None t2 = None - defCount = self.converter.getDefinitionCount() + defCount = self.converter.get_definition_count() with self.lock: inputWrappers = [] @@ -329,24 +329,24 @@ def compileFunctionOverload(self, functionType, overloadIx, arguments, arguments self.timesCompiled += 1 - callTarget = self.converter.convertTypedFunctionCall( + callTarget = self.converter.convert_typed_function_call( functionType, overloadIx, inputWrappers, - assertIsRoot=True + assert_is_root=True ) - callTarget = self.converter.demasqueradeCallTargetOutput(callTarget) + callTarget = self.converter.demasquerade_call_target_output(callTarget) assert callTarget is not None - wrappingCallTargetName = self.converter.generateCallConverter(callTarget) + wrappingCallTargetName = self.converter.generate_call_converter(callTarget) t1 = time.time() - self.converter.buildAndLinkNewModule() + self.converter.build_and_link_new_module() t2 = time.time() - fp = self.converter.functionPointerByName(wrappingCallTargetName) + fp = self.converter.function_pointer_by_name(wrappingCallTargetName) overload._installNativePointer( fp.fp, @@ -354,7 +354,7 @@ def compileFunctionOverload(self, functionType, overloadIx, arguments, arguments [i.typeRepresentation for i in callTarget.input_types] ) - self.converter.flushDelayedVMIs() + self.converter.flush_delayed_VMIs() return callTarget finally: @@ -363,14 +363,14 @@ def compileFunctionOverload(self, functionType, overloadIx, arguments, arguments f"typed_python runtime spent {time.time()-t0:.3f} seconds " + (f"({t2 - t1:.3f})" if t2 is not None else "") + " adding " + - f"{self.converter.getDefinitionCount() - defCount} functions." + f"{self.converter.get_definition_count() - defCount} functions." ) def compileClassDispatch(self, interfaceClass, implementingClass, slotIndex): t0 = time.time() with self.lock: - self.converter.compileSingleClassDispatch( + self.converter.compile_single_class_dispatch( interfaceClass, implementingClass, slotIndex ) @@ -392,7 +392,7 @@ def compileClassDestructor(self, cls): t0 = time.time() with self.lock: - self.converter.compileClassDestructor(cls) + self.converter.compile_class_destructor(cls) if self.verbosityLevel > 0: print( @@ -505,7 +505,7 @@ def Entrypoint(pyFunc): # check if we are already in the middle of the compilation process, due to the Entrypointed # code being called through a module import, and throw an error if so. if is_importing(): - compiling_func = Runtime.singleton().converter._currentlyConverting + compiling_func = Runtime.singleton().converter._currently_converting compiling_func_link_name = Runtime.singleton().converter._link_name_for_identity[compiling_func] error_message = f"Can't import Entrypointed code {pyFunc.__module__}.{pyFunc.__qualname__} \ while {compiling_func_link_name} is being compiled." diff --git a/typed_python/compiler/type_wrappers/alternative_wrapper.py b/typed_python/compiler/type_wrappers/alternative_wrapper.py index dfdddd14b..bb9f7d131 100644 --- a/typed_python/compiler/type_wrappers/alternative_wrapper.py +++ b/typed_python/compiler/type_wrappers/alternative_wrapper.py @@ -103,7 +103,7 @@ def convert_typeof(self, context, instance): typeWrapper(SubclassOf(self.typeRepresentation)), lambda outPtr: outPtr.expr.store( - context.converter.defineNativeFunction( + context.converter.define_native_function( "concrete_typeof_" + str(self.typeRepresentation), ('concrete_typeof', self), [self], @@ -306,7 +306,7 @@ def convert_default_initialize(self, context, instance): def on_refcount_zero(self, context, instance): return ( - context.converter.defineNativeFunction( + context.converter.define_native_function( "destructor_" + str(self.typeRepresentation), ('destructor', self), [self], @@ -364,7 +364,7 @@ def convert_attribute(self, context, instance, attribute, nocheck=False): else: outputType = mergeTypeWrappers(possibleTypes) - native = context.converter.defineNativeFunction( + native = context.converter.define_native_function( 'getattr(' + self.typeRepresentation.__name__ + ", " + attribute + ")", ('getattr', self, attribute), [self], @@ -526,7 +526,7 @@ def convert_type_call(self, context, typeInst, args, kwargs): return context.push( self, lambda new_alt: - context.converter.defineNativeFunction( + context.converter.define_native_function( 'construct(' + str(self) + ")", ('util', self, 'construct'), tupletype.ElementTypes, diff --git a/typed_python/compiler/type_wrappers/class_wrapper.py b/typed_python/compiler/type_wrappers/class_wrapper.py index 6f95dec17..42eeb4b35 100644 --- a/typed_python/compiler/type_wrappers/class_wrapper.py +++ b/typed_python/compiler/type_wrappers/class_wrapper.py @@ -528,7 +528,7 @@ def on_refcount_zero(self, context, instance): ) def compileDestructor(self, converter): - return converter.defineNativeFunction( + return converter.define_native_function( "destructor_" + str(self.typeRepresentation), ('destructor', self), [self], @@ -841,7 +841,7 @@ def funcObj(cls, *args, **kwargs): argSignatureStrings = [str(x) for x in matchArgTypes[1:]] argSignatureStrings.extend([f"{k}={v}" for k, v in kwargTypes.items()]) - dispatchToOverloads = context.converter.defineNativeFunction( + dispatchToOverloads = context.converter.define_native_function( f'call_method.{self}.{methodName}({",".join(argSignatureStrings)})', ('call_method', self, methodName, tuple(matchArgTypes), tuple(kwargTypes.items())), list(argTypes) + list(kwargTypes.values()), @@ -926,7 +926,7 @@ def makeOverloadDispatcher(overload, outputType, conversionLevel): assert isinstance(overloadRetType, type), type(overloadRetType) - testSingleOverloadForm = context.converter.defineNativeFunction( + testSingleOverloadForm = context.converter.define_native_function( f'call_overload.{self}.{methodName}.{overloadIndex}.' f'{conversionLevel.LEVEL}.{argTypes[1:]}.{kwargTypes}->{overloadRetType}', ('call_overload', self, methodName, overloadIndex, conversionLevel.LEVEL, @@ -1393,7 +1393,7 @@ def convert_type_call(self, context, typeInst, args, kwargs): return context.push( self, lambda new_class: - context.converter.defineNativeFunction( + context.converter.define_native_function( 'construct(' + self.typeRepresentation.__name__ + ")(" + ",".join([ (argNames[i] + '=' if argNames[i] is not None else "") + diff --git a/typed_python/compiler/type_wrappers/const_dict_wrapper.py b/typed_python/compiler/type_wrappers/const_dict_wrapper.py index 7186c62fa..aea3cb0b3 100644 --- a/typed_python/compiler/type_wrappers/const_dict_wrapper.py +++ b/typed_python/compiler/type_wrappers/const_dict_wrapper.py @@ -406,7 +406,7 @@ def on_refcount_zero(self, context, instance): return runtime_functions.free.call(instance.nonref_expr.cast(native_ast.UInt8Ptr)) else: return ( - context.converter.defineNativeFunction( + context.converter.define_native_function( "destructor_" + str(self.constDictType), ('destructor', self), [self], diff --git a/typed_python/compiler/type_wrappers/dict_wrapper.py b/typed_python/compiler/type_wrappers/dict_wrapper.py index 3689460ef..4c65ef35c 100644 --- a/typed_python/compiler/type_wrappers/dict_wrapper.py +++ b/typed_python/compiler/type_wrappers/dict_wrapper.py @@ -145,7 +145,7 @@ def on_refcount_zero(self, context, instance): assert instance.isReference return ( - context.converter.defineNativeFunction( + context.converter.define_native_function( "destructor_" + str(self.typeRepresentation), ('destructor', self), [self], diff --git a/typed_python/compiler/type_wrappers/held_class_wrapper.py b/typed_python/compiler/type_wrappers/held_class_wrapper.py index 7bcfb8642..e5f0d254a 100644 --- a/typed_python/compiler/type_wrappers/held_class_wrapper.py +++ b/typed_python/compiler/type_wrappers/held_class_wrapper.py @@ -174,7 +174,7 @@ def convert_type_call(self, context, typeInst, args, kwargs): return context.push( self, lambda new_class: - context.converter.defineNativeFunction( + context.converter.define_native_function( 'construct(' + self.typeRepresentation.__name__ + ")(" + ",".join([ (argNames[i] + '=' if argNames[i] is not None else "") + @@ -469,7 +469,7 @@ def convert_comparison(self, context, left, op, right): if not right.expr_type.can_convert_to_type(left.expr_type, ConversionLevel.Signature): return context.constant(False) - native = context.converter.defineNativeFunction( + native = context.converter.define_native_function( f'held_class_equality_check({left.expr_type} -> {right.expr_type})', ('held_class_equality_check', left.expr_type, right.expr_type), [left.expr_type, right.expr_type], diff --git a/typed_python/compiler/type_wrappers/list_of_wrapper.py b/typed_python/compiler/type_wrappers/list_of_wrapper.py index b101be9ed..942972a2a 100644 --- a/typed_python/compiler/type_wrappers/list_of_wrapper.py +++ b/typed_python/compiler/type_wrappers/list_of_wrapper.py @@ -77,7 +77,7 @@ def convert_method_call(self, context, instance, methodname, args, kwargs): if count is None: return - native = context.converter.defineNativeFunction( + native = context.converter.define_native_function( 'pop(' + self.typeRepresentation.__name__ + ")", ('util', self, 'pop'), [self, int], @@ -104,7 +104,7 @@ def convert_method_call(self, context, instance, methodname, args, kwargs): return context.pushPod( None, - context.converter.defineNativeFunction( + context.converter.define_native_function( 'resize(' + self.typeRepresentation.__name__ + ")", ('util', self, 'resize'), [self, int], @@ -123,7 +123,7 @@ def convert_method_call(self, context, instance, methodname, args, kwargs): return context.pushPod( None, - context.converter.defineNativeFunction( + context.converter.define_native_function( 'resize(' + self.typeRepresentation.__name__ + ")", ('util', self, 'resize'), [self, int, self.underlyingWrapperType], @@ -144,7 +144,7 @@ def convert_method_call(self, context, instance, methodname, args, kwargs): return context.pushPod( None, - context.converter.defineNativeFunction( + context.converter.define_native_function( 'append(' + self.typeRepresentation.__name__ + ")", ('util', self, 'append'), [self, self.underlyingWrapperType], @@ -158,7 +158,7 @@ def convert_method_call(self, context, instance, methodname, args, kwargs): return context.push( self, lambda out: - context.converter.defineNativeFunction( + context.converter.define_native_function( 'copy(' + self.typeRepresentation.__name__ + ")", ('util', self, 'copy'), [self], @@ -170,7 +170,7 @@ def convert_method_call(self, context, instance, methodname, args, kwargs): if len(args) == 0: return context.pushPod( None, - context.converter.defineNativeFunction( + context.converter.define_native_function( 'clear(' + self.typeRepresentation.__name__ + ")", ('util', self, 'clear'), [self], @@ -187,7 +187,7 @@ def convert_method_call(self, context, instance, methodname, args, kwargs): return context.pushPod( None, - context.converter.defineNativeFunction( + context.converter.define_native_function( 'reserve(' + self.typeRepresentation.__name__ + ")", ('util', self, 'reserve'), [self, int], @@ -199,7 +199,7 @@ def convert_method_call(self, context, instance, methodname, args, kwargs): if len(args) == 0: return context.pushPod( int, - context.converter.defineNativeFunction( + context.converter.define_native_function( 'reserved(' + self.typeRepresentation.__name__ + ")", ('util', self, 'reserved'), [self], @@ -346,7 +346,7 @@ def generateReserved(self, context, out, listInst): def convert_default_initialize(self, context, tgt): context.pushEffect( - context.converter.defineNativeFunction( + context.converter.define_native_function( 'empty(' + self.typeRepresentation.__name__ + ")", ('util', self, 'empty'), [], @@ -399,7 +399,7 @@ def convert_type_call(self, context, typeInst, args, kwargs): return context.push( self, lambda out: - context.converter.defineNativeFunction( + context.converter.define_native_function( 'copy(' + str(self) + "," + str(args[0].expr_type) + ")", ('util', self, 'copy'), [args[0].expr_type], diff --git a/typed_python/compiler/type_wrappers/one_of_wrapper.py b/typed_python/compiler/type_wrappers/one_of_wrapper.py index e39867c5f..7d1b5a10e 100644 --- a/typed_python/compiler/type_wrappers/one_of_wrapper.py +++ b/typed_python/compiler/type_wrappers/one_of_wrapper.py @@ -474,7 +474,7 @@ def convert_to_type_with_target(self, context, expr, targetVal, conversionLevel, def convert_to_self_with_target(self, context, targetVal, otherExpr, conversionLevel, mayThrowOnFailure=False): assert targetVal.isReference - native = context.converter.defineNativeFunction( + native = context.converter.define_native_function( f'type_convert({otherExpr.expr_type} -> {targetVal.expr_type}, conversionLevel={conversionLevel.LEVEL})', ('type_convert', otherExpr.expr_type, targetVal.expr_type, conversionLevel.LEVEL), [PointerTo(self.typeRepresentation), otherExpr.expr_type], diff --git a/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py b/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py index 8ebcde74a..f347a2767 100644 --- a/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py +++ b/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py @@ -342,7 +342,7 @@ def convert_comprehension(self, context, instance, ConvertionContextType): # import this wrapper. Note that we have to import it here to break the import cycles. # there's definitely a better way to organize this code. - singleConvertedOverload = context.functionContext.converter.convert( + singleConvertedOverload = context.functionContext.converter.convert_python_to_native( overload.name, overload.functionCode, overload.realizedGlobals, @@ -351,7 +351,7 @@ def convert_comprehension(self, context, instance, ConvertionContextType): list(overload.closureVarLookups), [typeWrapper(self.closurePathToCellType(path, closureType)) for path in overload.closureVarLookups.values()], None, - conversionType=ConvertionContextType + conversion_type=ConvertionContextType ) if not singleConvertedOverload: @@ -507,7 +507,7 @@ def convert_call(self, context, left, args, kwargs): # just one overload will do. We can just instantiate this particular function # with a signature that comes from the method overload signature itself. - singleConvertedOverload = context.functionContext.converter.convert( + singleConvertedOverload = context.functionContext.converter.convert_python_to_native( overload.name, overload.functionCode, overload.realizedGlobals, @@ -634,7 +634,7 @@ def compileCall( argNames = [None for _ in argTypes] + list(kwargTypes) - res = converter.defineNativeFunction( + res = converter.define_native_function( f'implement_function.{self}{argTypes}.{kwargTypes}->{returnType}', ('implement_function.', self, returnType, self, tuple(argTypes), tuple(kwargTypes.items())), ([self] if provideClosureArgument else []) + list(argTypes) + list(kwargTypes.values()), @@ -729,7 +729,7 @@ def determinePossibleReturnTypes(converter, func, argTypes, kwargTypes): kwargTypes ) - callTarget = converter.convert( + callTarget = converter.convert_python_to_native( o.name, o.functionCode, o.realizedGlobals, @@ -799,7 +799,7 @@ def makeOverloadImplementor(overload, conversionLevel): overloadIndex = overload.index - testSingleOverloadForm = context.converter.defineNativeFunction( + testSingleOverloadForm = context.converter.define_native_function( f'check_can_call_overload.{self}.{overloadIndex}.{conversionLevel}.{argTypes}.{kwargTypes}', ('check_can_call_overload', self, overloadIndex, conversionLevel.LEVEL, tuple(argTypes), tuple(kwargTypes.items())), @@ -893,7 +893,7 @@ def makeOverloadImplementor(overload, conversionLevel, returnType, forceConvertT overloadRetType = returnType.typeRepresentation if overloadRetType is None: - testSingleOverloadForm = context.converter.defineNativeFunction( + testSingleOverloadForm = context.converter.define_native_function( f'implement_overload.{self}.{overloadIndex}.{conversionLevel}.{argTypes}.{kwargTypes}-> ', ('implement_overload', self, overloadIndex, conversionLevel.LEVEL, overloadRetType, tuple(argTypes), tuple(kwargTypes.items())), @@ -914,7 +914,7 @@ def makeOverloadImplementor(overload, conversionLevel, returnType, forceConvertT + args ) else: - testSingleOverloadForm = context.converter.defineNativeFunction( + testSingleOverloadForm = context.converter.define_native_function( f'implement_overload.{self}.{overloadIndex}.{conversionLevel}.{argTypes}' f'.{kwargTypes}->{overloadRetType}', ('implement_overload', self, overloadIndex, conversionLevel.LEVEL, diff --git a/typed_python/compiler/type_wrappers/set_wrapper.py b/typed_python/compiler/type_wrappers/set_wrapper.py index 4227498f9..62e55b348 100644 --- a/typed_python/compiler/type_wrappers/set_wrapper.py +++ b/typed_python/compiler/type_wrappers/set_wrapper.py @@ -321,7 +321,7 @@ def on_refcount_zero(self, context, instance): assert instance.isReference return ( - context.converter.defineNativeFunction( + context.converter.define_native_function( "destructor_" + str(self.typeRepresentation), ('destructor', self), [self], diff --git a/typed_python/compiler/type_wrappers/tuple_of_wrapper.py b/typed_python/compiler/type_wrappers/tuple_of_wrapper.py index f1f7488b4..d84cf8931 100644 --- a/typed_python/compiler/type_wrappers/tuple_of_wrapper.py +++ b/typed_python/compiler/type_wrappers/tuple_of_wrapper.py @@ -339,7 +339,7 @@ def on_refcount_zero(self, context, instance): assert instance.isReference return ( - context.converter.defineNativeFunction( + context.converter.define_native_function( "destructor_" + str(self.typeRepresentation), ('destructor', self), [self], diff --git a/typed_python/compiler/type_wrappers/tuple_wrapper.py b/typed_python/compiler/type_wrappers/tuple_wrapper.py index b5801d5c2..394de93fc 100644 --- a/typed_python/compiler/type_wrappers/tuple_wrapper.py +++ b/typed_python/compiler/type_wrappers/tuple_wrapper.py @@ -500,7 +500,7 @@ def convert_to_self_with_target(self, context, targetVal, sourceVal, conversionL return context.constant(True) else: - native = context.converter.defineNativeFunction( + native = context.converter.define_native_function( f'type_convert({sourceVal.expr_type} -> {targetVal.expr_type}, conversionLevel={conversionLevel.LEVEL})', ('type_convert', sourceVal.expr_type, targetVal.expr_type, conversionLevel.LEVEL), [self.typeRepresentation, sourceVal.expr_type], @@ -603,7 +603,7 @@ def makeContains(context, unused, instance, toFind): context.pushReturnValue(context.constant(False)) - containsMethod = context.converter.defineNativeFunction( + containsMethod = context.converter.define_native_function( f'tuple.contains.{self}.{toFind.expr_type}', ('tuple.contains', self, toFind.expr_type), (self, toFind.expr_type), diff --git a/typed_python/compiler/type_wrappers/typed_cell_wrapper.py b/typed_python/compiler/type_wrappers/typed_cell_wrapper.py index 93b58c469..990001f2d 100644 --- a/typed_python/compiler/type_wrappers/typed_cell_wrapper.py +++ b/typed_python/compiler/type_wrappers/typed_cell_wrapper.py @@ -76,7 +76,7 @@ def on_refcount_zero(self, context, instance): assert instance.isReference return ( - context.converter.defineNativeFunction( + context.converter.define_native_function( "destructor_" + str(self.typeRepresentation), ('destructor', self), [self], diff --git a/typed_python/compiler/typeof.py b/typed_python/compiler/typeof.py index 270f13ff8..ce1af35d9 100644 --- a/typed_python/compiler/typeof.py +++ b/typed_python/compiler/typeof.py @@ -38,7 +38,7 @@ def __call__(self, *args, **kwargs): converter = typed_python.compiler.runtime.Runtime.singleton().converter - if callTarget is None and converter.isCurrentlyConverting(): + if callTarget is None and converter.is_currently_converting(): # return the 'empty' OneOf, which can't actually be constructed. return typed_python.OneOf() @@ -61,11 +61,11 @@ def resultTypeForCall(self, argTypes, kwargTypes): converter = typed_python.compiler.runtime.Runtime.singleton().converter - callTarget = converter.convertTypedFunctionCall( + callTarget = converter.convert_typed_function_call( type(funcObj), 0, argumentSignature, - assertIsRoot=False + assert_is_root=False ) return callTarget