Dev (#8)

* batch_upload_generator() added as iterator option. Can be used for progress status tracking

* batch_upload() updated to use new batch_upload_generator

* clear_db() updated to iteratively cycle larger datasets

* start_logging() and stop_logging() deprecated. Logging updated to use standard logger by name - 'neo4j_uploader'

* upload() marked for deprecation

Author: jalakoo

README.md

@@ -1,10 +1,13 @@
 # Neo4j-uploader
+
 For uploading specially formatted dictionary data to a Neo4j database instance.
 
 ## Installation
+
 `pip install neo4j-uploader`
 
 ## Usage
+
 ```
 from neo4j_uploader import batch_upload
 
@@ -44,7 +47,7 @@ data = {
     ],
     "relationships": [
         {
-            "type":"loves",
+            "type":"LOVES",
             "from_node": {
                 "record_key":"_from_uid",
                 "node_key":"uid",
@@ -93,5 +96,16 @@ data = {
 upload(credentials, data, node_key="name)
 ```
 
+## Progress Tracking
+
+The `batch_upload_generator` function can be used as a generator. Example usage:
+
+```
+for result in batch_upload(config, data):
+    print(f"Upload progress: {result.nodes_created} nodes created")
+
+```
+
 ## Documentation
+
 [Documentation](https://jalakoo.github.io/neo4j-uploader/neo4j_uploader.html) for the current version.

neo4j_uploader/__init__.py

@@ -1,79 +1,14 @@
 import logging
 
-class ModuleLogger(object):
-    '''
-    @private
-    Used to pass logging information to the calling module.
-    '''
-
-    is_enabled : bool = False
-    _logger = None
-
-    def __new__(cls):
-        if not hasattr(cls, 'instance'):
-            cls.instance = super(ModuleLogger, cls).__new__(cls)
-        return cls.instance
-
-    def logger(self):
-        if self.is_enabled is False:
-            return EmptyLogger()
-        
-        if self._logger is None:
-            self._logger = logging.getLogger(__name__)
-            # For custom formatting - This will cause duplicate logs for each message
-            # FORMAT = "[%(asctime)s: %(filename)s: %(lineno)s - %(funcName)20s()] %(message)s"
-            # formatter = logging.Formatter(FORMAT)
-            # mh = logging.StreamHandler()
-            # mh.setFormatter(formatter)
-            # self._logger.addHandler(mh)
-            self._logger.addHandler(logging.NullHandler())
-        
-        return self._logger
-
-    def notset(self, arg: str):
-        if self.is_enabled:
-            return self.logger().notset(arg)
-        pass
-
-    def debug(self, arg: str):
-        if self.is_enabled:
-            return self.logger().debug(arg)
-        pass
-  
-    def info(self, arg: str):
-        if self.is_enabled:
-            return self.logger().info(arg)
-        pass
-
-    def warning(self, arg: str):
-        if self.is_enabled:
-            return self.logger().warning(arg)
-        pass
-
-    def error(self, arg: str):
-        if self.is_enabled:
-            return self.logger().error(arg)
-        pass
-
-    def critical(self, arg: str):
-        if self.is_enabled:
-            return self.logger().critical(arg)
-        pass
-
-class EmptyLogger:
-    '''
-    @private
-    '''
-
-    def notset(self, arg:str):
-        pass
-    def debug(self, arg: str):
-        pass
-    def info(self, arg: str):
-        pass
-    def warning(self, arg: str):
-        pass
-    def error(self, arg: str):
-        pass
-    def critical(self, arg: str):
-        pass
+module_name = "neo4j_uploader"
+logger = logging.getLogger(module_name)
+if not len(logger.handlers):
+    stream_handler = logging.StreamHandler()
+    formatter = logging.Formatter(
+        f"%(levelname)s:{module_name}:%(asctime)s:%(message)s"
+    )
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
+
+# To remove/clear the module stream handler
+# logger.removeHandler(stream_handler)

neo4j_uploader/_logger.py

@@ -1,30 +1,59 @@
-import asyncio
 from neo4j import GraphDatabase
-from neo4j_uploader._logger import ModuleLogger
+from neo4j_uploader._logger import logger
+from typing import Tuple
 
-def validate_credentials(creds: (str, str, str)):
+
+def validate_credentials(creds: Tuple[str, str, str]):
     host, user, password = creds
     with GraphDatabase.driver(host, auth=(user, password)) as driver:
         driver.verify_connectivity()
 
-def upload_query(creds: (str, str, str), query, params={}, database: str = "neo4j"):
+
+def upload_query(
+    creds: Tuple[str, str, str],
+    query,
+    params={},
+    database: str = "neo4j",
+):
     host, user, password = creds
     with GraphDatabase.driver(host, auth=(user, password)) as driver:
-        _, summary, _ = driver.execute_query(query, params, database=database) 
+        _, summary, _ = driver.execute_query(query, params, database=database)
         return summary
 
-def execute_query(creds: (str, str, str), query, params={}, database: str = "neo4j"):
+
+def execute_query(
+    creds: Tuple[str, str, str],
+    query,
+    params={},
+    database: str = "neo4j",
+):
     host, user, password = creds
-    ModuleLogger().debug(f'Using host: {host}, user: {user} to execute query: {query}')
+    logger.debug(f"Using host: {host}, user: {user} to execute query: {query}")
     # Returns a tuple of records, summary, keys
     with GraphDatabase.driver(host, auth=(user, password)) as driver:
         return driver.execute_query(query, params, database=database)
 
-def drop_constraints(creds: (str, str, str), database: str = "neo4j"):
-    query = 'SHOW CONSTRAINTS'
+
+def run_query(
+    uri: str,
+    username: str,
+    password: str,
+    query: str,
+    params: dict = {},
+    database: str = "neo4j",
+):
+    with GraphDatabase.driver(uri, auth=(username, password)) as driver:
+        return driver.execute_query(query, params, database=database)
+
+
+def drop_constraints(
+    creds: Tuple[str, str, str],
+    database: str = "neo4j",
+):
+    query = "SHOW CONSTRAINTS"
     result = execute_query(creds, query, database=database)
 
-    ModuleLogger().info(f"Drop constraints results: {result}")
+    logger.info(f"Drop constraints results: {result}")
 
     # Have to make a drop constraint request for each individually!
     for record in result.records:
@@ -33,31 +62,42 @@ def drop_constraints(creds: (str, str, str), database: str = "neo4j"):
             drop_query = f"DROP CONSTRAINT {constraint_name}"
             drop_result = execute_query(creds, drop_query, database=database)
 
-        ModuleLogger().info(f"Drop constraint {constraint_name} results: {drop_result}")
+        logger.info(f"Drop constraint {constraint_name} results: {drop_result}")
 
     # This should now show empty
     result = execute_query(creds, query, database=database)
 
     return result
 
-def reset(creds : (str, str, str), database: str = "neo4j"):
 
+def reset(
+    creds: Tuple[str, str, str],
+    database: str = "neo4j",
+):
     drop_constraints(creds, database)
 
-    # Clears nodes and relationships - but labels remain and can only be cleared via GUI
-    query = """MATCH (n) DETACH DELETE n"""
-    records, summary, keys = execute_query(creds, query, database=database)
+    deleted_nodes_count = -1
+    while deleted_nodes_count != 0:
+        query = """
+        MATCH (n)
+        OPTIONAL MATCH (n)-[r]-()
+        WITH n, r LIMIT 50000
+        DELETE n, r
+        RETURN count(n) as deletedNodesCount
+        """
+        records, summary, keys = execute_query(creds, query, database=database)
+        deleted_nodes_count = records[0]["deletedNodesCount"]
 
-    ModuleLogger().info(f"Reset results: {summary}")
     return summary
 
+
 def create_new_node_constraints(
-        creds: (str, str, str),
-        node_key: str,
-        database: str = "neo4j"
+    creds: Tuple[str, str, str],
+    node_key: str,
+    database: str = "neo4j",
 ):
     query = f"""CREATE CONSTRAINT node_key IF NOT EXISTS FOR (u:`{node_key}`)\nREQUIRE u.`node_key` IS UNIQUE"""
     result = execute_query(creds, query, database=database)
 
-    ModuleLogger().info(f"Create new constraints results: {result}")
+    logger.info(f"Create new constraints results: {result}")
     return result