Worked on JSON serializer helper

Author: joachimmetz

acstore/containers/manager.py

@@ -75,7 +75,7 @@ def GetSchema(cls, container_type):
     container_class = cls._attribute_container_classes.get(
         container_type, None)
     if not container_class:
-      raise ValueError(f'Unsupported container type: {container_type:s}')
+      raise ValueError(f'Unsupported container type: {container_type!s}')
 
     return getattr(container_class, 'SCHEMA', {})
 

acstore/helpers/json_serializer.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+"""Attribute container JSON serializer."""
+
+from acstore.containers import manager as containers_manager
+from acstore.helpers import schema as schema_helper
+
+
+class AttributeContainerJSONSerializer(object):
+  """Attribute container JSON serializer."""
+
+  _CONTAINERS_MANAGER = containers_manager.AttributeContainersManager
+
+  @classmethod
+  def ConvertAttributeContainerToJSON(cls, attribute_container):
+    """Converts an attribute container object into a JSON dictioary.
+
+    The resulting dictionary of the JSON serialized objects consists of:
+    {
+        '__type__': 'AttributeContainer'
+        '__container_type__': ...
+        ...
+    }
+
+    Here '__type__' indicates the object base type. In this case
+    'AttributeContainer'.
+
+    '__container_type__' indicates the container type and rest of the elements
+    of the dictionary that make up the attributes of the container.
+
+    Args:
+      attribute_container (AttributeContainer): attribute container.
+
+    Returns:
+      dict[str, object]: JSON serialized objects.
+    """
+    try:
+      schema = cls._CONTAINERS_MANAGER.GetSchema(
+          attribute_container.CONTAINER_TYPE)
+    except ValueError:
+      schema = {}
+
+    json_dict = {
+        '__type__': 'AttributeContainer',
+        '__container_type__': attribute_container.CONTAINER_TYPE}
+
+    for attribute_name, attribute_value in attribute_container.GetAttributes():
+      data_type = schema.get(attribute_name, None)
+      if data_type:
+        serializer = schema_helper.SchemaHelper.GetAttributeSerializer(
+            data_type, 'json')
+
+        attribute_value = serializer.SerializeValue(attribute_value)
+
+      # JSON will not serialize certain runtime types like set, therefore
+      # these are cast to list first.
+      if isinstance(attribute_value, set):
+        attribute_value = list(attribute_value)
+
+      json_dict[attribute_name] = attribute_value
+
+    return json_dict
+
+  @classmethod
+  def ConvertJSONToAttributeContainer(cls, json_dict):
+    """Converts a JSON dictionary into an attribute container object.
+
+    The dictionary of the JSON serialized objects consists of:
+    {
+        '__type__': 'AttributeContainer'
+        '__container_type__': ...
+        ...
+    }
+
+    Here '__type__' indicates the object base type. In this case
+    'AttributeContainer'.
+
+    '__container_type__' indicates the container type and rest of the elements
+    of the dictionary that make up the attributes of the container.
+
+    Args:
+      json_dict (dict[str, object]): JSON serialized objects.
+
+    Returns:
+      AttributeContainer: attribute container.
+    """
+    # Use __container_type__ to indicate the attribute container type.
+    container_type = json_dict.get('__container_type__', None)
+
+    attribute_container = cls._CONTAINERS_MANAGER.CreateAttributeContainer(
+        container_type)
+
+    supported_attribute_names = attribute_container.GetAttributeNames()
+    for attribute_name, attribute_value in json_dict.items():
+      if attribute_name in ('__container_type__', '__type__'):
+        continue
+
+      # Be strict about which attributes to set.
+      if attribute_name not in supported_attribute_names:
+        continue
+
+      setattr(attribute_container, attribute_name, attribute_value)
+
+    return attribute_container

acstore/interface.py

@@ -238,3 +238,55 @@ def UpdateAttributeContainer(self, container):
     """
     self._RaiseIfNotWritable()
     self._WriteExistingAttributeContainer(container)
+
+
+class AttributeContainerStoreWithReadCache(AttributeContainerStore):
+  """Interface of an attribute container store with read cache.
+
+  Attributes:
+    format_version (int): storage format version.
+  """
+
+  # pylint: disable=abstract-method
+
+  # The maximum number of cached attribute containers
+  _MAXIMUM_CACHED_CONTAINERS = 32 * 1024
+
+  def __init__(self):
+    """Initializes an attribute container store with read cache."""
+    super(AttributeContainerStoreWithReadCache, self).__init__()
+    self._attribute_container_cache = collections.OrderedDict()
+
+  def _CacheAttributeContainerByIndex(self, attribute_container, index):
+    """Caches a specific attribute container.
+
+    Args:
+      attribute_container (AttributeContainer): attribute container.
+      index (int): attribute container index.
+    """
+    if len(self._attribute_container_cache) >= self._MAXIMUM_CACHED_CONTAINERS:
+      self._attribute_container_cache.popitem(last=True)
+
+    lookup_key = f'{attribute_container.CONTAINER_TYPE:s}.{index:d}'
+    self._attribute_container_cache[lookup_key] = attribute_container
+    self._attribute_container_cache.move_to_end(lookup_key, last=False)
+
+  def _GetCachedAttributeContainer(self, container_type, index):
+    """Retrieves a specific cached attribute container.
+
+    Args:
+      container_type (str): attribute container type.
+      index (int): attribute container index.
+
+    Returns:
+      AttributeContainer: attribute container or None if not available.
+
+    Raises:
+      IOError: when there is an error querying the attribute container store.
+      OSError: when there is an error querying the attribute container store.
+    """
+    lookup_key = f'{container_type:s}.{index:d}'
+    attribute_container = self._attribute_container_cache.get(lookup_key, None)
+    if attribute_container:
+      self._attribute_container_cache.move_to_end(lookup_key, last=False)
+    return attribute_container

acstore/sqlite_store.py

@@ -2,8 +2,8 @@
 """SQLite-based attribute container store."""
 
 import ast
-import collections
 import itertools
+import json
 import os
 import pathlib
 import sqlite3
@@ -122,7 +122,8 @@ def DeserializeValue(self, data_type, value):
       elif data_type not in self._MAPPINGS:
         serializer = schema_helper.SchemaHelper.GetAttributeSerializer(
             data_type, 'json')
-        value = serializer.DeserializeValue(value)
+        json_dict = json.loads(value)
+        value = serializer.DeserializeValue(json_dict)
 
     return value
 
@@ -160,12 +161,14 @@ def SerializeValue(self, data_type, value):
         if isinstance(value, set):
           value = list(value)
 
-        return serializer.SerializeValue(value)
+        json_dict = serializer.SerializeValue(value)
+        return json.dumps(json_dict)
 
     return value
 
 
-class SQLiteAttributeContainerStore(interface.AttributeContainerStore):
+class SQLiteAttributeContainerStore(
+    interface.AttributeContainerStoreWithReadCache):
   """SQLite-based attribute container store.
 
   Attributes:
@@ -205,15 +208,11 @@ class SQLiteAttributeContainerStore(interface.AttributeContainerStore):
   _INSERT_METADATA_VALUE_QUERY = (
       'INSERT INTO metadata (key, value) VALUES (?, ?)')
 
-  # The maximum number of cached attribute containers
-  _MAXIMUM_CACHED_CONTAINERS = 32 * 1024
-
   _MAXIMUM_WRITE_CACHE_SIZE = 50
 
   def __init__(self):
     """Initializes a SQLite attribute container store."""
     super(SQLiteAttributeContainerStore, self).__init__()
-    self._attribute_container_cache = collections.OrderedDict()
     self._connection = None
     self._cursor = None
     self._is_open = False
@@ -224,20 +223,6 @@ def __init__(self):
     self.format_version = self._FORMAT_VERSION
     self.serialization_format = 'json'
 
-  def _CacheAttributeContainerByIndex(self, attribute_container, index):
-    """Caches a specific attribute container.
-
-    Args:
-      attribute_container (AttributeContainer): attribute container.
-      index (int): attribute container index.
-    """
-    if len(self._attribute_container_cache) >= self._MAXIMUM_CACHED_CONTAINERS:
-      self._attribute_container_cache.popitem(last=True)
-
-    lookup_key = f'{attribute_container.CONTAINER_TYPE:s}.{index:d}'
-    self._attribute_container_cache[lookup_key] = attribute_container
-    self._attribute_container_cache.move_to_end(lookup_key, last=False)
-
   def _CacheAttributeContainerForWrite(
       self, container_type, column_names, values):
     """Caches an attribute container for writing.
@@ -515,26 +500,6 @@ def _GetAttributeContainersWithFilter(
           if self._storage_profiler:
             self._storage_profiler.StopTiming('get_containers')
 
-  def _GetCachedAttributeContainer(self, container_type, index):
-    """Retrieves a specific cached attribute container.
-
-    Args:
-      container_type (str): attribute container type.
-      index (int): attribute container index.
-
-    Returns:
-      AttributeContainer: attribute container or None if not available.
-
-    Raises:
-      IOError: when there is an error querying the attribute container store.
-      OSError: when there is an error querying the attribute container store.
-    """
-    lookup_key = f'{container_type:s}.{index:d}'
-    attribute_container = self._attribute_container_cache.get(lookup_key, None)
-    if attribute_container:
-      self._attribute_container_cache.move_to_end(lookup_key, last=False)
-    return attribute_container
-
   def _HasTable(self, table_name):
     """Determines if a specific table exists.
 
@@ -835,7 +800,7 @@ def Close(self):
       OSError: if the attribute container store is already closed.
     """
     if not self._is_open:
-      raise IOError('Storage file already closed.')
+      raise IOError('Attribute container store already closed.')
 
     if self._connection:
       self._Flush()
@@ -1034,7 +999,7 @@ def Open(self, path=None, read_only=True, **unused_kwargs):  # pylint: disable=a
       ValueError: if path is missing.
     """
     if self._is_open:
-      raise IOError('Storage file already opened.')
+      raise IOError('Attribute container store already opened.')
 
     if not path:
       raise ValueError('Missing path.')

config/dpkg/changelog

@@ -2,4 +2,4 @@ acstore (20240121-1) unstable; urgency=low
 
   * Auto-generated
 
- -- Log2Timeline maintainers <log2timeline-maintainers@googlegroups.com>  Sun, 21 Jan 2024 06:24:34 +0100
+ -- Log2Timeline maintainers <log2timeline-maintainers@googlegroups.com>  Sun, 21 Jan 2024 08:55:35 +0100

docs/sources/api/acstore.helpers.rst

@@ -4,6 +4,14 @@ acstore.helpers package
 Submodules
 ----------
 
+acstore.helpers.json\_serializer module
+---------------------------------------
+
+.. automodule:: acstore.helpers.json_serializer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 acstore.helpers.schema module
 -----------------------------