simplified sqlite script (#3)

Author: maartendecat

sqlite/README.md

@@ -0,0 +1,59 @@
+# Elimity Insights example SQLite connector
+
+This Python package implements a basic connector importing data from a SQLite (version 3.37.0 or
+higher) database. 
+
+Note: For larger datasets we recommend avoiding loading all data into memory at once. 
+[This](../sqlite_streaming) example connector extends the basic example with streaming to avoid this.
+
+## Example data
+
+This example script expects the database to contain users and roles, both with metadata.
+
+You can use the following query to initialize an example database:
+
+```sqlite
+BEGIN;
+
+CREATE TABLE users (
+    display_name TEXT NOT NULL,
+    first_name TEXT NOT NULL,
+    id INTEGER PRIMARY KEY,
+    last_name TEXT NOT NULL,
+    last_logon INTEGER
+) STRICT;
+
+INSERT INTO users (display_name, first_name, last_name, last_logon)
+VALUES
+    ('dduck', 'Donald', 'Duck', 1668603363),
+    ('mmouse', 'Mickey', 'Mouse', NULL);
+CREATE TABLE roles (
+    id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    security_level INTEGER NOT NULL
+) STRICT;
+
+INSERT INTO roles (name, security_level)
+VALUES
+    ('Reader', 0),
+    ('Writer', 1);
+
+CREATE TABLE user_roles (
+    id INTEGER PRIMARY KEY,
+    role_id INTEGER NOT NULL REFERENCES roles (id),
+    user_id INTEGER NOT NULL REFERENCES users (id),
+    UNIQUE (role_id, user_id)
+) STRICT;
+
+INSERT INTO user_roles (role_id, user_id)
+VALUES
+    (1, 1),
+    (1, 2),
+    (2, 2);
+
+COMMIT
+```
+
+## Data model for Elimity Insights
+
+The data model for a custom source in Elimity Insights can be found in the [`data-model.json`](data-model.json) file.

sqlite/.isort.cfg renamed to sqlite_basic/.isort.cfg

@@ -0,0 +1,98 @@
+from datetime import datetime
+from sqlite3 import connect
+from typing import List
+
+from elimity_insights_client import (
+    AttributeAssignment,
+    Client,
+    Config,
+    DateTime,
+    DateTimeValue,
+    DomainGraph,
+    Entity,
+    NumberValue,
+    Relationship,
+    StringValue,
+)
+
+# Configuration
+# Note: If you want, you can turn these variables into command line inputs using [https://docs.python.org/3/library/argparse.html](ArgParse).
+# See the example connector `sqlite_streaming` for an example of how to do this.
+DATABASE = "mydb.sqlite3"
+ELIMITY_INSIGHTS_URL = "https://example.elimity.com"
+ELIMITY_INSIGHTS_SOURCE_ID = "1"
+ELIMITY_INSIGHTS_SOURCE_API_TOKEN = "34kj328ukj291ukjsdkj394ukjdf"
+
+
+def main() -> None:
+    config = Config(
+        ELIMITY_INSIGHTS_SOURCE_ID,
+        ELIMITY_INSIGHTS_URL,
+        ELIMITY_INSIGHTS_SOURCE_API_TOKEN,
+    )
+    client = Client(config)
+    with connect(DATABASE, isolation_level=None) as connection:
+
+        connection.execute("BEGIN")
+
+        # To load the data of the database into Elimity Insights, we have to construct a "domain graph".
+        # Essentially, this domain graph contains the list of entities and their attribute assignments,
+        # and the relationships between these entities.
+        entities: List[Entity] = []
+        relationships: List[Relationship] = []
+
+        # Step 1. Turn the users into entities with attribute assignments.
+        for display_name, first_name, id, last_logon, last_name in connection.execute(
+            """
+                SELECT display_name, first_name, CAST(id AS TEXT), last_logon, last_name
+                FROM users
+            """
+        ):
+            user_assignments: List[AttributeAssignment] = [
+                AttributeAssignment("firstName", StringValue(first_name)),
+                AttributeAssignment("lastName", StringValue(last_name)),
+            ]
+            if last_logon is not None:
+                last_logon_datetime = datetime.utcfromtimestamp(last_logon)
+                last_logon_time = DateTime(
+                    last_logon_datetime.year,
+                    last_logon_datetime.month,
+                    last_logon_datetime.day,
+                    last_logon_datetime.hour,
+                    last_logon_datetime.minute,
+                    last_logon_datetime.second,
+                )
+                last_logon_value = DateTimeValue(last_logon_time)
+                user_assignments.append(
+                    AttributeAssignment("lastLogon", last_logon_value)
+                )
+            entities.append(Entity(user_assignments, id, display_name, "user"))
+
+        # Step 2. Turn the roles into entities with attribute assignments
+        for id, name, security_level in connection.execute(
+            """
+                SELECT CAST(id AS TEXT), name, security_level
+                FROM roles
+            """
+        ):
+            role_assignments: List[AttributeAssignment] = [
+                AttributeAssignment("securityLevel", NumberValue(security_level))
+            ]
+            entities.append(Entity(role_assignments, id, name, "role"))
+
+        # Step 3. Turn the relationships table into relationships
+        for user_id, role_id in connection.execute(
+            """
+                SELECT CAST(user_id AS TEXT), CAST(role_id AS TEXT)
+                FROM user_roles
+            """
+        ):
+            # Relationships can be assigned attributes as well, but this example does not include that.
+            assignments: List[AttributeAssignment] = []
+            relationships.append(
+                Relationship(assignments, user_id, "user", role_id, "role")
+            )
+
+        # Step 4. Upload the entities and relationships as a domain graph
+        graph = DomainGraph(entities, relationships)
+        client.reload_domain_graph(graph)

sqlite_basic/README.md

@@ -0,0 +1,2 @@
+[settings]
+profile = black

sqlite/data-model.json renamed to sqlite_basic/data-model.json

@@ -0,0 +1,80 @@
+# Elimity Insights example SQLite connector with data streaming
+
+This Python package implements an example connector importing data from a SQLite (version 3.37.0 or
+higher) database. 
+
+This example extends [the basic example](../sqlite_basic) with data streaming
+to avoid that all data is loaded into memory at once, which is useful for large datasets.
+
+In addition, this example illustrates how to receive input parameters from the command line
+using [ArgParse](https://docs.python.org/3/library/argparse.html).
+
+## Usage
+
+```console
+(venv) $ elimity-insights-example-sqlite-connector --help
+usage: elimity-insights-example-sqlite-connector [-h] --database DATABASE --source-id SOURCE_ID --source-token SOURCE_TOKEN --url URL
+
+Example Elimity Insights custom connector importing from a SQLite database
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --database DATABASE   path to the SQLite database file
+  --source-id SOURCE_ID
+                        identifier for authenticating the source in Elimity Insights
+  --source-token SOURCE_TOKEN
+                        token for authenticating the source in Elimity Insights
+  --url URL             URL of the Elimity Insights server
+```
+
+## Example data
+
+This example script expects the database to contain users and roles, both with metadata.
+
+You can use the following query to initialize an example database:
+
+```sqlite
+BEGIN;
+
+CREATE TABLE users (
+    display_name TEXT NOT NULL,
+    first_name TEXT NOT NULL,
+    id INTEGER PRIMARY KEY,
+    last_name TEXT NOT NULL,
+    last_logon INTEGER
+) STRICT;
+
+INSERT INTO users (display_name, first_name, last_name, last_logon)
+VALUES
+    ('dduck', 'Donald', 'Duck', 1668603363),
+    ('mmouse', 'Mickey', 'Mouse', NULL);
+CREATE TABLE roles (
+    id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    security_level INTEGER NOT NULL
+) STRICT;
+
+INSERT INTO roles (name, security_level)
+VALUES
+    ('Reader', 0),
+    ('Writer', 1);
+
+CREATE TABLE user_roles (
+    id INTEGER PRIMARY KEY,
+    role_id INTEGER NOT NULL REFERENCES roles (id),
+    user_id INTEGER NOT NULL REFERENCES users (id),
+    UNIQUE (role_id, user_id)
+) STRICT;
+
+INSERT INTO user_roles (role_id, user_id)
+VALUES
+    (1, 1),
+    (1, 2),
+    (2, 2);
+
+COMMIT
+```
+
+## Data model for Elimity Insights
+
+The data model for a custom source in Elimity Insights can be found in the [`data-model.json`](data-model.json) file.

sqlite_basic/elimity_insights_example_sqlite_connector.py

@@ -0,0 +1,53 @@
+{
+  "entityAttributeTypes": [
+    {
+      "archived": false,
+      "description": "",
+      "entityTypeId": "user",
+      "id": "firstName",
+      "name": "First name",
+      "type": "string"
+    },
+    {
+      "archived": false,
+      "description": "",
+      "entityTypeId": "user",
+      "id": "lastName",
+      "name": "Last name",
+      "type": "string"
+    },
+    {
+      "archived": false,
+      "description": "",
+      "entityTypeId": "user",
+      "id": "lastLogon",
+      "name": "Last logon",
+      "type": "dateTime"
+    },
+    {
+      "archived": false,
+      "description": "",
+      "entityTypeId": "role",
+      "id": "securityLevel",
+      "name": "Security level",
+      "type": "number"
+    }
+  ],
+  "entityTypes": [
+    {
+      "anonymized": false,
+      "icon": "person",
+      "id": "user",
+      "plural": "Users",
+      "singular": "User"
+    },
+    {
+      "anonymized": false,
+      "icon": "label",
+      "id": "role",
+      "plural": "Roles",
+      "singular": "Role"
+    }
+  ],
+  "relationshipAttributeTypes": []
+}

sqlite/mypy.ini renamed to sqlite_basic/mypy.ini

@@ -22,46 +22,7 @@ def main() -> None:
     config = Config(args.source_id, args.url, args.source_token)
     client = Client(config)
     with connect(args.database, isolation_level=None) as connection:
-        sql = """
-            BEGIN;
-
-            CREATE TABLE users (
-                display_name TEXT NOT NULL,
-                first_name TEXT NOT NULL,
-                id INTEGER PRIMARY KEY,
-                last_name TEXT NOT NULL,
-                last_logon INTEGER
-            ) STRICT;
-
-            INSERT INTO users (display_name, first_name, last_name, last_logon)
-            VALUES
-                ('dduck', 'Donald', 'Duck', 1668603363),
-                ('mmouse', 'Mickey', 'Mouse', NULL);
-            CREATE TABLE roles (
-                id INTEGER PRIMARY KEY,
-                name TEXT NOT NULL,
-                security_level INTEGER NOT NULL
-            ) STRICT;
-
-            INSERT INTO roles (name, security_level)
-            VALUES
-                ('Reader', 0),
-                ('Writer', 1);
-
-            CREATE TABLE user_roles (
-                id INTEGER PRIMARY KEY,
-                role_id INTEGER NOT NULL REFERENCES roles (id),
-                user_id INTEGER NOT NULL REFERENCES users (id),
-                UNIQUE (role_id, user_id)
-            ) STRICT;
-
-            INSERT INTO user_roles (role_id, user_id)
-            VALUES
-                (1, 1),
-                (1, 2),
-                (2, 2)
-        """
-        connection.executescript(sql if args.generate_database else "BEGIN")
+        connection.execute("BEGIN")
         entities = _entities(connection)
         relationships = _relationships(connection)
         graph = DomainGraph(entities, relationships)
@@ -135,11 +96,6 @@ def _string_attribute_assignment(
     description="Example Elimity Insights custom connector importing from a SQLite database"
 )
 _add_flag("path to the SQLite database file", "--database")
-_parser.add_argument(
-    "--generate-database",
-    action="store_true",
-    help="generate a new sample database before importing",
-)
 _add_flag(
     "identifier for authenticating the source in Elimity Insights", "--source-id", True
 )

sqlite/poetry.lock renamed to sqlite_basic/poetry.lock

@@ -0,0 +1,3 @@
+[mypy]
+disallow_any_explicit = True
+strict = True