Fix startup (#2)

Fixed errors related to connecting to RonDB, creating NdbRecords, etc.

Author: olapiv

pink/rondis/common.h

@@ -4,6 +4,8 @@
 #define MAX_CONNECTIONS 1
 #define MAX_NDB_PER_CONNECTION 1
 
+#define REDIS_DB_NAME "redis"
+
 #define FOREIGN_KEY_RESTRICT_ERROR 256
 
 #define RONDB_INTERNAL_ERROR 2

pink/rondis/rondb.cc

@@ -29,7 +29,7 @@ int initialize_connections(const char *connect_string)
         printf("RonDB data node connection nr. %d is ready\n", i);
         for (unsigned int j = 0; j < MAX_NDB_PER_CONNECTION; j++)
         {
-            Ndb *ndb = new Ndb(rondb_conn[i], "redis_0");
+            Ndb *ndb = new Ndb(rondb_conn[i], REDIS_DB_NAME);
             if (ndb == nullptr)
             {
                 printf("Failed creating Ndb object nr. %d for cluster connection %d\n", j, i);
@@ -61,7 +61,14 @@ int setup_rondb(const char *connect_string)
     Ndb *ndb = rondb_ndb[0][0];
     NdbDictionary::Dictionary *dict = ndb->getDictionary();
 
-    return init_string_records(dict);
+    if (init_string_records(dict) != 0)
+    {
+        printf("Failed initializing records for Redis data type STRING; error: %s\n",
+               ndb->getNdbError().message);
+        return -1;
+    }
+
+    return 0;
 }
 
 void rondb_end()

pink/rondis/sql/STRING_key.sql

@@ -1,19 +1,24 @@
-CREATE TABLE redis_string_keys(
+CREATE TABLE redis.string_keys(
+    -- Redis actually supports a max key size of 512MiB,
+    -- but we choose not to support that here
     redis_key VARBINARY(3000) NOT NULL,
     -- We will use a ndb auto-increment
     -- This is to save space when referencing the key in the value table
     rondb_key BIGINT UNSIGNED,
-    -- Max 512MiB
-    value_data_type ENUM('string', 'number', 'binary_string') value_start VARBINARY(26500) NOT NULL,
+    -- TODO: Replace with Enum below
+    value_data_type INT UNSIGNED NOT NULL,
+    -- value_data_type ENUM('string', 'number', 'binary_string'),
     -- Max 512MiB --> 512 * 1,048,576 bytes = 536,870,912 characters
     -- --> To describe the length, one needs at least UINT (4,294,967,295)
-    value_len INT UNSIGNED NOT NULL,
+    tot_value_len INT UNSIGNED NOT NULL,
+    -- Technically implicit
+    num_rows INT UNSIGNED NOT NULL,
     value_start VARBINARY(26500) NOT NULL,
-    -- Redis supports get/set of seconds/miliseconds
+    -- Redis supports get/set of seconds/milliseconds
     expiry_date INT UNSIGNED,
     -- Easier to sort and delete keys this way
     KEY expiry_index(expiry_date),
     PRIMARY KEY (redis_key) USING HASH,
-    UNIQUE KEY (rondb_key) USING HASH,
+    UNIQUE KEY (rondb_key) USING HASH
 ) ENGINE NDB -- Each CHAR will use 1 byte
-CHARSET = latin1 COMMENT = "NDB_TABLE=PARTITION_BALANCE=FOR_RP_BY_LDM_X_8"
+CHARSET = latin1 COMMENT = "NDB_TABLE=PARTITION_BALANCE=FOR_RP_BY_LDM_X_8";

pink/rondis/sql/STRING_value.sql

@@ -1,8 +1,8 @@
-CREATE TABLE redis_string_values(
+CREATE TABLE redis.string_values(
     rondb_key BIGINT UNSIGNED NOT NULL,
     ordinal INT UNSIGNED NOT NULL,
     value VARBINARY(29500) NOT NULL,
     PRIMARY KEY (rondb_key, ordinal),
-    FOREIGN KEY (rondb_key) REFERENCES redis_string_key(rondb_key) ON UPDATE RESTRICT ON DELETE CASCADE
+    FOREIGN KEY (rondb_key) REFERENCES redis.string_keys(rondb_key) ON UPDATE RESTRICT ON DELETE CASCADE
 ) ENGINE NDB,
-COMMENT = "NDB_TABLE=PARTITION_BALANCE=RP_BY_LDM_X_8" PARTITION BY KEY (rondb_key)
+COMMENT = "NDB_TABLE=PARTITION_BALANCE=RP_BY_LDM_X_8" PARTITION BY KEY (rondb_key);

pink/rondis/string/db_interactions.cc

@@ -51,7 +51,7 @@ int create_key_row(std::string *response,
     }
     write_op->setValue(KEY_TABLE_COL_tot_value_len, value_len);
     write_op->setValue(KEY_TABLE_COL_num_rows, value_rows);
-    write_op->setValue(KEY_TABLE_COL_row_state, row_state);
+    write_op->setValue(KEY_TABLE_COL_value_data_type, row_state);
     write_op->setValue(KEY_TABLE_COL_expiry_date, 0);
 
     if (value_len > INLINE_VALUE_LEN)
@@ -149,7 +149,7 @@ int create_key_row(std::string *response,
         insert_op->equal(KEY_TABLE_COL_redis_key, buf);
         insert_op->setValue(KEY_TABLE_COL_tot_value_len, value_len);
         insert_op->setValue("value_rows", value_rows);
-        insert_op->setValue(KEY_TABLE_COL_row_state, row_state);
+        insert_op->setValue(KEY_TABLE_COL_value_data_type, row_state);
         insert_op->setValue(KEY_TABLE_COL_expiry_date, 0);
         {
             int ret_code = insert_op->getNdbError().code;

pink/rondis/string/table_definitions.cc

@@ -21,164 +21,125 @@ int init_key_records(NdbDictionary::Dictionary *dict)
     const NdbDictionary::Table *tab = dict->getTable(KEY_TABLE_NAME);
     if (tab == nullptr)
     {
-        printf("Failed getting table for key table of STRING\n");
+        printf("Failed getting Ndb table %s\n", KEY_TABLE_NAME);
         return -1;
     }
+
     const NdbDictionary::Column *redis_key_col = tab->getColumn(KEY_TABLE_COL_redis_key);
     const NdbDictionary::Column *rondb_key_col = tab->getColumn(KEY_TABLE_COL_rondb_key);
     const NdbDictionary::Column *expiry_date_col = tab->getColumn(KEY_TABLE_COL_expiry_date);
     const NdbDictionary::Column *value_start_col = tab->getColumn(KEY_TABLE_COL_value_start);
     const NdbDictionary::Column *tot_value_len_col = tab->getColumn(KEY_TABLE_COL_tot_value_len);
     const NdbDictionary::Column *num_rows_col = tab->getColumn(KEY_TABLE_COL_num_rows);
-    const NdbDictionary::Column *row_state_col = tab->getColumn(KEY_TABLE_COL_row_state);
+    const NdbDictionary::Column *value_data_type_col = tab->getColumn(KEY_TABLE_COL_value_data_type);
 
     if (redis_key_col == nullptr ||
         rondb_key_col == nullptr ||
         expiry_date_col == nullptr ||
         value_start_col == nullptr ||
         tot_value_len_col == nullptr ||
         num_rows_col == nullptr ||
-        row_state_col == nullptr)
+        value_data_type_col == nullptr)
     {
-        printf("Failed getting columns for key table of STRING\n");
+        printf("Failed getting Ndb columns for table %s\n", KEY_TABLE_NAME);
         return -1;
     }
 
-    NdbDictionary::RecordSpecification primary_redis_main_key_spec[1];
-    NdbDictionary::RecordSpecification all_redis_main_key_spec[7];
-
-    primary_redis_main_key_spec[0].column = redis_key_col;
-    primary_redis_main_key_spec[0].offset = offsetof(struct key_table, redis_key);
-    primary_redis_main_key_spec[0].nullbit_byte_offset = 0;
-    primary_redis_main_key_spec[0].nullbit_bit_in_byte = 0;
-    pk_key_record =
-        dict->createRecord(tab,
-                           primary_redis_main_key_spec,
-                           1,
-                           sizeof(primary_redis_main_key_spec[0]));
-    if (pk_key_record == nullptr)
+    std::map<const NdbDictionary::Column *, std::pair<size_t, int>> pk_lookup_column_map = {
+        {redis_key_col, {offsetof(struct key_table, redis_key), 0}},
+    };
+    if (init_record(dict, tab, pk_lookup_column_map, pk_key_record) != 0)
     {
-        printf("Failed creating record for key table of STRING\n");
+        printf("Failed creating pk-lookup record for table %s\n", KEY_TABLE_NAME);
         return -1;
     }
 
-    all_redis_main_key_spec[0].column = redis_key_col;
-    all_redis_main_key_spec[0].offset = offsetof(struct key_table, redis_key);
-    all_redis_main_key_spec[0].nullbit_byte_offset = 0;
-    all_redis_main_key_spec[0].nullbit_bit_in_byte = 0;
-
-    all_redis_main_key_spec[1].column = rondb_key_col;
-    all_redis_main_key_spec[1].offset = offsetof(struct key_table, rondb_key);
-    all_redis_main_key_spec[1].nullbit_byte_offset = 0;
-    all_redis_main_key_spec[1].nullbit_bit_in_byte = 0;
-
-    all_redis_main_key_spec[2].column = expiry_date_col;
-    all_redis_main_key_spec[2].offset = offsetof(struct key_table, expiry_date);
-    all_redis_main_key_spec[2].nullbit_byte_offset = 0;
-    all_redis_main_key_spec[2].nullbit_bit_in_byte = 1;
-
-    all_redis_main_key_spec[3].column = value_start_col;
-    all_redis_main_key_spec[3].offset = offsetof(struct key_table, value_start);
-    all_redis_main_key_spec[3].nullbit_byte_offset = 0;
-    all_redis_main_key_spec[3].nullbit_bit_in_byte = 0;
-
-    all_redis_main_key_spec[4].column = tot_value_len_col;
-    all_redis_main_key_spec[4].offset = offsetof(struct key_table, tot_value_len);
-    all_redis_main_key_spec[4].nullbit_byte_offset = 0;
-    all_redis_main_key_spec[4].nullbit_bit_in_byte = 0;
-
-    all_redis_main_key_spec[5].column = num_rows_col;
-    all_redis_main_key_spec[5].offset = offsetof(struct key_table, num_rows);
-    all_redis_main_key_spec[5].nullbit_byte_offset = 0;
-    all_redis_main_key_spec[5].nullbit_bit_in_byte = 0;
-
-    all_redis_main_key_spec[6].column = row_state_col;
-    all_redis_main_key_spec[6].offset = offsetof(struct key_table, row_state);
-    all_redis_main_key_spec[6].nullbit_byte_offset = 0;
-    all_redis_main_key_spec[6].nullbit_bit_in_byte = 0;
-
-    entire_key_record = dict->createRecord(tab,
-                                           all_redis_main_key_spec,
-                                           8,
-                                           sizeof(all_redis_main_key_spec[0]));
-    if (entire_key_record == nullptr)
+    std::map<const NdbDictionary::Column *, std::pair<size_t, int>> read_all_column_map = {
+        // TODO: Fix this one
+        // {redis_key_col, {offsetof(struct key_table, redis_key), 0}},
+        {rondb_key_col, {offsetof(struct key_table, rondb_key), 0}},
+        {expiry_date_col, {offsetof(struct key_table, expiry_date), 1}},
+        {value_start_col, {offsetof(struct key_table, value_start), 0}},
+        {tot_value_len_col, {offsetof(struct key_table, tot_value_len), 0}},
+        {num_rows_col, {offsetof(struct key_table, num_rows), 0}},
+        {value_data_type_col, {offsetof(struct key_table, value_data_type), 0}}
+    };
+
+    if (init_record(dict, tab, read_all_column_map, entire_key_record) != 0)
     {
-        printf("Failed creating record for key table of STRING\n");
+        printf("Failed creating read-all cols record for table %s\n", KEY_TABLE_NAME);
         return -1;
     }
     return 0;
 }
 
 int init_value_records(NdbDictionary::Dictionary *dict)
 {
-    const NdbDictionary::Table *tab = dict->getTable("redis_key_value");
+    const NdbDictionary::Table *tab = dict->getTable(VALUE_TABLE_NAME);
     if (tab == nullptr)
     {
-        printf("Failed getting table for value table of STRING\n");
+        printf("Failed getting Ndb table %s\n", VALUE_TABLE_NAME);
         return -1;
     }
+
     const NdbDictionary::Column *rondb_key_col = tab->getColumn(VALUE_TABLE_COL_rondb_key);
     const NdbDictionary::Column *ordinal_col = tab->getColumn(VALUE_TABLE_COL_ordinal);
     const NdbDictionary::Column *value_col = tab->getColumn(VALUE_TABLE_COL_value);
     if (rondb_key_col == nullptr ||
         ordinal_col == nullptr ||
         value_col == nullptr)
     {
-        printf("Failed getting columns for value table of STRING\n");
+        printf("Failed getting Ndb columns for table %s\n", VALUE_TABLE_NAME);
         return -1;
     }
 
-    NdbDictionary::RecordSpecification primary_redis_key_value_spec[2];
-    NdbDictionary::RecordSpecification all_redis_key_value_spec[3];
-
-    primary_redis_key_value_spec[0].column = rondb_key_col;
-    primary_redis_key_value_spec[0].offset = offsetof(struct value_table, rondb_key);
-    primary_redis_key_value_spec[0].nullbit_byte_offset = 0;
-    primary_redis_key_value_spec[0].nullbit_bit_in_byte = 0;
+    std::map<const NdbDictionary::Column *, std::pair<size_t, int>> pk_lookup_column_map = {
+        {rondb_key_col, {offsetof(struct value_table, rondb_key), 0}},
+        {ordinal_col, {offsetof(struct value_table, ordinal), 0}}};
 
-    primary_redis_key_value_spec[1].column = ordinal_col;
-    primary_redis_key_value_spec[1].offset = offsetof(struct value_table, ordinal);
-    primary_redis_key_value_spec[1].nullbit_byte_offset = 0;
-    primary_redis_key_value_spec[1].nullbit_bit_in_byte = 0;
-
-    pk_value_record = dict->createRecord(tab,
-                                         primary_redis_key_value_spec,
-                                         2,
-                                         sizeof(primary_redis_key_value_spec[0]));
-    if (pk_value_record == nullptr)
+    if (init_record(dict, tab, pk_lookup_column_map, pk_value_record) != 0)
     {
-        printf("Failed creating record for value table of STRING\n");
+        printf("Failed creating pk-lookup record for table %s\n", VALUE_TABLE_NAME);
         return -1;
     }
 
-    all_redis_key_value_spec[0].column = rondb_key_col;
-    all_redis_key_value_spec[0].offset = offsetof(struct value_table, rondb_key);
-    all_redis_key_value_spec[0].nullbit_byte_offset = 0;
-    all_redis_key_value_spec[0].nullbit_bit_in_byte = 0;
-
-    all_redis_key_value_spec[1].column = ordinal_col;
-    all_redis_key_value_spec[1].offset = offsetof(struct value_table, ordinal);
-    all_redis_key_value_spec[1].nullbit_byte_offset = 0;
-    all_redis_key_value_spec[1].nullbit_bit_in_byte = 0;
-
-    all_redis_key_value_spec[2].column = value_col;
-    all_redis_key_value_spec[2].offset = offsetof(struct value_table, value);
-    all_redis_key_value_spec[2].nullbit_byte_offset = 0;
-    all_redis_key_value_spec[2].nullbit_bit_in_byte = 0;
-
-    entire_value_record = dict->createRecord(tab,
-                                             all_redis_key_value_spec,
-                                             3,
-                                             sizeof(all_redis_key_value_spec[0]));
-    if (entire_value_record == nullptr)
+    std::map<const NdbDictionary::Column *, std::pair<size_t, int>> read_all_column_map = {
+        {rondb_key_col, {offsetof(struct value_table, rondb_key), 0}},
+        {ordinal_col, {offsetof(struct value_table, ordinal), 0}},
+        {value_col, {offsetof(struct value_table, value), 0}}};
+
+    if (init_record(dict, tab, read_all_column_map, entire_value_record) != 0)
     {
-        printf("Failed creating record for value table of STRING\n");
+        printf("Failed creating read-all cols record for table %s\n", VALUE_TABLE_NAME);
         return -1;
     }
 
     return 0;
 }
 
+int init_record(NdbDictionary::Dictionary *dict,
+                const NdbDictionary::Table *tab,
+                std::map<const NdbDictionary::Column *, std::pair<size_t, int>> column_info_map,
+                NdbRecord *&record)
+{
+    NdbDictionary::RecordSpecification col_specs[column_info_map.size()];
+    int i = 0;
+    for (const auto &entry : column_info_map)
+    {
+        col_specs[i].column = entry.first;
+        col_specs[i].offset = entry.second.first;
+        col_specs[i].nullbit_byte_offset = 0;
+        col_specs[i].nullbit_bit_in_byte = entry.second.second;
+        ++i;
+    }
+    record = dict->createRecord(tab,
+                                col_specs,
+                                column_info_map.size(),
+                                sizeof(col_specs[0]));
+
+    return (record == nullptr) ? -1 : 0;
+}
+
 int init_string_records(NdbDictionary::Dictionary *dict)
 {
     int res = init_key_records(dict);

pink/rondis/string/table_definitions.h

@@ -12,7 +12,7 @@
     KEY TABLE
 */
 
-#define KEY_TABLE_NAME "redis_string_keys"
+#define KEY_TABLE_NAME "string_keys"
 #define MAX_KEY_VALUE_LEN 3000
 #define INLINE_VALUE_LEN 26500
 
@@ -28,18 +28,17 @@ extern NdbRecord *entire_key_record;
 #define KEY_TABLE_COL_redis_key "redis_key"
 #define KEY_TABLE_COL_rondb_key "rondb_key"
 #define KEY_TABLE_COL_expiry_date "expiry_date"
-#define KEY_TABLE_COL_row_state "row_state"
+#define KEY_TABLE_COL_value_data_type "value_data_type"
 #define KEY_TABLE_COL_tot_value_len "tot_value_len"
 #define KEY_TABLE_COL_num_rows "num_rows"
 #define KEY_TABLE_COL_value_start "value_start"
 
 struct key_table
 {
-    Uint32 null_bits; // TODO: What's this for?
     char redis_key[MAX_KEY_VALUE_LEN + 2];
     Uint64 rondb_key;
     Uint32 expiry_date;
-    Uint32 row_state;
+    Uint32 value_data_type;
     Uint32 tot_value_len;
     // Technically implicit
     Uint32 num_rows;
@@ -50,7 +49,7 @@ struct key_table
     VALUE TABLE
 */
 
-#define VALUE_TABLE_NAME "redis_string_values"
+#define VALUE_TABLE_NAME "string_values"
 #define EXTENSION_VALUE_LEN 29500
 
 int init_value_records(NdbDictionary::Dictionary *dict);
@@ -74,7 +73,12 @@ struct value_table
 };
 
 /*
-    EXPORT
+    SHARED/EXPORT
 */
 
+int init_record(NdbDictionary::Dictionary *dict,
+                const NdbDictionary::Table *tab,
+                std::map<const NdbDictionary::Column *, std::pair<size_t, int>> column_info_map,
+                NdbRecord *&record);
+
 int init_string_records(NdbDictionary::Dictionary *dict);