Skip to content

Commit a211a81

Browse files
regitregit
committed
datajson: add remove_key option to dataset
This option allows to remove the key corresponding to the match value from the JSON object before creating the JSON object that will be added to the `extra` data. For example, matching on the following JSON on the `ip` key: ```json {"ip": "10.16.1.11", "test": "success", "context":3} ``` with a match like: ``` dataset:isset,src_ip,type ip,load src.lst,format jsonline,enrichment_key src_ip,value_key ip; ``` will produce the following: ```json "extra": { "src_ip": { "ip": "10.16.1.11", "test": "success", "context": 3 } ``` if we add the `remove_key` option to the match: ``` dataset:isset,src_ip,type ip,load src.lst,format jsonline,enrichment_key src_ip,value_key ip, remove_key; ``` it will produce the following: ```json "extra": { "src_ip": { "test": "success", "context": 3 } ``` The option is set to false by default. Ticket: #7372
1 parent 8f315d6 commit a211a81

File tree

4 files changed

+35
-10
lines changed

4 files changed

+35
-10
lines changed

‎src/datajson.c

+20-2
Original file line numberDiff line numberDiff line change
@@ -341,8 +341,12 @@ static uint32_t DatajsonAddStringElement(Dataset *set, json_t *value, char *json
341341

342342
*found = true;
343343

344-
const char *val = json_string_value(key);
344+
char val[DATAJSON_JSON_LENGTH];
345+
strlcpy(val, json_string_value(key), DATAJSON_JSON_LENGTH - 1);
345346
DataJsonType elt = { .value = NULL, .len = 0 };
347+
if (set->remove_key) {
348+
json_object_del(value, json_key);
349+
}
346350
elt.value = json_dumps(value, JSON_COMPACT);
347351
elt.len = strlen(elt.value);
348352

@@ -402,6 +406,9 @@ static uint32_t DatajsonAddMd5Element(Dataset *set, json_t *value, char *json_ke
402406
return 0;
403407
}
404408
DataJsonType elt = { .value = NULL, .len = 0 };
409+
if (set->remove_key) {
410+
json_object_del(value, json_key);
411+
}
405412
elt.value = json_dumps(value, JSON_COMPACT);
406413
elt.len = strlen(elt.value);
407414

@@ -461,6 +468,9 @@ static uint32_t DatajsonAddSha256Element(Dataset *set, json_t *value, char *json
461468
return 0;
462469
}
463470
DataJsonType elt = { .value = NULL, .len = 0 };
471+
if (set->remove_key) {
472+
json_object_del(value, json_key);
473+
}
464474
elt.value = json_dumps(value, JSON_COMPACT);
465475
elt.len = strlen(elt.value);
466476

@@ -515,6 +525,9 @@ static uint32_t DatajsonAddIpv4Element(Dataset *set, json_t *value, char *json_k
515525
return 0;
516526
}
517527
DataJsonType elt = { .value = NULL, .len = 0 };
528+
if (set->remove_key) {
529+
json_object_del(value, json_key);
530+
}
518531
elt.value = json_dumps(value, JSON_COMPACT);
519532
elt.len = strlen(elt.value);
520533

@@ -571,6 +584,9 @@ static uint32_t DatajsonAddIPv6Element(Dataset *set, json_t *value, char *json_k
571584
return 0;
572585
}
573586
DataJsonType elt = { .value = NULL, .len = 0 };
587+
if (set->remove_key) {
588+
json_object_del(value, json_key);
589+
}
574590
elt.value = json_dumps(value, JSON_COMPACT);
575591
elt.len = strlen(elt.value);
576592

@@ -610,7 +626,8 @@ static int DatajsonLoadIPv6(Dataset *set, char *json_key, char *array_key, Datas
610626
}
611627

612628
Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load, uint64_t memcap,
613-
uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format)
629+
uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format,
630+
bool remove_key)
614631
{
615632
uint64_t default_memcap = 0;
616633
uint32_t default_hashsize = 0;
@@ -659,6 +676,7 @@ Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load,
659676

660677
strlcpy(set->name, name, sizeof(set->name));
661678
set->type = type;
679+
set->remove_key = remove_key;
662680
if (load && strlen(load)) {
663681
strlcpy(set->load, load, sizeof(set->load));
664682
SCLogDebug("set \'%s\' loading \'%s\' from \'%s\'", set->name, load, set->load);

‎src/datajson.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ typedef struct DataJsonResultType {
4242
/* Common functions */
4343

4444
Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load, uint64_t memcap,
45-
uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format);
45+
uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format,
46+
bool remove_key);
4647

4748
DataJsonResultType DatajsonLookup(Dataset *set, const uint8_t *data, const uint32_t data_len);
4849

‎src/datasets.h

+1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ typedef struct Dataset {
5050
uint32_t id;
5151
bool from_yaml; /* Mark whether the set was retrieved from YAML */
5252
bool hidden; /* Mark the old sets hidden in case of reload */
53+
bool remove_key; /* Mark that value key should be removed from extra data */
5354
THashTableContext *hash;
5455

5556
char load[PATH_MAX];

‎src/detect-dataset.c

+12-7
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam
159159
enum DatasetTypes *type, char *load, size_t load_size, char *save, size_t save_size,
160160
uint64_t *memcap, uint32_t *hashsize, DatasetFormats *format, char *value_key,
161161
size_t value_key_size, char *array_key, size_t array_key_size, char *enrichment_key,
162-
size_t enrichment_key_size)
162+
size_t enrichment_key_size, bool *remove_key)
163163
{
164164
bool cmd_set = false;
165165
bool name_set = false;
@@ -206,7 +206,11 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam
206206
name_set = true;
207207
} else {
208208
if (val == NULL) {
209-
return -1;
209+
/* only non fixed place option without value is remove_key */
210+
if (strcmp(key, "remove_key") == 0) {
211+
*remove_key = true;
212+
} else
213+
return -1;
210214
}
211215

212216
if (strcmp(key, "type") == 0) {
@@ -452,6 +456,7 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
452456
char value_key[SIG_JSON_CONTENT_KEY_LEN] = "";
453457
char array_key[SIG_JSON_CONTENT_KEY_LEN] = "";
454458
char enrichment_key[SIG_JSON_CONTENT_KEY_LEN] = "";
459+
bool remove_key = false;
455460

456461
if (DetectBufferGetActiveList(de_ctx, s) == -1) {
457462
SCLogError("datasets are only supported for sticky buffers");
@@ -467,7 +472,7 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
467472
if (!DetectDatasetParse(rawstr, cmd_str, sizeof(cmd_str), name, sizeof(name), &type, load,
468473
sizeof(load), save, sizeof(save), &memcap, &hashsize, &format, value_key,
469474
sizeof(value_key), array_key, sizeof(array_key), enrichment_key,
470-
sizeof(enrichment_key))) {
475+
sizeof(enrichment_key), &remove_key)) {
471476
return -1;
472477
}
473478

@@ -526,11 +531,11 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
526531
Dataset *set = NULL;
527532

528533
if (format == DATASET_FORMAT_JSON) {
529-
set = DatajsonGet(
530-
name, type, load, memcap, hashsize, value_key, array_key, DATASET_FORMAT_JSON);
534+
set = DatajsonGet(name, type, load, memcap, hashsize, value_key, array_key,
535+
DATASET_FORMAT_JSON, remove_key);
531536
} else if (format == DATASET_FORMAT_JSONLINE) {
532-
set = DatajsonGet(
533-
name, type, load, memcap, hashsize, value_key, NULL, DATASET_FORMAT_JSONLINE);
537+
set = DatajsonGet(name, type, load, memcap, hashsize, value_key, NULL,
538+
DATASET_FORMAT_JSONLINE, remove_key);
534539
} else {
535540
set = DatasetGet(name, type, save, load, memcap, hashsize);
536541
}

0 commit comments

Comments
 (0)