Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Change: Use JSON streaming parser for EPSS #2293

Merged
merged 2 commits into from
Sep 23, 2024
Merged

Change: Use JSON streaming parser for EPSS #2293

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5c8772f
Change: Use JSON streaming parser for EPSS
a-h-abdelsalam Sep 17, 2024
99fe1b3
Use open then fdopen to avoid TOCTOU code scanning alert.
a-h-abdelsalam Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
181 changes: 118 additions & 63 deletions src/manage_sql_secinfo.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3371,121 +3371,176 @@
static int
update_epss_scores ()
{
GError *error = NULL;
gchar *current_json_path;
gchar *file_contents = NULL;
cJSON *parsed, *epss_scores_list, *list_item;
gchar *error_message = NULL;

Check warning on line 3375 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3375 

Added line #L3375 was not covered by tests
FILE *epss_scores_file;
cJSON *epss_entry;
gvm_json_pull_event_t event;
gvm_json_pull_parser_t parser;
inserts_t inserts;

current_json_path = g_build_filename (GVM_SCAP_DATA_DIR,
"epss-scores-current.json",
NULL);
int fd = open(current_json_path, O_RDONLY);

Check warning on line 3385 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3385 

Added line #L3385 was not covered by tests

if (! g_file_get_contents (current_json_path, &file_contents, NULL, &error))
if (fd < 0)
{
int ret;
if (error->code == G_FILE_ERROR_NOENT)
if (errno == ENOENT)
{
g_info ("%s: EPSS scores file '%s' not found",
__func__, current_json_path);
ret = 0;
}
else
{
g_warning ("%s: Error loading EPSS scores file: %s",
__func__, error->message);
g_warning ("%s: Failed to open EPSS scores file: %s",

Check warning on line 3398 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3398 

Added line #L3398 was not covered by tests
__func__, strerror (errno));
ret = -1;
}
g_error_free (error);
g_free (current_json_path);
g_free (current_json_path);

Check warning on line 3402 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3402 

Added line #L3402 was not covered by tests
return ret;
}

epss_scores_file = fdopen(fd, "r");

Check warning on line 3406 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3406 

Added line #L3406 was not covered by tests
if (epss_scores_file == NULL)
{
g_warning ("%s: Failed to convert file descriptor to FILE*: %s",

Check warning on line 3409 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3409 

Added line #L3409 was not covered by tests
__func__,
strerror (errno));
g_free (current_json_path);
close(fd);
return -1;

Check warning on line 3414 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3412-L3414 

Added lines #L3412 - L3414 were not covered by tests
}

g_info ("Updating EPSS scores from %s", current_json_path);
g_free (current_json_path);

parsed = cJSON_Parse (file_contents);
g_free (file_contents);
gvm_json_pull_event_init (&event);
gvm_json_pull_parser_init (&parser, epss_scores_file);

Check warning on line 3421 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3420-L3421 

Added lines #L3420 - L3421 were not covered by tests

if (parsed == NULL)
{
g_warning ("%s: EPSS scores file is not valid JSON", __func__);
return -1;
}

if (! cJSON_IsObject (parsed))
{
g_warning ("%s: EPSS scores file is not a JSON object", __func__);
cJSON_Delete (parsed);
return -1;
}
gvm_json_pull_parser_next (&parser, &event);

Check warning on line 3423 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3423 

Added line #L3423 was not covered by tests

epss_scores_list = cJSON_GetObjectItem (parsed, "epss_scores");
if (epss_scores_list == NULL)
if (event.type == GVM_JSON_PULL_EVENT_OBJECT_START)
{
g_warning ("%s: Missing epss_scores field",
__func__);
cJSON_Delete (parsed);
return -1;
}
gboolean epss_scores_found = FALSE;

Check warning on line 3427 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3427 

Added line #L3427 was not covered by tests
while (!epss_scores_found)
{
gvm_json_pull_parser_next (&parser, &event);
gvm_json_path_elem_t *path_tail = g_queue_peek_tail (event.path);

Check warning on line 3431 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3430-L3431 

Added lines #L3430 - L3431 were not covered by tests
if (event.type == GVM_JSON_PULL_EVENT_ARRAY_START
&& path_tail && strcmp (path_tail->key, "epss_scores") == 0)

Check warning on line 3433 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3433 

Added line #L3433 was not covered by tests
{
epss_scores_found = TRUE;

Check warning on line 3435 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3435 

Added line #L3435 was not covered by tests
}
else if (event.type == GVM_JSON_PULL_EVENT_ERROR)

Check warning on line 3437 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3437 

Added line #L3437 was not covered by tests
{
g_warning ("%s: Parser error: %s", __func__, event.error_message);
gvm_json_pull_event_cleanup (&event);
gvm_json_pull_parser_cleanup (&parser);
fclose (epss_scores_file);
return -1;

Check warning on line 3443 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3439-L3443 

Added lines #L3439 - L3443 were not covered by tests
}
else if (event.type == GVM_JSON_PULL_EVENT_OBJECT_END
&& g_queue_is_empty (event.path))

Check warning on line 3446 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3445-L3446 

Added lines #L3445 - L3446 were not covered by tests
{
g_warning ("%s: Unexpected json object end. Missing epss_scores field", __func__);
gvm_json_pull_event_cleanup (&event);
gvm_json_pull_parser_cleanup (&parser);
fclose (epss_scores_file);
return -1;

Check warning on line 3452 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3448-L3452 

Added lines #L3448 - L3452 were not covered by tests
}
}

sql_begin_immediate ();
inserts_init (&inserts,
sql_begin_immediate ();
inserts_init (&inserts,

Check warning on line 3457 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3456-L3457 

Added lines #L3456 - L3457 were not covered by tests
EPSS_MAX_CHUNK_SIZE,
setting_secinfo_sql_buffer_threshold_bytes (),
"INSERT INTO scap2.epss_scores"
" (cve, epss, percentile)"
" VALUES ",
" ON CONFLICT (cve) DO NOTHING");

cJSON_ArrayForEach (list_item, epss_scores_list)
{
cJSON *cve_json, *epss_json, *percentile_json;

EPSS_JSON_FAIL_IF (! cJSON_IsObject (list_item),
"Unexpected non-object item in EPSS scores file")
gvm_json_pull_parser_next (&parser, &event);

Check warning on line 3465 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3465 

Added line #L3465 was not covered by tests
while (event.type == GVM_JSON_PULL_EVENT_OBJECT_START)
{
cJSON *cve_json, *epss_json, *percentile_json;

cve_json = cJSON_GetObjectItem (list_item, "cve");
epss_json = cJSON_GetObjectItem (list_item, "epss");
percentile_json = cJSON_GetObjectItem (list_item, "percentile");
epss_entry = gvm_json_pull_expand_container (&parser, &error_message);

Check warning on line 3470 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3470 

Added line #L3470 was not covered by tests

EPSS_JSON_FAIL_IF (cve_json == NULL,
"Item missing mandatory 'cve' field");
if (error_message)
{
g_warning ("%s: Error expanding EPSS item: %s", __func__, error_message);
g_free (error_message);
goto fail_insert;

Check warning on line 3476 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3474-L3476 

Added lines #L3474 - L3476 were not covered by tests
}

EPSS_JSON_FAIL_IF (epss_json == NULL,
"Item missing mandatory 'epss' field");

EPSS_JSON_FAIL_IF (percentile_json == NULL,
"Item missing mandatory 'percentile' field");
cve_json = cJSON_GetObjectItemCaseSensitive (epss_entry, "cve");
epss_json = cJSON_GetObjectItemCaseSensitive (epss_entry, "epss");
percentile_json = cJSON_GetObjectItemCaseSensitive (epss_entry, "percentile");

Check warning on line 3481 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3479-L3481 

Added lines #L3479 - L3481 were not covered by tests

EPSS_JSON_FAIL_IF (! cJSON_IsString (cve_json),
"Field 'cve' in item is not a string");
EPSS_JSON_FAIL_IF (cve_json == NULL,

Check warning on line 3483 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3483 

Added line #L3483 was not covered by tests
"Item missing mandatory 'cve' field");

EPSS_JSON_FAIL_IF (! cJSON_IsNumber(epss_json),
"Field 'epss' in item is not a number");

EPSS_JSON_FAIL_IF (! cJSON_IsNumber(percentile_json),
"Field 'percentile' in item is not a number");
EPSS_JSON_FAIL_IF (epss_json == NULL,

Check warning on line 3486 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3486 

Added line #L3486 was not covered by tests
"Item missing mandatory 'epss' field");

EPSS_JSON_FAIL_IF (percentile_json == NULL,

Check warning on line 3489 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3489 

Added line #L3489 was not covered by tests
"Item missing mandatory 'percentile' field");

EPSS_JSON_FAIL_IF (! cJSON_IsString (cve_json),

Check warning on line 3492 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3492 

Added line #L3492 was not covered by tests
"Field 'cve' in item is not a string");

EPSS_JSON_FAIL_IF (! cJSON_IsNumber(epss_json),

Check warning on line 3495 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3495 

Added line #L3495 was not covered by tests
"Field 'epss' in item is not a number");

EPSS_JSON_FAIL_IF (! cJSON_IsNumber(percentile_json),

Check warning on line 3498 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3498 

Added line #L3498 was not covered by tests
"Field 'percentile' in item is not a number");

insert_epss_score_entry (&inserts,
cve_json->valuestring,
epss_json->valuedouble,
percentile_json->valuedouble);
insert_epss_score_entry (&inserts,
cve_json->valuestring,

Check warning on line 3502 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3501-L3502 

Added lines #L3501 - L3502 were not covered by tests
epss_json->valuedouble,
percentile_json->valuedouble);

gvm_json_pull_parser_next (&parser, &event);
cJSON_Delete (epss_entry);

Check warning on line 3507 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3506-L3507 

Added lines #L3506 - L3507 were not covered by tests
}
}
else if (event.type == GVM_JSON_PULL_EVENT_ERROR)

Check warning on line 3510 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3510 

Added line #L3510 was not covered by tests
{
g_warning ("%s: Parser error: %s", __func__, event.error_message);
gvm_json_pull_event_cleanup (&event);
gvm_json_pull_parser_cleanup (&parser);
fclose (epss_scores_file);
return -1;

Check warning on line 3516 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3512-L3516 

Added lines #L3512 - L3516 were not covered by tests
}
else
{
g_warning ("%s: EPSS scores file is not a JSON object.", __func__);
gvm_json_pull_event_cleanup (&event);
gvm_json_pull_parser_cleanup (&parser);
fclose (epss_scores_file);
return -1;

Check warning on line 3524 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3520-L3524 

Added lines #L3520 - L3524 were not covered by tests
}

inserts_run (&inserts, TRUE);
sql_commit ();
cJSON_Delete (parsed);

gvm_json_pull_event_cleanup (&event);
gvm_json_pull_parser_cleanup (&parser);
fclose (epss_scores_file);

Check warning on line 3531 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3529-L3531 

Added lines #L3529 - L3531 were not covered by tests
return 0;

fail_insert:
inserts_free (&inserts);
sql_rollback ();
char *printed_item = cJSON_Print (list_item);
char *printed_item = cJSON_Print (epss_entry);

Check warning on line 3537 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3537 

Added line #L3537 was not covered by tests
g_message ("%s: invalid item: %s", __func__, printed_item);
cJSON_Delete (epss_entry);

Check warning on line 3539 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3539 

Added line #L3539 was not covered by tests
free (printed_item);
cJSON_Delete (parsed);
gvm_json_pull_event_cleanup (&event);
gvm_json_pull_parser_cleanup (&parser);
fclose (epss_scores_file);

Check warning on line 3543 in src/manage_sql_secinfo.c

View check run for this annotation

Codecov / codecov/patch

src/manage_sql_secinfo.c#L3541-L3543 

Added lines #L3541 - L3543 were not covered by tests
return -1;
}

Expand Down
Loading