From e2b88e9208a29a89a2dad60801d331d24d208884 Mon Sep 17 00:00:00 2001 From: Williams Date: Tue, 24 Sep 2024 14:32:58 -0400 Subject: [PATCH 1/2] reordering post processing for efficiency --- tr_sys/tr_ars/utils.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/tr_sys/tr_ars/utils.py b/tr_sys/tr_ars/utils.py index 25b61e3e..23780ed9 100644 --- a/tr_sys/tr_ars/utils.py +++ b/tr_sys/tr_ars/utils.py @@ -577,20 +577,15 @@ def pre_merge_process(data,key, agent_name,inforesid): def post_process(mesg,key, agent_name): data = mesg.decompress_dict() - logging.info("Pre node annotation for agent %s pk: %s" % (agent_name, str(key))) + + logging.info("pre blocklist for "+str(key)) try: - annotate_nodes(mesg,data,agent_name) - logging.info("node annotation successful for agent %s and pk: %s" % (agent_name, str(key))) + remove_blocked(mesg, data) except Exception as e: status='E' code=444 - log_tuple =[ - f'node annotation internal error: {str(e)}', - datetime.now().strftime('%H:%M:%S'), - "DEBUG" - ] - add_log_entry(data,log_tuple) - logging.exception(f"problem with node annotation for agent: {agent_name} pk: {str(key)}") + logging.info(e.__cause__) + logging.exception(f"Problem with block list removal for agent: {agent_name} pk: {str(key)}") mesg.status=status mesg.code=code mesg.save() @@ -612,18 +607,28 @@ def post_process(mesg,key, agent_name): mesg.status=status mesg.code=code mesg.save() - logging.info("pre blocklist for "+str(key)) + + + + logging.info("Pre node annotation for agent %s pk: %s" % (agent_name, str(key))) try: - remove_blocked(mesg, data) + annotate_nodes(mesg,data,agent_name) + logging.info("node annotation successful for agent %s and pk: %s" % (agent_name, str(key))) except Exception as e: status='E' code=444 - logging.info(e.__cause__) - logging.exception(f"Problem with block list removal for agent: {agent_name} pk: {str(key)}") + log_tuple =[ + f'node annotation internal error: {str(e)}', + datetime.now().strftime('%H:%M:%S'), + "DEBUG" + ] + add_log_entry(data,log_tuple) + logging.exception(f"problem with node annotation for agent: {agent_name} pk: {str(key)}") mesg.status=status mesg.code=code mesg.save() + logging.info("pre appraiser for agent %s and pk %s" % (agent_name, str(key))) try: appraise(mesg,data,agent_name) @@ -671,7 +676,7 @@ def post_process(mesg,key, agent_name): ] add_log_entry(data,log_tuple) status ='E' - code=400 + code=444 mesg.save_compressed_dict(data) return mesg, code, status @@ -970,7 +975,7 @@ def scrub_null_attributes(data): -def appraise(mesg,data, agent_name,retry_counter=0): +def appraise(mesg, data, agent_name,retry_counter=0): headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} json_data = json.dumps(data) logging.info('sending data for agent: %s to APPRAISER URL: %s' % (agent_name, APPRAISER_URL)) From 999871f50497b7e7ccea4f9ddd70bcbc91a0862f Mon Sep 17 00:00:00 2001 From: Williams Date: Wed, 25 Sep 2024 13:39:43 -0400 Subject: [PATCH 2/2] reordering and adding timeout to appraise --- tr_sys/tr_ars/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tr_sys/tr_ars/utils.py b/tr_sys/tr_ars/utils.py index 23780ed9..f8b2f341 100644 --- a/tr_sys/tr_ars/utils.py +++ b/tr_sys/tr_ars/utils.py @@ -981,7 +981,7 @@ def appraise(mesg, data, agent_name,retry_counter=0): logging.info('sending data for agent: %s to APPRAISER URL: %s' % (agent_name, APPRAISER_URL)) with tracer.start_as_current_span("get_appraisal") as span: try: - with requests.post(APPRAISER_URL,data=json_data,headers=headers, stream=True) as r: + with requests.post(APPRAISER_URL,data=json_data,headers=headers, stream=True,timeout=600) as r: logging.info("Appraiser being called at: "+APPRAISER_URL) logging.info('the response for agent %s to appraiser code is: %s' % (agent_name, r.status_code)) if r.status_code==200: