diff --git a/tests/common/devices/sonic_asic.py b/tests/common/devices/sonic_asic.py index b08bc2616d..b6b76964da 100644 --- a/tests/common/devices/sonic_asic.py +++ b/tests/common/devices/sonic_asic.py @@ -163,6 +163,19 @@ def run_sonic_db_cli_cmd(self, sonic_db_cmd): cmd = "{} {}".format(self.sonic_db_cli, sonic_db_cmd) return self.sonichost.command(cmd, verbose=False) + def sonic_db_get_keys(self, db_id, pattern): + """ + Get all keys for a given pattern in given redis database for a particular asic + :param db_id: ID of redis database + :param pattern: Redis key pattern + :return: A list of key name in string + """ + cmd = '{} KEYS \"{}\"'.format(db_id, pattern) + logger.debug('Getting keys from asic{} redis by command: {}'.format(self.asic_index, cmd)) + output = self.run_sonic_db_cli_cmd(cmd) + content = output['stdout'].strip() + return content.split('\n') if content else None + def run_redis_cli_cmd(self, redis_cmd): if self.namespace != DEFAULT_NAMESPACE: redis_cli = "/usr/bin/redis-cli" diff --git a/tests/platform_tests/counterpoll/test_counterpoll_watermark.py b/tests/platform_tests/counterpoll/test_counterpoll_watermark.py index 4479e9168d..de2c731157 100644 --- a/tests/platform_tests/counterpoll/test_counterpoll_watermark.py +++ b/tests/platform_tests/counterpoll/test_counterpoll_watermark.py @@ -11,7 +11,6 @@ from tests.common.config_reload import config_reload from tests.common.fixtures.duthost_utils import backup_and_restore_config_db # noqa F401 from tests.common.helpers.assertions import pytest_assert -from tests.common.helpers.sonic_db import redis_get_keys from tests.common.utilities import get_inventory_files, get_host_visible_vars from tests.common.utilities import skip_release, wait_until from tests.common.reboot import reboot @@ -61,8 +60,13 @@ def dut_vars(duthosts, enum_rand_one_per_hwsku_hostname, request): yield dut_vars +def get_keys_on_asics(duthost, db_id, key): + return {asic.asic_index: asic.sonic_db_get_keys(db_id, key) for asic in duthost.asics} + + def check_counters_populated(duthost, key): - return bool(redis_get_keys(duthost, 'COUNTERS_DB', key)) + keys = get_keys_on_asics(duthost, "COUNTERS_DB", key) + return bool(keys.values()) def test_counterpoll_queue_watermark_pg_drop(duthosts, localhost, enum_rand_one_per_hwsku_hostname, dut_vars, @@ -132,51 +136,54 @@ def test_counterpoll_queue_watermark_pg_drop(duthosts, localhost, enum_rand_one_ for map_to_verify in maps_to_verify: map_prefix = map_to_verify['prefix'] maps = map_to_verify[MAPS] - map_output = redis_get_keys(duthost, 'COUNTERS_DB', MAPS_LONG_PREFIX.format(map_prefix)) - map = [] - failed = "" - for map_entry in maps: - map.append(map_entry) - msg = "no {} maps found in COUNTERS_DB".format(map_prefix) - pytest_assert(map_output, msg) - for line in map_output: - try: - map.remove(line) - except ValueError: - failed = "MAP {} was not found in {} MAPS list".format(line, map_prefix) - pytest_assert("" == failed, failed) - pytest_assert(len(map) == 0, "{} maps mismatch, one or more queue was not found in redis COUNTERS_DB" - .format(map_prefix)) + map_outputs = get_keys_on_asics(duthost, 'COUNTERS_DB', MAPS_LONG_PREFIX.format(map_prefix)) + for asic_idx, map_output in map_outputs.items(): + map = [] + failed = "" + for map_entry in maps: + map.append(map_entry) + msg = "no {} maps found in COUNTERS_DB on asic{}".format(map_prefix, asic_idx) + pytest_assert(map_output, msg) + for line in map_output: + try: + map.remove(line) + except ValueError: + failed = "MAP {} was not found in {} MAPS list".format(line, map_prefix) + pytest_assert("" == failed, failed) + pytest_assert(len(map) == 0, + "{} maps mismatch, one or more queue was not found in redis COUNTERS_DB on asic{}" + .format(map_prefix, asic_idx)) failed_list = [] with allure.step("Verifying {} STATS in FLEX_COUNTER_DB on {}...".format(tested_counterpoll, duthost.hostname)): - stats_output = redis_get_keys(duthost, 'FLEX_COUNTER_DB', '*{}*'.format(map_prefix)) - counted = 0 - # build expected counterpoll stats vs unexpected - expected_types = [] - unexpected_types = [] - for counterpoll, v in list(RELEVANT_MAPS.items()): - types_to_check = v[CounterpollConstants.TYPE] - if counterpoll in tested_counterpoll: - for type in types_to_check: - expected_types.append(FLEX_COUNTER_PREFIX + type) - else: - for type in types_to_check: - unexpected_types.append(FLEX_COUNTER_PREFIX + type) - logging.info("expected types for for counterpoll {}:\n{}".format(tested_counterpoll, expected_types)) - logging.info("unexpected types for for counterpoll {}:\n{}".format(tested_counterpoll, unexpected_types)) - for line in stats_output: - for expected in expected_types: - if expected in line: - counted += 1 - for unexpected in unexpected_types: - if unexpected in line: - failed_list.append("found for {} unexpected stat counter in FLEX_COUNTER_DB: {}" - .format(tested_counterpoll, line)) - logging.info("counted {} {} STATs type in FLEX_COUNTER_DB on {}..." - .format(counted, tested_counterpoll, duthost.hostname)) - pytest_assert(len(failed_list) == 0, failed_list) - pytest_assert(counted > 0, "counted {} for {}".format(counted, tested_counterpoll)) + stats_outputs = get_keys_on_asics(duthost, 'FLEX_COUNTER_DB', '*{}*'.format(map_prefix)) + for asic_idx, stats_output in stats_outputs.items(): + counted = 0 + # build expected counterpoll stats vs unexpected + expected_types = [] + unexpected_types = [] + for counterpoll, v in list(RELEVANT_MAPS.items()): + types_to_check = v[CounterpollConstants.TYPE] + if counterpoll in tested_counterpoll: + for type in types_to_check: + expected_types.append(FLEX_COUNTER_PREFIX + type) + else: + for type in types_to_check: + unexpected_types.append(FLEX_COUNTER_PREFIX + type) + logging.info("expected types for for counterpoll {}:\n{}".format(tested_counterpoll, expected_types)) + logging.info("unexpected types for for counterpoll {}:\n{}".format(tested_counterpoll, unexpected_types)) + for line in stats_output: + for expected in expected_types: + if expected in line: + counted += 1 + for unexpected in unexpected_types: + if unexpected in line: + failed_list.append("found for {} unexpected stat counter in FLEX_COUNTER_DB on asic{}: {}" + .format(tested_counterpoll, asic_idx, line)) + logging.info("counted {} {} STATs type in FLEX_COUNTER_DB on {} asic{}..." + .format(counted, tested_counterpoll, duthost.hostname, asic_idx)) + pytest_assert(len(failed_list) == 0, failed_list) + pytest_assert(counted > 0, "counted {} for {}".format(counted, tested_counterpoll)) # for watermark only, also count stats with actual values in COUNTERS_DB if CounterpollConstants.WATERMARK in tested_counterpoll: @@ -193,19 +200,24 @@ def test_counterpoll_queue_watermark_pg_drop(duthosts, localhost, enum_rand_one_ # count FLEXCOUNTER_DB countrpolls and put in results dict key per countrpoll with allure.step("check all counterpolls {} results on {} ...".format(RELEVANT_COUNTERPOLLS, duthost.hostname)): for counterpoll in RELEVANT_COUNTERPOLLS: - counted_dict[counterpoll] = 0 + counted_dict[counterpoll] = {} + for asic in duthost.asics: + counted_dict[counterpoll][asic.asic_index] = 0 for map_prefix in MAPS_PREFIX_FOR_ALL_COUNTERPOLLS: - stats_output = redis_get_keys(duthost, 'FLEX_COUNTER_DB', '*{}*'.format(map_prefix)) - for line in stats_output: - for counterpoll, v in list(RELEVANT_MAPS.items()): - types_to_check = v[CounterpollConstants.TYPE] - for type in types_to_check: - if type in line: - counted_dict[counterpoll] += 1 + stats_outputs = get_keys_on_asics(duthost, 'FLEX_COUNTER_DB', '*{}*'.format(map_prefix)) + for asic_idx, stats_output in stats_outputs.items(): + for line in stats_output: + for counterpoll, v in list(RELEVANT_MAPS.items()): + types_to_check = v[CounterpollConstants.TYPE] + for type in types_to_check: + if type in line: + counted_dict[counterpoll][asic_idx] += 1 logging.info("counted_dict {}".format(counted_dict)) # verify each queue/watermark/pg-drop counterpoll has stats in FLEX_COUNTER_DB for counterpoll in RELEVANT_COUNTERPOLLS: - pytest_assert(counted_dict[counterpoll] > 0) + for asic in duthost.asics: + pytest_assert(counted_dict[counterpoll][asic.asic_index] > 0, + "No stats in FLEX_COUNTER_DB for {} asic{}".format(duthost.hostname, asic.asic_index)) # for watermark only, also count stats with actual values in COUNTERS_DB if CounterpollConstants.WATERMARK in tested_counterpoll: with allure.step("counting watermark STATS in FLEX_COUNTER_DB on {}...".format(duthost.hostname)): @@ -236,14 +248,15 @@ def verify_counterpoll_status(duthost, counterpoll_list, expected): def count_watermark_stats_in_counters_db(duthost): - watermark_stats_output = redis_get_keys(duthost, 'COUNTERS_DB', '*{}*' - .format(CounterpollConstants.WATERMARK.upper())) - watermark_stats = {} - for watermark_type in WATERMARK_COUNTERS_DB_STATS_TYPE: - watermark_stats[watermark_type] = 0 - for line in watermark_stats_output: - if watermark_type in line: - watermark_stats[watermark_type] += 1 - logging.info("watermark_stats {}".format(watermark_stats)) - for k, v in list(watermark_stats.items()): - pytest_assert(v > 0, "watermark_stats {} in COUNTERS_DB: {}, expected > 0".format(k, v)) + watermark_stats_output = get_keys_on_asics(duthost, 'COUNTERS_DB', '*{}*' + .format(CounterpollConstants.WATERMARK.upper())) + for asic_idx, output in watermark_stats_output.items(): + watermark_stats = {} + for watermark_type in WATERMARK_COUNTERS_DB_STATS_TYPE: + watermark_stats[watermark_type] = 0 + for line in output: + if watermark_type in line: + watermark_stats[watermark_type] += 1 + logging.info("watermark_stats on {} {}".format(asic_idx, watermark_stats)) + for k, v in list(watermark_stats.items()): + pytest_assert(v > 0, "watermark_stats {} in COUNTERS_DB on asic{}: {}, expected > 0".format(k, asic_idx, v))