Skip to content

Commit

Permalink
fix: refactor load (#1129)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ajasnosz
ajasnosz authored Dec 3, 2024
1 parent 58d66d0 commit d6341a1
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 37 deletions.
49 changes: 37 additions & 12 deletions splunk_connect_for_snmp/inventory/loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,6 @@ def assign_inventory_values(inventory_ui_collection):

def load():
inventory_errors = False
target = None
# DB managers initialization
mongo_client = pymongo.MongoClient(MONGO_URI)
profiles_manager = ProfilesManager(mongo_client)
Expand Down Expand Up @@ -205,28 +204,45 @@ def load():
logger.info(f"Loading inventory from {INVENTORY_PATH}")
inventory_lines, inventory_group_port_mapping = inventory_processor.get_all_hosts()

# Function to delete inventory records that are
# Function to delete inventory records that are in groups
hosts_from_groups_to_delete = return_hosts_from_deleted_groups(
previous_groups, new_groups, inventory_group_port_mapping
)

inventory_errors = manage_inventory_records(
config_profiles,
expiry_time_changed,
hosts_from_groups_to_delete,
inventory_errors,
inventory_lines,
inventory_record_manager,
mongo_client,
)

return inventory_errors


def manage_inventory_records(
config_profiles,
expiry_time_changed,
hosts_from_groups_to_delete,
inventory_errors,
inventory_lines,
inventory_record_manager,
mongo_client,
):
# Remove hosts from inventory that were deleted from group
for host in hosts_from_groups_to_delete:
inventory_record_manager.delete(host)

# Update or remove inventory and inventory_ui records
for new_source_record in inventory_lines:
try:
ir = InventoryRecord(**new_source_record)
target = transform_address_to_key(ir.address, ir.port)
if ir.delete:
inventory_record_manager.delete(target)
if CONFIG_FROM_MONGO:
if ir.group is None:
mongo_client.sc4snmp.inventory_ui.delete_one(
{"address": ir.address, "port": ir.port}
)
else:
mongo_client.sc4snmp.inventory_ui.delete_one(
{"address": ir.group}
)
delete_from_ui_inventory(ir, mongo_client)
else:
inventory_record_manager.update(
ir, new_source_record, config_profiles, expiry_time_changed
Expand All @@ -245,10 +261,19 @@ def load():
new_source_record["address"], new_source_record["port"]
)
logger.exception(f"Exception raised for {target}: {e}")

return inventory_errors


def delete_from_ui_inventory(ir, mongo_client):
if CONFIG_FROM_MONGO:
if ir.group is None:
mongo_client.sc4snmp.inventory_ui.delete_one(
{"address": ir.address, "port": ir.port}
)
else:
mongo_client.sc4snmp.inventory_ui.delete_one({"address": ir.group})


if __name__ == "__main__":
r = load()
if r:
Expand Down
42 changes: 17 additions & 25 deletions ui_tests/tests/test_splunk_integration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
from splunk_search import check_events_from_splunk
from webdriver.webriver_factory import WebDriverFactory

DEFAULT_PORT = 161

logger = Logger().get_logger()
driver = WebDriverFactory().get_driver()
p_header = HeaderPage()
Expand Down Expand Up @@ -77,7 +79,7 @@ def test_applying_changes_for_device_that_does_not_exists(setup):

# check data in Splunk
# check walk scheduled
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};walk"'
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};walk"'
events = check_events_from_splunk(
start_time="-3m@m",
url=setup["splunkd_url"],
Expand All @@ -89,7 +91,7 @@ def test_applying_changes_for_device_that_does_not_exists(setup):
assert len(events) == 1

# check no profiles polling
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};*;poll"'
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};*;poll"'
events = check_events_from_splunk(
start_time="-3m@m",
url=setup["splunkd_url"],
Expand Down Expand Up @@ -175,7 +177,7 @@ def test_setting_group_in_inventory(setup):

# check data in Splunk
# check walk scheduled
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};walk"'
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};walk"'
events = check_events_from_splunk(
start_time="-2m@m",
url=setup["splunkd_url"],
Expand All @@ -187,9 +189,7 @@ def test_setting_group_in_inventory(setup):
assert len(events) == 1

# check profiles polling
search_query = (
f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};12;poll"'
)
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};12;poll"'
events = check_events_from_splunk(
start_time="-1m@m",
url=setup["splunkd_url"],
Expand All @@ -201,9 +201,7 @@ def test_setting_group_in_inventory(setup):
assert len(events) > 1

# checking smart profiles not working
search_query = (
f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};600;poll"'
)
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};600;poll"'
events = check_events_from_splunk(
start_time="-2m@m",
url=setup["splunkd_url"],
Expand All @@ -215,9 +213,7 @@ def test_setting_group_in_inventory(setup):
assert len(events) == 0

# checking polling for mandatory profile - 1200 - this should be visible even when smart profiles are disabled
search_query = (
f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};1200;poll"'
)
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};1200;poll"'
events = check_events_from_splunk(
start_time="-2m@m",
url=setup["splunkd_url"],
Expand Down Expand Up @@ -290,7 +286,7 @@ def test_setting_host_in_inventory(setup):
p_profiles.add_varbind("SNMPv2-MIB", "sysName")
p_profiles.click_submit_button()

p_header.switch_to_profiles()
# p_header.switch_to_profiles()
p_profiles.click_add_profile_button()
p_profiles.set_profile_name(base_profile_name)
p_profiles.select_profile_type("base")
Expand All @@ -316,7 +312,7 @@ def test_setting_host_in_inventory(setup):

# check data in Splunk
# check walk scheduled
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};walk"'
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};walk"'
events = check_events_from_splunk(
start_time="-1m@m",
url=setup["splunkd_url"],
Expand All @@ -329,9 +325,7 @@ def test_setting_host_in_inventory(setup):

# check profiles polling
time.sleep(60) # wait to be sure that profile are being polled
search_query = (
f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};10;poll"'
)
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};10;poll"'
events = check_events_from_splunk(
start_time="-1m@m",
url=setup["splunkd_url"],
Expand All @@ -342,7 +336,7 @@ def test_setting_host_in_inventory(setup):
logger.info("Splunk received %s events in the last minute", len(events))
assert len(events) > 1

search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};7;poll"'
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};7;poll"'
events = check_events_from_splunk(
start_time="-1m@m",
url=setup["splunkd_url"],
Expand All @@ -354,7 +348,7 @@ def test_setting_host_in_inventory(setup):
assert len(events) > 1

# checking smart/base profiles
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};5;poll"'
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};5;poll"'
events = check_events_from_splunk(
start_time="-1m@m",
url=setup["splunkd_url"],
Expand Down Expand Up @@ -390,7 +384,7 @@ def test_setting_host_in_inventory(setup):
time.sleep(time_to_upgrade + 70) # wait for upgrade + walk time + polling

# check walk scheduled
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};walk"'
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};walk"'
events = check_events_from_splunk(
start_time="-1m@m",
url=setup["splunkd_url"],
Expand All @@ -403,9 +397,7 @@ def test_setting_host_in_inventory(setup):

# check profiles polling
time.sleep(60) # wait to be sure that disabled profile is no more polled
search_query = (
f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};10;poll"'
)
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};10;poll"'
events = check_events_from_splunk(
start_time="-1m@m",
url=setup["splunkd_url"],
Expand All @@ -416,7 +408,7 @@ def test_setting_host_in_inventory(setup):
logger.info("Splunk received %s events in the last minute", len(events))
assert len(events) == 0

search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};7;poll"'
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};7;poll"'
events = check_events_from_splunk(
start_time="-1m@m",
url=setup["splunkd_url"],
Expand All @@ -428,7 +420,7 @@ def test_setting_host_in_inventory(setup):
assert len(events) > 1

# checking smart/base profiles - no polling
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host};5;poll"'
search_query = f'index={config.LOGS_INDEX} "Sending due task sc4snmp;{host}:{DEFAULT_PORT};5;poll"'
events = check_events_from_splunk(
start_time="-1m@m",
url=setup["splunkd_url"],
Expand Down

0 comments on commit d6341a1

Please sign in to comment.