From fa886931500d8b233759d271eae3bc64cbbff280 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:36:55 +0100 Subject: [PATCH 01/29] Update README.md --- README.md | 74 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 24cd542..43d74c6 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,67 @@ # netbox-nmap-scan -This is a simple Python scripts that achieve the purpose of keeping an updated list of IP Address which are active/responding in your network. -To achieve that we are using nmap. +Automatically maintain an up-to-date inventory of active IP addresses in your network using Netbox and nmap. This Python-based tool scans your network prefixes and keeps your Netbox instance synchronized with the current state of your network. -By default the scripts is scanning all prefixes with the active status inside your Netbox instance. -If you don't want a prefix to get scan, create a new tag 'Disable Automatic Scanning' +## Features -![image](https://github.com/henrionlo/netbox-nmap-scan/assets/139378145/b7a223ae-3a55-42cb-8f28-87d282e103c8) +- Automatic scanning of all active prefixes in Netbox +- Custom tag support for excluding prefixes from scanning +- Tracking of last scan time for each IP address +- DNS resolution support +- Compatible with Python 3.12.6 and Netbox 4.1.10 -Create the tag 'autoscan', this will allow you to quickly know which IP Addresses has been added by the script. +## Prerequisites -![image](https://github.com/henrionlo/netbox-nmap-scan/assets/139378145/435cec58-1f92-42f2-b4eb-1448a4d22161) +- Python 3.12.6 or later +- Netbox 4.1.10 or later +- nmap installed on your system +- Required Python packages (listed in requirements.txt) -And create the following custom field in Customization, this way you can see when was the last time an ip address has been pinged by the scanning engine. +## Setup -![image](https://github.com/LoH-lu/netbox-nmap-scan/assets/139378145/c812ee55-71d0-4d8e-9b14-f337a5d867a5) +1. Create the following Netbox configurations: -The more prefixes you want to scan, the more time it will require to finish. + ### Tags + - `autoscan`: Identifies IP addresses added by this script + - `Disable Automatic Scanning`: Add this tag to prefixes you want to exclude from scanning + + ![Disable Scanning Tag Configuration](https://github.com/henrionlo/netbox-nmap-scan/assets/139378145/b7a223ae-3a55-42cb-8f28-87d282e103c8) + + ![Autoscan Tag Configuration](https://github.com/henrionlo/netbox-nmap-scan/assets/139378145/435cec58-1f92-42f2-b4eb-1448a4d22161) -Tested and working with Python 3.12.2 - 3.12.4 and Netbox 3.6.x - 4.0.x + ### Custom Fields + Add a custom field to track the last scan time for each IP address: + + ![Last Scan Time Custom Field](https://github.com/LoH-lu/netbox-nmap-scan/assets/139378145/c812ee55-71d0-4d8e-9b14-f337a5d867a5) -The How-To are located in https://github.com/henrionlo/netbox-nmap-scan/wiki +2. Follow the detailed installation guide in our [Wiki](https://github.com/henrionlo/netbox-nmap-scan/wiki) -TODO -- Add DNS server selection for the nmap command in the ini file (if required to have a different one from the system DNS running the script) -- Allow users to disable the DNS part of the script and only run the regular nmap command -- Cleanup of code and import -- Adding more description -- Better logging of errors and debug output -- All-in-One script for easier setup +## Usage + +The script will scan all prefixes with active status in your Netbox instance by default. Scanning time increases with the number of prefixes being scanned. + +For detailed usage instructions and examples, please refer to our [Wiki](https://github.com/henrionlo/netbox-nmap-scan/wiki). + +## Performance Considerations + +- Scanning time scales with the number of prefixes +- Consider scheduling scans during off-peak hours for large networks +- Use the `Disable Automatic Scanning` tag strategically to optimize scan times + +## Roadmap + +- [ ] DNS server configuration in INI file for custom DNS resolution +- [ ] Option to disable DNS resolution functionality +- [ ] Toggle for last scan time tracking +- [ ] All-in-One setup script for easier deployment + +## Contributing + +Contributions are welcome! Please feel free to submit a Pull Request. + +## Support + +For issues, questions, or contributions, please: +1. Check our [Wiki](https://github.com/henrionlo/netbox-nmap-scan/wiki) +2. Open an issue in this repository +3. Submit a pull request with your proposed changes From 79e5fa4f544374c93f00a3b4b7f56b2097c4e794 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:38:44 +0100 Subject: [PATCH 02/29] Update netbox_retrieve.py --- netbox_retrieve.py | 227 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 195 insertions(+), 32 deletions(-) diff --git a/netbox_retrieve.py b/netbox_retrieve.py index 3e4b14a..e330fc2 100644 --- a/netbox_retrieve.py +++ b/netbox_retrieve.py @@ -1,50 +1,213 @@ +#!/usr/bin/env python3 +""" +Netbox IPAM Prefixes Export Script. + +This script exports IPAM prefixes from a Netbox instance to a CSV file. +It includes comprehensive logging and error handling to ensure reliable operation. + +Requirements: + - pynetbox + - configparser +""" + import csv import os +import logging +import sys +from typing import List, Optional import configparser +from datetime import datetime +import pynetbox import netbox_connection -# Get the directory of the current script -script_dir = os.path.dirname(os.path.abspath(__file__)) +# Script configuration +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +LOG_DIR = os.path.join(SCRIPT_DIR, 'logs') +CSV_HEADERS = ['Prefix', 'VRF', 'Status', 'Tags', 'Tenant'] -def get_ipam_prefixes(netbox): +# Ensure log directory exists +os.makedirs(LOG_DIR, exist_ok=True) + +def setup_logging() -> logging.Logger: + """ + Configure logging with both file and console handlers. + + Returns: + logging.Logger: Configured logger instance + """ + # Create logger + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + + # Create formatters + file_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s' + ) + console_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(message)s' + ) + + # Create file handlers + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + debug_handler = logging.FileHandler( + os.path.join(LOG_DIR, f'netbox_export_debug_{timestamp}.log') + ) + debug_handler.setLevel(logging.DEBUG) + debug_handler.setFormatter(file_formatter) + + error_handler = logging.FileHandler( + os.path.join(LOG_DIR, f'netbox_export_error_{timestamp}.log') + ) + error_handler.setLevel(logging.ERROR) + error_handler.setFormatter(file_formatter) + + # Create console handler + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(console_formatter) + + # Add handlers to logger + logger.addHandler(debug_handler) + logger.addHandler(error_handler) + logger.addHandler(console_handler) + + return logger + +def get_ipam_prefixes( + netbox_instance: pynetbox.api +) -> Optional[List[pynetbox.models.ipam.Prefixes]]: """ Retrieve all IPAM prefixes from Netbox. Args: - - netbox (pynetbox.core.api.Api): The Netbox API object. + netbox_instance: The Netbox API object Returns: - - ipam_prefixes (pynetbox.core.query.Request): All IPAM prefixes retrieved from Netbox. + Optional[List[pynetbox.models.ipam.Prefixes]]: List of IPAM prefixes or None if error + + Raises: + Exception: If there's an error retrieving prefixes """ - ipam_prefixes = netbox.ipam.prefixes.all() - return ipam_prefixes + logger = logging.getLogger(__name__) + logger.info("Retrieving IPAM prefixes from Netbox") + + try: + ipam_prefixes = list(netbox_instance.ipam.prefixes.all()) + logger.info(f"Successfully retrieved {len(ipam_prefixes)} IPAM prefixes") + logger.debug(f"First prefix retrieved: {ipam_prefixes[0].prefix if ipam_prefixes else 'None'}") + return ipam_prefixes -def write_to_csv(data, filename): + except Exception: + logger.error("Failed to retrieve IPAM prefixes", exc_info=True) + raise + +def write_to_csv(data: List[pynetbox.models.ipam.Prefixes], filename: str) -> None: """ Write IPAM prefixes data to a CSV file. Args: - - data (pynetbox.core.query.Request): IPAM prefixes data retrieved from Netbox. - - filename (str): Name of the CSV file to write data to. - """ - file_path = os.path.join(script_dir, filename) # Construct the full path to the output file - with open(file_path, 'w', newline='') as file: - writer = csv.writer(file) - writer.writerow(['Prefix', 'VRF', 'Status', 'Tags', 'Tenant']) # Writing headers - for prefix in data: - tag_names = [tag.name for tag in prefix.tags] - tenant_name = prefix.tenant.name if prefix.tenant else 'N/A' - status_value = prefix.status.value if prefix.status else 'N/A' # Extract the value of the status field - vrf_name = prefix.vrf.name if prefix.vrf else 'N/A' # Extract the name of the VRF - writer.writerow([prefix.prefix, vrf_name, status_value, ', '.join(tag_names), tenant_name]) - -# Read URL and token from var.ini -config = configparser.ConfigParser() -config.read(os.path.join(script_dir, 'var.ini')) -url = config['credentials']['url'] -token = config['credentials']['token'] - -netbox = netbox_connection.connect_to_netbox(url, token) - -ipam_prefixes = get_ipam_prefixes(netbox) -write_to_csv(ipam_prefixes, 'ipam_prefixes.csv') + data: IPAM prefixes data retrieved from Netbox + filename: Name of the CSV file to write data to + + Raises: + Exception: If there's an error writing to the CSV file + """ + logger = logging.getLogger(__name__) + file_path = os.path.join(SCRIPT_DIR, filename) + + logger.info(f"Starting CSV export to: {file_path}") + logger.debug(f"Total records to write: {len(data)}") + + try: + with open(file_path, 'w', newline='', encoding='utf-8') as file: + writer = csv.writer(file) + writer.writerow(CSV_HEADERS) + + for prefix in data: + try: + # Extract data with safe fallbacks + tag_names = [tag.name for tag in prefix.tags] if hasattr(prefix, 'tags') else [] + tenant_name = prefix.tenant.name if prefix.tenant else 'N/A' + status_value = prefix.status.value if prefix.status else 'N/A' + vrf_name = prefix.vrf.name if prefix.vrf else 'N/A' + + writer.writerow([ + prefix.prefix, + vrf_name, + status_value, + ', '.join(tag_names), + tenant_name + ]) + logger.debug(f"Wrote prefix: {prefix.prefix}") + + except AttributeError as attr_err: + logger.warning(f"Missing attribute while processing prefix {prefix.prefix}: {str(attr_err)}") + except Exception: + logger.error(f"Error processing row for prefix {prefix.prefix}",exc_info=True) + + logger.info(f"Successfully exported data to {filename}") + + except Exception: + logger.error(f"Failed to write to CSV file: {filename}", exc_info=True) + raise + +def load_config() -> tuple: + """ + Load configuration from var.ini file. + + Returns: + tuple: (url, token) from configuration + + Raises: + Exception: If there's an error reading the configuration + """ + logger = logging.getLogger(__name__) + config_path = os.path.join(SCRIPT_DIR, 'var.ini') + + try: + logger.debug(f"Reading configuration from: {config_path}") + config = configparser.ConfigParser() + config.read(config_path) + + url = config['credentials']['url'] + token = config['credentials']['token'] + + logger.debug("Successfully loaded configuration") + return url, token + + except Exception: + logger.error(f"Failed to load configuration from {config_path}", exc_info=True) + raise + +def main() -> None: + """ + Main entry point of the script. + + Coordinates the export of IPAM prefixes from Netbox to CSV. + """ + logger = setup_logging() + logger.info("Starting Netbox IPAM export process") + + try: + # Load configuration + url, token = load_config() + + # Connect to Netbox + logger.info("Connecting to Netbox...") + netbox_instance = netbox_connection.connect_to_netbox(url, token) + logger.info("Successfully connected to Netbox") + + # Retrieve and export data + ipam_prefixes = get_ipam_prefixes(netbox_instance) + if ipam_prefixes: + write_to_csv(ipam_prefixes, 'ipam_prefixes.csv') + logger.info("Export process completed successfully") + else: + logger.warning("No IPAM prefixes found to export") + + except Exception: + logger.error("Script execution failed", exc_info=True) + sys.exit(1) + +if __name__ == "__main__": + main() From d255db4201dbafcdeb48681eb316573e08af5aae Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:39:03 +0100 Subject: [PATCH 03/29] Update and rename netbox_retrieve.py to netbox_export.py --- netbox_retrieve.py => netbox_export.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename netbox_retrieve.py => netbox_export.py (100%) diff --git a/netbox_retrieve.py b/netbox_export.py similarity index 100% rename from netbox_retrieve.py rename to netbox_export.py From c39030f594ef703c81e8b5fbed147dfa4d27b62e Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:39:21 +0100 Subject: [PATCH 04/29] Update and rename netbox_push.py to netbox_import.py --- netbox_import.py | 294 +++++++++++++++++++++++++++++++++++++++++++++++ netbox_push.py | 156 ------------------------- 2 files changed, 294 insertions(+), 156 deletions(-) create mode 100644 netbox_import.py delete mode 100644 netbox_push.py diff --git a/netbox_import.py b/netbox_import.py new file mode 100644 index 0000000..3de35a9 --- /dev/null +++ b/netbox_import.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +""" +Netbox IP Address Management Script. + +This script provides functionality to import and update IP address data from a CSV file +into a Netbox instance. It supports concurrent processing of records and includes +comprehensive logging of all operations. + +Requirements: + - pynetbox + - tqdm + - configparser + - urllib3 +""" + +import csv +import os +import logging +from concurrent.futures import ThreadPoolExecutor, as_completed +import configparser +from typing import Dict, List, Optional +import urllib3 +import pynetbox +from tqdm import tqdm +from netbox_connection import connect_to_netbox + +# Disable insecure HTTPS warnings +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +# Script configuration +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +LOG_DIR = os.path.join(SCRIPT_DIR, 'logs') +MAX_WORKERS = 5 + +# Ensure log directory exists +os.makedirs(LOG_DIR, exist_ok=True) + +# Configure logging with both file and console handlers +def setup_logging() -> logging.Logger: + """ + Configure logging with both file and console handlers. + + Returns: + logging.Logger: Configured logger instance + """ + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + + # Create formatters + file_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s' + ) + console_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(message)s' + ) + + # File handler for debugging (detailed logs) + debug_handler = logging.FileHandler( + os.path.join(LOG_DIR, 'netbox_import_debug.log') + ) + debug_handler.setLevel(logging.DEBUG) + debug_handler.setFormatter(file_formatter) + + # File handler for errors + error_handler = logging.FileHandler( + os.path.join(LOG_DIR, 'netbox_import_error.log') + ) + error_handler.setLevel(logging.ERROR) + error_handler.setFormatter(file_formatter) + + # Console handler + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(console_formatter) + + # Add all handlers to logger + logger.addHandler(debug_handler) + logger.addHandler(error_handler) + logger.addHandler(console_handler) + + return logger + +def parse_tags(tags_string: str) -> List[Dict[str, str]]: + """ + Convert a comma-separated string of tags into a list of tag dictionaries. + + Args: + tags_string (str): Comma-separated string of tags + + Returns: + List[Dict[str, str]]: List of tag dictionaries + """ + return [{'name': tag.strip()} for tag in tags_string.split(',') if tag.strip()] + +def process_row(row: Dict[str, str], pbar: tqdm, netbox_instance: pynetbox.api) -> None: + """ + Process a single row from the CSV file and update/create IP addresses in Netbox. + + Args: + row (Dict[str, str]): Dictionary representing a single row from the CSV file + pbar (tqdm): Progress bar instance + netbox_instance (pynetbox.api): Netbox API instance + + Raises: + Exception: If there's an error processing the row + """ + logger = logging.getLogger(__name__) + address = row.get('address', 'unknown') + + try: + logger.debug(f"Starting to process address: {address}") + + # Parse tags + tags_list = parse_tags(row['tags']) + logger.debug(f"Parsed tags for {address}: {tags_list}") + + # Prepare tenant and VRF data + tenant_data = {'name': row['tenant']} if row['tenant'] != 'N/A' else None + vrf_data = {'name': row['VRF']} if row['VRF'] != 'N/A' else None + + # Get existing address + existing_address = netbox_instance.ipam.ip_addresses.get(address=address) + + if existing_address: + _update_existing_address( + existing_address, row, tags_list, tenant_data, vrf_data + ) + else: + _create_new_address( + netbox_instance, address, row, tags_list, tenant_data, vrf_data + ) + + except Exception: + logger.error(f"Failed to process row for address {address}", exc_info=True) + raise + finally: + pbar.update(1) + +def _update_existing_address( + existing_address: object, + row: Dict[str, str], + tags_list: List[Dict[str, str]], + tenant_data: Optional[Dict[str, str]], + vrf_data: Optional[Dict[str, str]] +) -> None: + """ + Update an existing IP address in Netbox. + + Args: + existing_address: Existing Netbox IP address object + row (Dict[str, str]): Row data from CSV + tags_list (List[Dict[str, str]]): Processed tags + tenant_data (Optional[Dict[str, str]]): Tenant information + vrf_data (Optional[Dict[str, str]]): VRF information + """ + logger = logging.getLogger(__name__) + logger.info(f"Updating existing address: {row['address']}") + + try: + existing_address.status = row['status'] + existing_address.custom_fields = {'scantime': row['scantime']} + existing_address.dns_name = row['dns_name'] + existing_address.tags = tags_list + existing_address.tenant = tenant_data + existing_address.vrf = vrf_data + + existing_address.save() + logger.debug(f"Successfully updated address {row['address']} with status: {row['status']}") + + except Exception as exc: + logger.error(f"Error updating address {row['address']}: {str(exc)}",exc_info=True) + raise + +def _create_new_address( + netbox_instance: pynetbox.api, + address: str, + row: Dict[str, str], + tags_list: List[Dict[str, str]], + tenant_data: Optional[Dict[str, str]], + vrf_data: Optional[Dict[str, str]] +) -> None: + """ + Create a new IP address in Netbox. + + Args: + netbox_instance (pynetbox.api): Netbox API instance + address (str): IP address to create + row (Dict[str, str]): Row data from CSV + tags_list (List[Dict[str, str]]): Processed tags + tenant_data (Optional[Dict[str, str]]): Tenant information + vrf_data (Optional[Dict[str, str]]): VRF information + """ + logger = logging.getLogger(__name__) + logger.info(f"Creating new address: {address}") + + try: + netbox_instance.ipam.ip_addresses.create( + address=address, + status=row['status'], + custom_fields={'scantime': row['scantime']}, + dns_name=row['dns_name'], + tags=tags_list, + tenant=tenant_data, + vrf=vrf_data + ) + logger.debug(f"Successfully created new address: {address}") + + except pynetbox.core.query.RequestError as exc: + if 'Duplicate IP address' in str(exc): + logger.warning(f"Duplicate IP address found: {address}") + else: + logger.error(f"Error creating address {address}: {str(exc)}", exc_info=True) + raise + except Exception as exc: + logger.error(f"Unexpected error creating address {address}: {str(exc)}",exc_info=True) + raise + +def write_data_to_netbox(url: str, token: str, csv_file: str) -> None: + """ + Write data from a CSV file to Netbox. + + Args: + url (str): Base URL of the Netbox instance + token (str): Authentication token for Netbox API + csv_file (str): Path to the CSV file containing data + + Raises: + Exception: If there's an error in the overall process + """ + logger = logging.getLogger(__name__) + + try: + logger.info("Initializing Netbox connection...") + netbox_instance = connect_to_netbox(url, token) + logger.info("Successfully connected to Netbox") + + csv_file_path = os.path.join(SCRIPT_DIR, csv_file) + logger.debug(f"Reading CSV file from: {csv_file_path}") + + with open(csv_file_path, 'r', encoding='utf-8') as file: + reader = csv.DictReader(file) + rows = list(reader) + total_rows = len(rows) + logger.info(f"Found {total_rows} rows to process") + + with tqdm(total=total_rows, desc="Processing Rows") as pbar: + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + futures = [ + executor.submit( + process_row, row, pbar, netbox_instance + ) for row in rows + ] + + for future in as_completed(futures): + try: + future.result() + except Exception: + logger.error("Error in thread pool execution",exc_info=True) + continue + + logger.info("Completed processing all rows") + + except Exception as exc: + logger.error(f"Fatal error in write_data_to_netbox: {str(exc)}", exc_info=True) + raise + +def main() -> None: + """ + Main entry point of the script. + + Reads configuration, sets up logging, and initiates the data import process. + """ + logger = setup_logging() + logger.info("Starting Netbox import process") + + try: + # Read configuration + config = configparser.ConfigParser() + config_path = os.path.join(SCRIPT_DIR, 'var.ini') + config.read(config_path) + + url = config['credentials']['url'] + token = config['credentials']['token'] + + logger.debug(f"Configuration loaded from: {config_path}") + write_data_to_netbox(url, token, 'ipam_addresses.csv') + logger.info("Netbox import process completed successfully") + + except Exception: + logger.error("Script execution failed", exc_info=True) + raise + +if __name__ == "__main__": + main() diff --git a/netbox_push.py b/netbox_push.py deleted file mode 100644 index 4f71284..0000000 --- a/netbox_push.py +++ /dev/null @@ -1,156 +0,0 @@ -import csv -import os -import logging -from concurrent.futures import ThreadPoolExecutor, as_completed -import configparser -import pynetbox -from tqdm import tqdm -from netbox_connection import connect_to_netbox -import urllib3 - -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -# Get the directory of the current script -script_dir = os.path.dirname(os.path.abspath(__file__)) - -# Set up logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler(os.path.join(script_dir, 'netbox_import.log')) - ] -) -logger = logging.getLogger(__name__) - -def process_row(row, pbar): - """ - Process a single row from the CSV file and update/create IP addresses in Netbox. - Args: - - row (dict): A dictionary representing a single row from the CSV file. - - pbar (tqdm.tqdm): Progress bar to update the progress of processing rows. - """ - try: - logger.info(f"Processing address: {row['address']}") - - # Convert 'tags' from a comma-separated string to a list of dictionaries - tags_list = [{'name': tag.strip()} for tag in row['tags'].split(',')] - logger.debug(f"Tags for {row['address']}: {tags_list}") - - # Attempting to get existing address - existing_address = netbox.ipam.ip_addresses.get(address=row['address']) - - if existing_address: - logger.info(f"Updating existing address: {row['address']}") - try: - # Update the existing address - existing_address.status = row['status'] - existing_address.custom_fields = {'scantime': row['scantime']} - existing_address.dns_name = row['dns_name'] - existing_address.tags = tags_list - - if row['tenant'] != 'N/A': - existing_address.tenant = {'name': row['tenant']} - if row['VRF'] != 'N/A': - existing_address.vrf = {'name': row['VRF']} - - existing_address.save() - logger.info(f"Successfully updated address: {row['address']}") - - except Exception as e: - logger.error(f"Error updating address {row['address']}: {str(e)}") - raise - - else: - logger.info(f"Creating new address: {row['address']}") - try: - # Create a new address if it doesn't exist - tenant_data = {'name': row['tenant']} if row['tenant'] != 'N/A' else None - vrf_data = {'name': row['VRF']} if row['VRF'] != 'N/A' else None - - netbox.ipam.ip_addresses.create( - address=row['address'], - status=row['status'], - custom_fields={'scantime': row['scantime']}, - dns_name=row['dns_name'], - tags=tags_list, - tenant=tenant_data, - vrf=vrf_data - ) - logger.info(f"Successfully created address: {row['address']}") - - except pynetbox.core.query.RequestError as e: - if 'Duplicate IP address' in str(e): - logger.warning(f"Duplicate IP address found: {row['address']}") - else: - logger.error(f"Error creating address {row['address']}: {str(e)}") - raise - except Exception as e: - logger.error(f"Unexpected error creating address {row['address']}: {str(e)}") - raise - - except Exception as e: - logger.error(f"Failed to process row for address {row.get('address', 'unknown')}: {str(e)}") - raise - finally: - # Update progress bar for each processed row - pbar.update(1) - -def write_data_to_netbox(url, token, csv_file): - """ - Write data from a CSV file to Netbox. - Args: - - url (str): The base URL of the Netbox instance. - - token (str): The authentication token for accessing the Netbox API. - - csv_file (str): Path to the CSV file containing data to be written to Netbox. - """ - global netbox - try: - logger.info("Connecting to Netbox...") - netbox = connect_to_netbox(url, token) - logger.info("Successfully connected to Netbox") - - csv_file_path = os.path.join(script_dir, csv_file) - logger.info(f"Reading CSV file: {csv_file_path}") - - with open(csv_file_path, 'r') as file: - reader = csv.DictReader(file) - rows = list(reader) - total_rows = len(rows) - logger.info(f"Found {total_rows} rows to process") - - with tqdm(total=total_rows, desc="Processing Rows") as pbar: - with ThreadPoolExecutor(max_workers=5) as executor: - # Submit all tasks and store futures - futures = [executor.submit(process_row, row, pbar) for row in rows] - - # Wait for completion and handle any exceptions - for future in as_completed(futures): - try: - future.result() # This will raise any exceptions from the future - except Exception as e: - logger.error(f"Error processing row: {str(e)}") - # Continue processing other rows even if one fails - continue - - logger.info("Completed processing all rows") - - except Exception as e: - logger.error(f"Fatal error in write_data_to_netbox: {str(e)}") - raise - -if __name__ == "__main__": - try: - # Read URL and token from var.ini - config = configparser.ConfigParser() - config.read(os.path.join(script_dir, 'var.ini')) - url = config['credentials']['url'] - token = config['credentials']['token'] - - logger.info("Starting Netbox import process") - write_data_to_netbox(url, token, 'ipam_addresses.csv') - logger.info("Netbox import process completed successfully") - - except Exception as e: - logger.error(f"Script failed: {str(e)}") - raise From a8ac689524eaf4450fcc7215059aa41d25b55765 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:39:46 +0100 Subject: [PATCH 05/29] Update and rename nmap_compare.py to scan_processor.py --- nmap_compare.py | 99 ------------------ scan_processor.py | 258 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+), 99 deletions(-) delete mode 100644 nmap_compare.py create mode 100644 scan_processor.py diff --git a/nmap_compare.py b/nmap_compare.py deleted file mode 100644 index 6602ef4..0000000 --- a/nmap_compare.py +++ /dev/null @@ -1,99 +0,0 @@ -import csv -from datetime import datetime -import os - -# Get the directory of the current script -script_dir = os.path.dirname(os.path.abspath(__file__)) - -def get_file_path(directory, date_time): - """ - Generate a file path based on the directory and date. - - Args: - - directory (str): The directory where the file will be located. - - date (datetime.datetime): The date to be included in the file name. - - Returns: - - file_path (str): The full file path based on the directory and date. - """ - return os.path.join(script_dir, directory, f'nmap_results_{date_time.strftime("%Y-%m-%d_%H-%M-%S")}.csv') - -def get_latest_files(directory, num_files=2): - """ - Get the list of CSV files in a directory and sort them by modification time. - - Args: - - directory (str): The directory to search for CSV files. - - num_files (int): The number of latest files to retrieve. - - Returns: - - files (list): The list of latest CSV file names. - """ - full_directory = os.path.join(script_dir, directory) - files = [f for f in os.listdir(full_directory) if f.endswith('.csv')] - files.sort(key=lambda x: os.path.getmtime(os.path.join(full_directory, x)), reverse=True) - return files[:num_files] - -# Directory for result files -directory = 'results' - -# Get the two latest file paths -latest_files = get_latest_files(directory) -file_paths = [get_file_path(directory, datetime.strptime(file_name[13:32], "%Y-%m-%d_%H-%M-%S")) for file_name in latest_files] - -# Output file path -output_file_path = os.path.join(script_dir, 'ipam_addresses.csv') - -def read_csv(file_path): - """ - Read a CSV file and return a dictionary with addresses as keys. - - Args: - - file_path (str): The path to the CSV file. - - Returns: - - data (dict): A dictionary with addresses as keys and corresponding row data as values. - """ - data = {} - with open(file_path, 'r') as file: - reader = csv.DictReader(file) - for row in reader: - address = row['address'] - data[address] = row - return data - -def write_csv(data, file_path): - """ - Write data to a new CSV file. - - Args: - - data (dict): A dictionary containing row data with addresses as keys. - - file_path (str): The path to the output CSV file. - """ - with open(file_path, 'w', newline='') as file: - fieldnames = ['address', 'dns_name', 'status', 'scantime', 'tags', 'tenant', 'VRF'] - writer = csv.DictWriter(file, fieldnames=fieldnames) - - # Write header - writer.writeheader() - - # Write data - for row in data.values(): - writer.writerow(row) - -# Read data from the latest file -data = read_csv(file_paths[0]) - -# Check for deprecated addresses in the older file and update their status -if len(file_paths) == 2: - older_data = read_csv(file_paths[1]) - for address, older_row in older_data.items(): - if address not in data: - # Address is missing in latest file, mark as deprecated - older_row['status'] = 'deprecated' - data[address] = older_row - -# Write the updated data to the new CSV file -write_csv(data, output_file_path) - -print("Comparison and processing completed. Check the output file:", output_file_path) diff --git a/scan_processor.py b/scan_processor.py new file mode 100644 index 0000000..bd9e235 --- /dev/null +++ b/scan_processor.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +""" +Network Scan Results Processor. + +This script processes network scan results stored in CSV files, comparing the latest +scan with previous results to track deprecated addresses. It includes comprehensive +logging and error handling for reliable operation. + +The script: +1. Finds the latest CSV files in a specified directory +2. Reads and processes the scan results +3. Marks addresses as deprecated if they're missing from the latest scan +4. Outputs the combined results to a new CSV file +""" + +import csv +import os +import logging +from datetime import datetime +from typing import Dict, List +import sys + +# Script configuration +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +LOG_DIR = os.path.join(SCRIPT_DIR, 'logs') +RESULTS_DIR = os.path.join(SCRIPT_DIR, 'results') +CSV_FIELDNAMES = ['address', 'dns_name', 'status', 'scantime', 'tags', 'tenant', 'VRF'] + +# Ensure required directories exist +for directory in (LOG_DIR, RESULTS_DIR): + os.makedirs(directory, exist_ok=True) + +def setup_logging() -> logging.Logger: + """ + Configure logging with both file and console handlers. + + Returns: + logging.Logger: Configured logger instance + """ + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + + # Create formatters + file_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s' + ) + console_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(message)s' + ) + + # Create file handlers + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + debug_handler = logging.FileHandler( + os.path.join(LOG_DIR, f'scan_processor_debug_{timestamp}.log') + ) + debug_handler.setLevel(logging.DEBUG) + debug_handler.setFormatter(file_formatter) + + error_handler = logging.FileHandler( + os.path.join(LOG_DIR, f'scan_processor_error_{timestamp}.log') + ) + error_handler.setLevel(logging.ERROR) + error_handler.setFormatter(file_formatter) + + # Create console handler + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(console_formatter) + + # Add handlers to logger + logger.addHandler(debug_handler) + logger.addHandler(error_handler) + logger.addHandler(console_handler) + + return logger + +def get_file_path(dir_path: str, date_time: datetime) -> str: + """ + Generate a file path based on the directory and date. + + Args: + dir_path: The directory where the file will be located + date_time: The date to be included in the file name + + Returns: + The full file path + """ + return os.path.join( + SCRIPT_DIR, + dir_path, + f'nmap_results_{date_time.strftime("%Y-%m-%d_%H-%M-%S")}.csv' + ) + +def get_latest_files(dir_path: str, num_files: int = 2) -> List[str]: + """ + Get the list of CSV files in a directory and sort them by modification time. + + Args: + dir_path: The directory to search for CSV files + num_files: The number of latest files to retrieve + + Returns: + List of latest CSV file names + + Raises: + FileNotFoundError: If the directory doesn't exist + ValueError: If no CSV files are found in the directory + """ + logger = logging.getLogger(__name__) + full_directory = os.path.join(SCRIPT_DIR, dir_path) + + logger.debug(f"Searching for CSV files in: {full_directory}") + + try: + files = [f for f in os.listdir(full_directory) if f.endswith('.csv')] + + if not files: + logger.error(f"No CSV files found in {full_directory}") + raise ValueError(f"No CSV files found in {full_directory}") + + files.sort( + key=lambda x: os.path.getmtime(os.path.join(full_directory, x)), + reverse=True + ) + + selected_files = files[:num_files] + logger.info(f"Found {len(selected_files)} latest CSV files") + logger.debug(f"Selected files: {selected_files}") + + return selected_files + + except FileNotFoundError: + logger.error(f"Directory not found: {full_directory}") + raise + +def read_csv(file_path: str) -> Dict[str, Dict[str, str]]: + """ + Read a CSV file and return a dictionary with addresses as keys. + + Args: + file_path: The path to the CSV file + + Returns: + Dictionary with addresses as keys and corresponding row data as values + + Raises: + FileNotFoundError: If the file doesn't exist + csv.Error: If there's an error reading the CSV file + """ + logger = logging.getLogger(__name__) + logger.info(f"Reading CSV file: {file_path}") + + data = {} + try: + with open(file_path, 'r', encoding='utf-8') as file: + reader = csv.DictReader(file) + for row in reader: + address = row['address'] + data[address] = row + logger.debug(f"Processed row for address: {address}") + + logger.info(f"Successfully read {len(data)} records from {file_path}") + return data + + except FileNotFoundError: + logger.error(f"File not found: {file_path}") + raise + except csv.Error as exc: + logger.error(f"Error reading CSV file {file_path}: {str(exc)}") + raise + +def write_csv(data: Dict[str, Dict[str, str]], file_path: str) -> None: + """ + Write data to a new CSV file. + + Args: + data: Dictionary containing row data with addresses as keys + file_path: The path to the output CSV file + + Raises: + PermissionError: If writing to the file is not permitted + csv.Error: If there's an error writing the CSV file + """ + logger = logging.getLogger(__name__) + logger.info(f"Writing data to CSV file: {file_path}") + + try: + with open(file_path, 'w', newline='', encoding='utf-8') as file: + writer = csv.DictWriter(file, fieldnames=CSV_FIELDNAMES) + writer.writeheader() + + for row in data.values(): + writer.writerow(row) + logger.debug(f"Wrote row for address: {row['address']}") + + logger.info(f"Successfully wrote {len(data)} records to {file_path}") + + except PermissionError: + logger.error(f"Permission denied writing to file: {file_path}") + raise + except csv.Error as exc: + logger.error(f"Error writing CSV file {file_path}: {str(exc)}") + raise + +def process_scan_results() -> None: + """ + Main function to process network scan results. + + Coordinates the reading of input files, processing of data, and writing of results. + """ + logger = logging.getLogger(__name__) + logger.info("Starting scan results processing") + + try: + # Get the latest file paths + latest_files = get_latest_files(RESULTS_DIR) + file_paths = [ + get_file_path( + RESULTS_DIR, + datetime.strptime(file_name[13:32], "%Y-%m-%d_%H-%M-%S") + ) for file_name in latest_files + ] + + # Read data from the latest file + data = read_csv(file_paths[0]) + + # Process older file if available + if len(file_paths) == 2: + logger.info("Processing older file to identify deprecated addresses") + older_data = read_csv(file_paths[1]) + + # Check for deprecated addresses + deprecated_count = 0 + for address, older_row in older_data.items(): + if address not in data: + older_row['status'] = 'deprecated' + data[address] = older_row + deprecated_count += 1 + logger.debug(f"Marked address as deprecated: {address}") + + logger.info(f"Found {deprecated_count} deprecated addresses") + + # Write the updated data + output_file_path = os.path.join(SCRIPT_DIR, 'ipam_addresses.csv') + write_csv(data, output_file_path) + + logger.info(f"Processing completed. Output file: {output_file_path}") + + except Exception: + logger.error("Fatal error during processing", exc_info=True) + sys.exit(1) + +def main() -> None: + """Main entry point of the script.""" + process_scan_results() + +if __name__ == "__main__": + main() From 2363dd5fb464e5de880f16cd86fcaa4d4d655b1c Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:40:05 +0100 Subject: [PATCH 06/29] Update and rename nmap_scan_multi_dns.py to network_scan.py --- network_scan.py | 320 +++++++++++++++++++++++++++++++++++++++++ nmap_scan_multi_dns.py | 174 ---------------------- 2 files changed, 320 insertions(+), 174 deletions(-) create mode 100644 network_scan.py delete mode 100644 nmap_scan_multi_dns.py diff --git a/network_scan.py b/network_scan.py new file mode 100644 index 0000000..f6406df --- /dev/null +++ b/network_scan.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +""" +Network Scanner Script. + +This script performs nmap scans on network prefixes retrieved from a CSV file. +It includes comprehensive logging, error handling, and concurrent execution +capabilities for efficient scanning operations. + +The script: +1. Reads network prefixes from a CSV file +2. Performs concurrent nmap scans on active prefixes +3. Writes results to timestamped CSV files +4. Updates the original prefix list to remove scanned networks + +Requirements: + - nmap command-line tool + - csv + - concurrent.futures +""" + +import csv +import subprocess +import os +import sys +from datetime import datetime +from concurrent.futures import ThreadPoolExecutor, as_completed +import logging +import threading +from typing import Dict, List, Tuple, Optional +from dataclasses import dataclass +import queue + +# Script configuration +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +LOG_DIR = os.path.join(SCRIPT_DIR, 'logs') +RESULTS_DIR = os.path.join(SCRIPT_DIR, 'results') +MAX_WORKERS = 5 +NMAP_TIMEOUT = 300 # seconds +FILE_LOCK = threading.Lock() + +# CSV field definitions +INPUT_FIELDNAMES = ['Prefix', 'VRF', 'Status', 'Tags', 'Tenant'] +OUTPUT_FIELDNAMES = ['address', 'dns_name', 'status', 'tags', 'tenant', 'VRF', 'scantime'] + +# Ensure required directories exist +for directory in (LOG_DIR, RESULTS_DIR): + os.makedirs(directory, exist_ok=True) + +@dataclass +class ScanResult: + """Data class for storing scan results.""" + address: str + dns_name: Optional[str] + status: str + tags: str + tenant: str + vrf: str + scantime: str + +def setup_logging() -> logging.Logger: + """Configure logging with both file and console handlers.""" + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + + # Create formatters + file_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s' + ) + console_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(message)s' + ) + + # Create file handlers + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + debug_handler = logging.FileHandler( + os.path.join(LOG_DIR, f'network_scan_debug_{timestamp}.log') + ) + debug_handler.setLevel(logging.DEBUG) + debug_handler.setFormatter(file_formatter) + + error_handler = logging.FileHandler( + os.path.join(LOG_DIR, f'network_scan_error_{timestamp}.log') + ) + error_handler.setLevel(logging.ERROR) + error_handler.setFormatter(file_formatter) + + # Create console handler + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(console_formatter) + + # Add handlers to logger + logger.addHandler(debug_handler) + logger.addHandler(error_handler) + logger.addHandler(console_handler) + + return logger + +def read_from_csv(filename: str) -> List[Dict[str, str]]: + """ + Read data from a CSV file. + + Args: + filename: Path to the CSV file + + Returns: + List of dictionaries representing rows from the CSV file + + Raises: + FileNotFoundError: If the file doesn't exist + csv.Error: If there's an error reading the CSV file + """ + logger = logging.getLogger(__name__) + filepath = os.path.join(SCRIPT_DIR, filename) + + try: + logger.info(f"Reading CSV file: {filepath}") + with open(filepath, 'r', encoding='utf-8') as file: + reader = csv.DictReader(file) + data = [row for row in reader] + + logger.info(f"Successfully read {len(data)} rows from {filename}") + return data + + except FileNotFoundError: + logger.error(f"File not found: {filepath}") + raise + except csv.Error as exc: + logger.error(f"Error reading CSV file {filepath}: {str(exc)}") + raise + +def run_nmap_on_prefix( + prefix: str, + tenant: str, + vrf: str, + result_queue: queue.Queue +) -> Tuple[List[ScanResult], bool]: + """ + Run nmap scan on a given prefix. + + Args: + prefix: Network prefix to scan + tenant: Tenant associated with the prefix + vrf: VRF associated with the prefix + result_queue: Queue for storing scan results + + Returns: + Tuple containing list of scan results and success status + """ + logger = logging.getLogger(__name__) + logger.info(f"Starting scan on prefix: {prefix}") + + try: + command = [ + "nmap", + "-sn", # Ping scan + "-T4", # Aggressive timing + "--min-parallelism", "10", + "--max-retries", "2", + "-R", # DNS resolution + prefix + ] + + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + output, error = process.communicate(timeout=NMAP_TIMEOUT) + + if process.returncode != 0: + logger.error(f"Nmap error for {prefix}: {error}") + return [], False + + results = [] + for line in output.split('\n'): + if "Nmap scan report for" in line: + result = _parse_nmap_output(line, prefix, tenant, vrf) + if result: + results.append(result) + result_queue.put(result) + + logger.info(f"Completed scan on prefix: {prefix} - Found {len(results)} hosts") + return results, True + + except subprocess.TimeoutExpired: + logger.error(f"Scan timeout for prefix: {prefix}") + process.kill() + return [], False + except Exception as exc: + logger.error(f"Error scanning prefix {prefix}: {str(exc)}", exc_info=True) + return [], False + +def _parse_nmap_output( + line: str, + prefix: str, + tenant: str, + vrf: str +) -> Optional[ScanResult]: + """Parse a single line of nmap output.""" + logger = logging.getLogger(__name__) + try: + parts = line.split() + dns_name = None + + if len(parts) > 5: + dns_name = parts[4] + address = parts[5].strip('()') + else: + address = parts[-1].strip('()') + + subnet_mask = prefix.split('/')[-1] + address_with_mask = f"{address}/{subnet_mask}" + + return ScanResult( + address=address_with_mask, + dns_name=dns_name, + status='active', + tags='autoscan', + tenant=tenant, + vrf=vrf, + scantime=datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ) + + except Exception: + logger.error(f"Error parsing nmap output line: {line}", exc_info=True) + return None + +def write_results_to_csv( + results: List[ScanResult], + output_folder: str, + start_time: datetime +) -> None: + """Write scan results to CSV file.""" + logger = logging.getLogger(__name__) + start_time_str = start_time.strftime('%Y-%m-%d_%H-%M-%S') + output_file = os.path.join(output_folder, f'nmap_results_{start_time_str}.csv') + + try: + with FILE_LOCK: + is_new_file = not os.path.exists(output_file) + + with open(output_file, 'a', newline='', encoding='utf-8') as file: + writer = csv.DictWriter(file, fieldnames=OUTPUT_FIELDNAMES) + + if is_new_file: + writer.writeheader() + + for result in results: + writer.writerow(vars(result)) + + logger.debug(f"Wrote {len(results)} results to {output_file}") + + except Exception as exc: + logger.error(f"Error writing results to CSV: {str(exc)}", exc_info=True) + raise + +def process_network_prefixes() -> None: + """Main function to coordinate network scanning operations.""" + logger = logging.getLogger(__name__) + logger.info("Starting network scanning process") + + try: + # Read input data + data = read_from_csv('ipam_prefixes.csv') + + # Filter active prefixes + rows_to_scan = [ + row for row in data + if row['Status'] == 'active' and 'Disable Automatic Scanning' not in row['Tags'] + ] + + logger.info(f"Found {len(rows_to_scan)} prefixes to scan") + + # Set up result queue and start time + result_queue = queue.Queue() + script_start_time = datetime.now() + + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + futures = { + executor.submit( + run_nmap_on_prefix, + row['Prefix'], + row['Tenant'], + row['VRF'], + result_queue + ): row for row in rows_to_scan + } + + # Wait for all futures to complete + for future in as_completed(futures): + try: + success = future.result() + if success: + logger.info(f"Successfully scanned: {futures[future]['Prefix']}") + except Exception: + logger.error("Error processing scan result", exc_info=True) + + # Write final results from the result queue + while not result_queue.empty(): + results_batch = [] + while len(results_batch) < 100 and not result_queue.empty(): + results_batch.append(result_queue.get()) + if results_batch: + write_results_to_csv(results_batch, RESULTS_DIR, script_start_time) + + logger.info("Network scanning process completed") + + except Exception: + logger.error("Fatal error during scanning process", exc_info=True) + sys.exit(1) + +def main() -> None: + """Main entry point of the script.""" + process_network_prefixes() + +if __name__ == "__main__": + main() diff --git a/nmap_scan_multi_dns.py b/nmap_scan_multi_dns.py deleted file mode 100644 index 535cd7e..0000000 --- a/nmap_scan_multi_dns.py +++ /dev/null @@ -1,174 +0,0 @@ -import csv -import subprocess -import os -from datetime import datetime -from concurrent.futures import ThreadPoolExecutor -import concurrent.futures -import logging -import threading - -# Setup logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - -# Lock for writing to CSV file -csv_lock = threading.Lock() - -# Get the directory of the current script -script_dir = os.path.dirname(os.path.abspath(__file__)) - -def read_from_csv(filename): - """ - Read data from a CSV file. - - Args: - - filename (str): The path to the CSV file. - - Returns: - - data (list): A list of dictionaries representing rows from the CSV file. - """ - filepath = os.path.join(script_dir, filename) - with open(filepath, 'r') as file: - reader = csv.DictReader(file) - data = [row for row in reader] - return data - -def remove_scanned_prefixes(data, scanned_prefixes): - """ - Remove scanned prefixes from the original data and rewrite it to the CSV file. - - Args: - - data (list): The original data read from the CSV file. - - scanned_prefixes (list): A list of scanned prefixes to be removed from the data. - """ - # Remove the scanned prefixes from the original data - updated_data = [row for row in data if row['Prefix'] not in scanned_prefixes] - - # Rewrite the updated data to the CSV file - filepath = os.path.join(script_dir, 'ipam_prefixes.csv') - with open(filepath, 'w', newline='') as file: - fieldnames = ['Prefix', 'VRF', 'Status', 'Tags', 'Tenant'] - writer = csv.DictWriter(file, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(updated_data) - -def run_nmap_on_prefix(prefix, tenant, vrf): - """ - Run nmap scan on a given prefix. - - Args: - - prefix (str): The prefix to be scanned. - - tenant (str): The tenant associated with the prefix. - - vrf (str): The VRF associated with the prefix. - - Returns: - - results (list): A list of dictionaries containing scan results. - - success (bool): True if the scan was successful, False otherwise. - """ - logger.info(f"Starting scan on prefix: {prefix}") - # Run nmap on the prefix with DNS resolution and specified DNS servers - command = f"nmap -sn -T4 --min-parallelism 10 --max-retries 2 -R {prefix}" - process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = process.communicate() - - if error: - logger.error(f"Error: {error}") - return [], False - - results = [] - # Parse the standard output - lines = output.decode().split('\n') - for line in lines: - if "Nmap scan report for" in line: - parts = line.split() - dns_name = None - if len(parts) > 5: # Check if there are more than 5 parts in the line - dns_name = parts[4] # Extract DNS name - address = parts[5] # Extract IP address - else: - address = parts[-1] # Extract IP address - # Remove parenthesis from IP address if present - address = address.strip('()') - # Include the subnet mask from the prefix in the address - address_with_mask = f"{address}/{prefix.split('/')[-1]}" - results.append({ - 'address': address_with_mask, - 'dns_name': dns_name, # Add DNS name to the results - 'status': 'active', - 'tags': 'autoscan', - 'tenant': tenant, - 'VRF': vrf, # Add VRF to the results - 'scantime': datetime.now().strftime('%Y-%m-%d %H:%M:%S') # Add current date and time as scantime - }) - logger.info(f"Finished scan on prefix: {prefix}") - return results, True - -def run_nmap_on_prefixes(data, output_folder): - """ - Run nmap scans on prefixes in parallel and write results to CSV files. - - Args: - - data (list): The list of dictionaries containing prefix data. - - output_folder (str): The directory where output CSV files will be stored. - """ - results = [] - scanned_prefixes = [] - - # Create the full path for the output folder - output_folder_path = os.path.join(script_dir, output_folder) - - # Filter rows to scan only those with status 'active' and without the tag 'Disable Automatic Scanning' - rows_to_scan = [row for row in data if row['Status'] == 'active' and 'Disable Automatic Scanning' not in row['Tags']] - - script_start_time = datetime.now() # Get the script start time - - with ThreadPoolExecutor(max_workers=5) as executor: - # Use executor.map to asynchronously run the scans and get results - futures = {executor.submit(run_nmap_on_prefix, row['Prefix'], row['Tenant'], row['VRF']): row for row in rows_to_scan} - - for future in concurrent.futures.as_completed(futures): - prefix_results, success = future.result() - if success: - with csv_lock: - results.extend(prefix_results) - scanned_prefixes.append(futures[future]['Prefix']) - write_results_to_csv(prefix_results, output_folder_path, script_start_time) - - remove_scanned_prefixes(data, scanned_prefixes) - return results - -def write_results_to_csv(results, output_folder, script_start_time): - """ - Write scan results to CSV files. - - Args: - - results (list): A list of dictionaries containing scan results. - - output_folder (str): The directory where output CSV files will be stored. - """ - # Create the results folder if it doesn't exist - os.makedirs(output_folder, exist_ok=True) - - # Generate the current date and time as a string - start_time_str = script_start_time.strftime('%Y-%m-%d_%H-%M-%S') - - # Set the filename with the full path including the date and time - output_filename = os.path.join(output_folder, f'nmap_results_{start_time_str}.csv') - - # Check if the file is empty - is_empty = not os.path.exists(output_filename) or os.stat(output_filename).st_size == 0 - - with open(output_filename, 'a', newline='') as file: - fieldnames = ['address', 'dns_name', 'status', 'tags', 'tenant', 'VRF', 'scantime'] - writer = csv.DictWriter(file, fieldnames=fieldnames) - - # Add headers if the file is empty - if is_empty: - writer.writeheader() - - for result in results: - writer.writerow(result) - -if __name__ == "__main__": - data = read_from_csv('ipam_prefixes.csv') - output_folder = 'results' - run_nmap_on_prefixes(data, output_folder) From 06f3ea803962370cdd027df280d9fcd6de8c8528 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:58:40 +0100 Subject: [PATCH 07/29] Create requirements.txt --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..323e2c1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +pynetbox==7.4.1 +requests==2.32.3 +urllib3==2.3.0 +tqdm==4.67.1 From 0120909ef5cc12e94ee2cb8005e51f4f86a6b88d Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 12:00:40 +0100 Subject: [PATCH 08/29] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 43d74c6..9a20c48 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ For detailed usage instructions and examples, please refer to our [Wiki](https:/ - [ ] DNS server configuration in INI file for custom DNS resolution - [ ] Option to disable DNS resolution functionality - [ ] Toggle for last scan time tracking +- [ ] Toggle for the progress bar display while importing - [ ] All-in-One setup script for easier deployment ## Contributing From e8a0137263ac466d0580c9137f009aeafc8c6579 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:21:17 +0100 Subject: [PATCH 09/29] Update netbox_export.py From 358407e38b56d31b6ca75a7adce7b14f6464dd92 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:21:27 +0100 Subject: [PATCH 10/29] Update netbox_import.py From c3663068e2942c85dd4aa8a07a6636f05b734bc3 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:21:39 +0100 Subject: [PATCH 11/29] Update network_scan.py --- network_scan.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/network_scan.py b/network_scan.py index f6406df..b3ba375 100644 --- a/network_scan.py +++ b/network_scan.py @@ -54,7 +54,7 @@ class ScanResult: status: str tags: str tenant: str - vrf: str + VRF: str scantime: str def setup_logging() -> logging.Logger: @@ -132,7 +132,7 @@ def read_from_csv(filename: str) -> List[Dict[str, str]]: def run_nmap_on_prefix( prefix: str, tenant: str, - vrf: str, + VRF: str, result_queue: queue.Queue ) -> Tuple[List[ScanResult], bool]: """ @@ -141,7 +141,7 @@ def run_nmap_on_prefix( Args: prefix: Network prefix to scan tenant: Tenant associated with the prefix - vrf: VRF associated with the prefix + VRF: VRF associated with the prefix result_queue: Queue for storing scan results Returns: @@ -177,7 +177,7 @@ def run_nmap_on_prefix( results = [] for line in output.split('\n'): if "Nmap scan report for" in line: - result = _parse_nmap_output(line, prefix, tenant, vrf) + result = _parse_nmap_output(line, prefix, tenant, VRF) if result: results.append(result) result_queue.put(result) @@ -197,7 +197,7 @@ def _parse_nmap_output( line: str, prefix: str, tenant: str, - vrf: str + VRF: str ) -> Optional[ScanResult]: """Parse a single line of nmap output.""" logger = logging.getLogger(__name__) @@ -220,7 +220,7 @@ def _parse_nmap_output( status='active', tags='autoscan', tenant=tenant, - vrf=vrf, + VRF=VRF, scantime=datetime.now().strftime('%Y-%m-%d %H:%M:%S') ) @@ -314,6 +314,7 @@ def process_network_prefixes() -> None: def main() -> None: """Main entry point of the script.""" + logger = setup_logging() process_network_prefixes() if __name__ == "__main__": From b529f771e47a33cad1c607cc274320c23b54001a Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:21:50 +0100 Subject: [PATCH 12/29] Update scan_processor.py --- scan_processor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scan_processor.py b/scan_processor.py index bd9e235..cbf92e3 100644 --- a/scan_processor.py +++ b/scan_processor.py @@ -252,6 +252,7 @@ def process_scan_results() -> None: def main() -> None: """Main entry point of the script.""" + logger = setup_logging() process_scan_results() if __name__ == "__main__": From a76b08f8a157a1b50104a527961e835d9edca2af Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:43:26 +0100 Subject: [PATCH 13/29] Update network_scan.py --- network_scan.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/network_scan.py b/network_scan.py index b3ba375..0f4ca52 100644 --- a/network_scan.py +++ b/network_scan.py @@ -24,6 +24,7 @@ import sys from datetime import datetime from concurrent.futures import ThreadPoolExecutor, as_completed +import configparser import logging import threading from typing import Dict, List, Tuple, Optional @@ -150,17 +151,24 @@ def run_nmap_on_prefix( logger = logging.getLogger(__name__) logger.info(f"Starting scan on prefix: {prefix}") + config = configparser.ConfigParser() + config.read('var.ini') + enable_dns = config.getboolean('scan_options', 'enable_dns', fallback=True) + try: command = [ "nmap", "-sn", # Ping scan "-T4", # Aggressive timing "--min-parallelism", "10", - "--max-retries", "2", - "-R", # DNS resolution - prefix + "--max-retries", "2" ] + if enable_dns: + command.append("-R") + + command.append(prefix) + process = subprocess.Popen( command, stdout=subprocess.PIPE, @@ -201,6 +209,13 @@ def _parse_nmap_output( ) -> Optional[ScanResult]: """Parse a single line of nmap output.""" logger = logging.getLogger(__name__) + + config = configparser.ConfigParser() + config.read('var.ini') + enable_scantime = config.getboolean('scan_options', 'enable_scantime', fallback=True) + + scantime = datetime.now().strftime('%Y-%m-%d %H:%M:%S') if enable_scantime else None + try: parts = line.split() dns_name = None @@ -221,7 +236,7 @@ def _parse_nmap_output( tags='autoscan', tenant=tenant, VRF=VRF, - scantime=datetime.now().strftime('%Y-%m-%d %H:%M:%S') + scantime=scantime ) except Exception: From 3f632a993e8602a60b99c1cd2fafd0d710c8ef07 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:43:40 +0100 Subject: [PATCH 14/29] Update netbox_import.py --- netbox_import.py | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/netbox_import.py b/netbox_import.py index 3de35a9..cdcb940 100644 --- a/netbox_import.py +++ b/netbox_import.py @@ -134,7 +134,8 @@ def process_row(row: Dict[str, str], pbar: tqdm, netbox_instance: pynetbox.api) logger.error(f"Failed to process row for address {address}", exc_info=True) raise finally: - pbar.update(1) + if pbar: + pbar.update(1) def _update_existing_address( existing_address: object, @@ -229,6 +230,10 @@ def write_data_to_netbox(url: str, token: str, csv_file: str) -> None: """ logger = logging.getLogger(__name__) + config = configparser.ConfigParser() + config.read('var.ini') + show_progress = config.getboolean('scan_options', 'show_progress', fallback=True) + try: logger.info("Initializing Netbox connection...") netbox_instance = connect_to_netbox(url, token) @@ -241,22 +246,28 @@ def write_data_to_netbox(url: str, token: str, csv_file: str) -> None: reader = csv.DictReader(file) rows = list(reader) total_rows = len(rows) - logger.info(f"Found {total_rows} rows to process") - - with tqdm(total=total_rows, desc="Processing Rows") as pbar: - with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: - futures = [ - executor.submit( - process_row, row, pbar, netbox_instance - ) for row in rows - ] - - for future in as_completed(futures): - try: - future.result() - except Exception: - logger.error("Error in thread pool execution",exc_info=True) - continue + + if show_progress: + pbar = tqdm(total=total_rows, desc="Processing Rows") + else: + pbar = None + + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + futures = [ + executor.submit( + process_row, row, pbar if show_progress else None, netbox_instance + ) for row in rows + ] + + for future in as_completed(futures): + try: + future.result() + except Exception: + logger.error("Error in thread pool execution", exc_info=True) + continue + + if pbar: + pbar.close() logger.info("Completed processing all rows") From 746fa06489daa1fbc71865d92cf6f130d5e2c073 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:43:53 +0100 Subject: [PATCH 15/29] Update var.ini --- var.ini | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/var.ini b/var.ini index 51e021b..8d8eb0d 100644 --- a/var.ini +++ b/var.ini @@ -1,3 +1,8 @@ [credentials] token = netbox_token -url = netbox_url \ No newline at end of file +url = netbox_url + +[scan_options] +enable_dns = true +enable_scantime = true +show_progress = true From 6caae91d9b9c09a70b4a3bcbd33fb2509f43864e Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:46:43 +0100 Subject: [PATCH 16/29] Update netbox_import.py --- netbox_import.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/netbox_import.py b/netbox_import.py index cdcb940..25bde6f 100644 --- a/netbox_import.py +++ b/netbox_import.py @@ -246,7 +246,8 @@ def write_data_to_netbox(url: str, token: str, csv_file: str) -> None: reader = csv.DictReader(file) rows = list(reader) total_rows = len(rows) - + logger.info(f"Found {total_rows} rows to process") + if show_progress: pbar = tqdm(total=total_rows, desc="Processing Rows") else: From 9b03139f9725f549c21733eccd45ce8a5fe28f84 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:51:55 +0100 Subject: [PATCH 17/29] Update network_scan.py --- network_scan.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/network_scan.py b/network_scan.py index 0f4ca52..e45ca65 100644 --- a/network_scan.py +++ b/network_scan.py @@ -166,6 +166,8 @@ def run_nmap_on_prefix( if enable_dns: command.append("-R") + else: + command.append("-n") command.append(prefix) From 238a0b1a47e4deeaf1e80c994d91d15b7ee17010 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:52:09 +0100 Subject: [PATCH 18/29] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a20c48..608f5a6 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ For detailed usage instructions and examples, please refer to our [Wiki](https:/ ## Roadmap - [ ] DNS server configuration in INI file for custom DNS resolution -- [ ] Option to disable DNS resolution functionality +- [X] Option to disable DNS resolution functionality - [ ] Toggle for last scan time tracking - [ ] Toggle for the progress bar display while importing - [ ] All-in-One setup script for easier deployment From 0d7bc56577169087ae37b0c8d06ac30d65775610 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:52:56 +0100 Subject: [PATCH 19/29] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 608f5a6..b254e37 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ For detailed usage instructions and examples, please refer to our [Wiki](https:/ - [ ] DNS server configuration in INI file for custom DNS resolution - [X] Option to disable DNS resolution functionality -- [ ] Toggle for last scan time tracking +- [X] Toggle for last scan time tracking - [ ] Toggle for the progress bar display while importing - [ ] All-in-One setup script for easier deployment From 4af4afcbde3ee0ddaa707f38d58b5b81bd53042a Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:54:52 +0100 Subject: [PATCH 20/29] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b254e37..c2708be 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ For detailed usage instructions and examples, please refer to our [Wiki](https:/ - [ ] DNS server configuration in INI file for custom DNS resolution - [X] Option to disable DNS resolution functionality - [X] Toggle for last scan time tracking -- [ ] Toggle for the progress bar display while importing +- [X] Toggle for the progress bar display while importing - [ ] All-in-One setup script for easier deployment ## Contributing From 407fc3fefe9126b55609617528fd9c1c00eaa82e Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:55:17 +0100 Subject: [PATCH 21/29] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c2708be..d6a2f9a 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ For detailed usage instructions and examples, please refer to our [Wiki](https:/ - [X] Option to disable DNS resolution functionality - [X] Toggle for last scan time tracking - [X] Toggle for the progress bar display while importing +- [ ] Toggle for the logger in the Python console - [ ] All-in-One setup script for easier deployment ## Contributing From ad60027b07898f6c1f90a3918229d1324211e30e Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Mon, 30 Dec 2024 11:22:50 +0100 Subject: [PATCH 22/29] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d6a2f9a..d4eba25 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Automatically maintain an up-to-date inventory of active IP addresses in your ne - Custom tag support for excluding prefixes from scanning - Tracking of last scan time for each IP address - DNS resolution support -- Compatible with Python 3.12.6 and Netbox 4.1.10 +- Tested with Python 3.12.6 and Netbox 4.1.10 ## Prerequisites From 9e8e2c30685462c7b68edd70748f414c429abf4d Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Mon, 30 Dec 2024 11:29:02 +0100 Subject: [PATCH 23/29] Update network_scan.py Added prefix removal at the end of a scan --- network_scan.py | 108 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 23 deletions(-) diff --git a/network_scan.py b/network_scan.py index e45ca65..c9be11a 100644 --- a/network_scan.py +++ b/network_scan.py @@ -10,12 +10,7 @@ 1. Reads network prefixes from a CSV file 2. Performs concurrent nmap scans on active prefixes 3. Writes results to timestamped CSV files -4. Updates the original prefix list to remove scanned networks - -Requirements: - - nmap command-line tool - - csv - - concurrent.futures +4. Removes each scanned network from input file immediately after scanning """ import csv @@ -27,9 +22,11 @@ import configparser import logging import threading -from typing import Dict, List, Tuple, Optional +from typing import Dict, List, Tuple, Optional, Set from dataclasses import dataclass import queue +import tempfile +import shutil # Script configuration SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -97,6 +94,58 @@ def setup_logging() -> logging.Logger: return logger +def remove_prefix_from_csv(filename: str, prefix_to_remove: str) -> None: + """ + Remove a single prefix from the CSV file immediately after successful scan. + Uses file locking to ensure thread safety. + + Args: + filename: Path to the CSV file + prefix_to_remove: The prefix to remove from the file + """ + logger = logging.getLogger(__name__) + filepath = os.path.join(SCRIPT_DIR, filename) + + with FILE_LOCK: # Ensure thread safety when modifying the file + try: + # Create a temporary file + temp_file = tempfile.NamedTemporaryFile( + mode='w', + delete=False, + newline='', + encoding='utf-8', + dir=os.path.dirname(filepath) + ) + + removed = False + # Read and write in a single pass + with open(filepath, 'r', encoding='utf-8') as file: + reader = csv.DictReader(file) + writer = csv.DictWriter(temp_file, fieldnames=reader.fieldnames) + writer.writeheader() + + for row in reader: + if row['Prefix'] != prefix_to_remove: + writer.writerow(row) + else: + removed = True + + temp_file.close() + + # Atomic replacement of the original file + shutil.move(temp_file.name, filepath) + + if removed: + logger.info(f"Successfully removed prefix {prefix_to_remove} from {filename}") + else: + logger.warning(f"Prefix {prefix_to_remove} not found in {filename}") + + except Exception as exc: + logger.error(f"Error removing prefix {prefix_to_remove} from CSV: {str(exc)}", exc_info=True) + if 'temp_file' in locals() and os.path.exists(temp_file.name): + os.unlink(temp_file.name) + raise + def read_from_csv(filename: str) -> List[Dict[str, str]]: """ Read data from a CSV file. @@ -116,9 +165,10 @@ def read_from_csv(filename: str) -> List[Dict[str, str]]: try: logger.info(f"Reading CSV file: {filepath}") - with open(filepath, 'r', encoding='utf-8') as file: - reader = csv.DictReader(file) - data = [row for row in reader] + with FILE_LOCK: # Add lock when reading to ensure consistency + with open(filepath, 'r', encoding='utf-8') as file: + reader = csv.DictReader(file) + data = [row for row in reader] logger.info(f"Successfully read {len(data)} rows from {filename}") return data @@ -134,16 +184,18 @@ def run_nmap_on_prefix( prefix: str, tenant: str, VRF: str, - result_queue: queue.Queue + result_queue: queue.Queue, + input_filename: str ) -> Tuple[List[ScanResult], bool]: """ - Run nmap scan on a given prefix. + Run nmap scan on a given prefix and remove it from input file if successful. Args: prefix: Network prefix to scan tenant: Tenant associated with the prefix VRF: VRF associated with the prefix result_queue: Queue for storing scan results + input_filename: Name of the input CSV file to update Returns: Tuple containing list of scan results and success status @@ -192,8 +244,13 @@ def run_nmap_on_prefix( results.append(result) result_queue.put(result) - logger.info(f"Completed scan on prefix: {prefix} - Found {len(results)} hosts") - return results, True + # If scan was successful, remove the prefix immediately + if process.returncode == 0: + remove_prefix_from_csv(input_filename, prefix) + logger.info(f"Completed scan on prefix: {prefix} - Found {len(results)} hosts") + return results, True + + return [], False except subprocess.TimeoutExpired: logger.error(f"Scan timeout for prefix: {prefix}") @@ -281,7 +338,8 @@ def process_network_prefixes() -> None: try: # Read input data - data = read_from_csv('ipam_prefixes.csv') + input_filename = 'ipam_prefixes.csv' + data = read_from_csv(input_filename) # Filter active prefixes rows_to_scan = [ @@ -302,26 +360,30 @@ def process_network_prefixes() -> None: row['Prefix'], row['Tenant'], row['VRF'], - result_queue + result_queue, + input_filename ): row for row in rows_to_scan } # Wait for all futures to complete for future in as_completed(futures): try: - success = future.result() + results, success = future.result() if success: - logger.info(f"Successfully scanned: {futures[future]['Prefix']}") + logger.info(f"Successfully processed: {futures[future]['Prefix']}") except Exception: logger.error("Error processing scan result", exc_info=True) # Write final results from the result queue + results_to_write = [] while not result_queue.empty(): - results_batch = [] - while len(results_batch) < 100 and not result_queue.empty(): - results_batch.append(result_queue.get()) - if results_batch: - write_results_to_csv(results_batch, RESULTS_DIR, script_start_time) + results_to_write.append(result_queue.get()) + if len(results_to_write) >= 100: + write_results_to_csv(results_to_write, RESULTS_DIR, script_start_time) + results_to_write = [] + + if results_to_write: + write_results_to_csv(results_to_write, RESULTS_DIR, script_start_time) logger.info("Network scanning process completed") From 75d38d5953b938a81549702816ec7c4cf95f356e Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Mon, 30 Dec 2024 11:54:19 +0100 Subject: [PATCH 24/29] Create main.py --- main.py | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 main.py diff --git a/main.py b/main.py new file mode 100644 index 0000000..2b2e334 --- /dev/null +++ b/main.py @@ -0,0 +1,116 @@ +import subprocess +import sys +import os +import logging +from datetime import datetime +from typing import List + +# Define the directory for logs and scripts +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +LOG_DIR = os.path.join(SCRIPT_DIR, 'logs') + +# Ensure the log directory exists +os.makedirs(LOG_DIR, exist_ok=True) + +def setup_logging() -> logging.Logger: + """ + Configure logging with both file and console handlers. + + Returns: + logging.Logger: Configured logger instance + """ + # Create logger + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + + # Create formatters + file_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s' + ) + console_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(message)s' + ) + + # Create file handlers + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + debug_handler = logging.FileHandler( + os.path.join(LOG_DIR, f'script_execution_debug_{timestamp}.log') + ) + debug_handler.setLevel(logging.DEBUG) + debug_handler.setFormatter(file_formatter) + + error_handler = logging.FileHandler( + os.path.join(LOG_DIR, f'script_execution_error_{timestamp}.log') + ) + error_handler.setLevel(logging.ERROR) + error_handler.setFormatter(file_formatter) + + # Create console handler + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(console_formatter) + + # Add handlers to logger + logger.addHandler(debug_handler) + logger.addHandler(error_handler) + logger.addHandler(console_handler) + + return logger + +def run_script(script_name: str, logger: logging.Logger) -> bool: + """ + Executes a Python script using the current Python interpreter. + + Args: + script_name (str): The name of the script to run. + logger (logging.Logger): The logger instance for logging messages. + + Returns: + bool: True if the script runs successfully, False otherwise. + """ + logger.info(f"Running {script_name}...") + + try: + # Run the script using the current Python interpreter + result = subprocess.run( + [sys.executable, script_name], + capture_output=True, + text=True, + check=True + ) + logger.info(f"{script_name} completed successfully.") + logger.debug(f"Output:\n{result.stdout}") + return True + except subprocess.CalledProcessError as e: + logger.error(f"Error running {script_name}:") + logger.error(e.stderr) # Log the error message if the script fails + return False + +def main(): + """ + Main function to run a list of Python scripts sequentially. + + The function will stop execution if any script fails, preventing subsequent + scripts from running if an error is encountered. + """ + logger = setup_logging() + + # List of scripts to execute in order + scripts: List[str] = [ + "netbox_export.py", + "network_scan.py", + "scan_processor.py", + "netbox_import.py" + ] + + # Iterate over the list of scripts and run each one + for script in scripts: + if not run_script(script, logger): + logger.error(f"Execution stopped due to an error in {script}") + break # Stop execution if a script fails + else: + logger.info("All scripts executed successfully.") + +if __name__ == "__main__": + # Run the main function if the script is executed directly + main() From 5329fb9c2366dff7e6a0dfab24dd4a0a7166dc1c Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Mon, 30 Dec 2024 11:56:00 +0100 Subject: [PATCH 25/29] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d4eba25..4d00ddb 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ For detailed usage instructions and examples, please refer to our [Wiki](https:/ - [X] Toggle for last scan time tracking - [X] Toggle for the progress bar display while importing - [ ] Toggle for the logger in the Python console -- [ ] All-in-One setup script for easier deployment +- [X] All-in-One setup script for easier deployment ## Contributing From 90efd81e89f4a504e08ea0379738c78a15e7a9ea Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Mon, 30 Dec 2024 13:32:07 +0100 Subject: [PATCH 26/29] Update network_scan.py --- network_scan.py | 85 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 29 deletions(-) diff --git a/network_scan.py b/network_scan.py index c9be11a..b7d198e 100644 --- a/network_scan.py +++ b/network_scan.py @@ -9,7 +9,7 @@ The script: 1. Reads network prefixes from a CSV file 2. Performs concurrent nmap scans on active prefixes -3. Writes results to timestamped CSV files +3. Writes results to CSV file immediately after each scan 4. Removes each scanned network from input file immediately after scanning """ @@ -22,9 +22,8 @@ import configparser import logging import threading -from typing import Dict, List, Tuple, Optional, Set +from typing import Dict, List, Tuple, Optional from dataclasses import dataclass -import queue import tempfile import shutil @@ -105,7 +104,7 @@ def remove_prefix_from_csv(filename: str, prefix_to_remove: str) -> None: """ logger = logging.getLogger(__name__) filepath = os.path.join(SCRIPT_DIR, filename) - + with FILE_LOCK: # Ensure thread safety when modifying the file try: # Create a temporary file @@ -116,36 +115,73 @@ def remove_prefix_from_csv(filename: str, prefix_to_remove: str) -> None: encoding='utf-8', dir=os.path.dirname(filepath) ) - + removed = False # Read and write in a single pass with open(filepath, 'r', encoding='utf-8') as file: reader = csv.DictReader(file) writer = csv.DictWriter(temp_file, fieldnames=reader.fieldnames) writer.writeheader() - + for row in reader: if row['Prefix'] != prefix_to_remove: writer.writerow(row) else: removed = True - + temp_file.close() - + # Atomic replacement of the original file shutil.move(temp_file.name, filepath) - + if removed: logger.info(f"Successfully removed prefix {prefix_to_remove} from {filename}") else: logger.warning(f"Prefix {prefix_to_remove} not found in {filename}") - + except Exception as exc: logger.error(f"Error removing prefix {prefix_to_remove} from CSV: {str(exc)}", exc_info=True) if 'temp_file' in locals() and os.path.exists(temp_file.name): os.unlink(temp_file.name) raise +def write_scan_results( + results: List[ScanResult], + output_folder: str, + start_time: datetime +) -> None: + """ + Write scan results to CSV file immediately. + Uses file locking to ensure thread safety. + + Args: + results: List of scan results to write + output_folder: Folder to write results to + start_time: Script start time for file naming + """ + logger = logging.getLogger(__name__) + start_time_str = start_time.strftime('%Y-%m-%d_%H-%M-%S') + output_file = os.path.join(output_folder, f'nmap_results_{start_time_str}.csv') + + try: + with FILE_LOCK: + is_new_file = not os.path.exists(output_file) + + with open(output_file, 'a', newline='', encoding='utf-8') as file: + writer = csv.DictWriter(file, fieldnames=OUTPUT_FIELDNAMES) + + if is_new_file: + writer.writeheader() + + for result in results: + writer.writerow(vars(result)) + + logger.debug(f"Wrote {len(results)} results to {output_file}") + + except Exception as exc: + logger.error(f"Error writing results to CSV: {str(exc)}", exc_info=True) + raise + def read_from_csv(filename: str) -> List[Dict[str, str]]: """ Read data from a CSV file. @@ -184,17 +220,17 @@ def run_nmap_on_prefix( prefix: str, tenant: str, VRF: str, - result_queue: queue.Queue, + script_start_time: datetime, input_filename: str ) -> Tuple[List[ScanResult], bool]: """ - Run nmap scan on a given prefix and remove it from input file if successful. + Run nmap scan on a given prefix, write results and remove from input file if successful. Args: prefix: Network prefix to scan tenant: Tenant associated with the prefix VRF: VRF associated with the prefix - result_queue: Queue for storing scan results + script_start_time: Start time of the script for result file naming input_filename: Name of the input CSV file to update Returns: @@ -242,10 +278,13 @@ def run_nmap_on_prefix( result = _parse_nmap_output(line, prefix, tenant, VRF) if result: results.append(result) - result_queue.put(result) - # If scan was successful, remove the prefix immediately + # If scan was successful: + # 1. Write results immediately + # 2. Remove prefix from input file if process.returncode == 0: + if results: + write_scan_results(results, RESULTS_DIR, script_start_time) remove_prefix_from_csv(input_filename, prefix) logger.info(f"Completed scan on prefix: {prefix} - Found {len(results)} hosts") return results, True @@ -349,8 +388,7 @@ def process_network_prefixes() -> None: logger.info(f"Found {len(rows_to_scan)} prefixes to scan") - # Set up result queue and start time - result_queue = queue.Queue() + # Record script start time for consistent file naming script_start_time = datetime.now() with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: @@ -360,7 +398,7 @@ def process_network_prefixes() -> None: row['Prefix'], row['Tenant'], row['VRF'], - result_queue, + script_start_time, input_filename ): row for row in rows_to_scan } @@ -374,17 +412,6 @@ def process_network_prefixes() -> None: except Exception: logger.error("Error processing scan result", exc_info=True) - # Write final results from the result queue - results_to_write = [] - while not result_queue.empty(): - results_to_write.append(result_queue.get()) - if len(results_to_write) >= 100: - write_results_to_csv(results_to_write, RESULTS_DIR, script_start_time) - results_to_write = [] - - if results_to_write: - write_results_to_csv(results_to_write, RESULTS_DIR, script_start_time) - logger.info("Network scanning process completed") except Exception: From 0cff3acc0bab5415058db8efb1b8723ab6083763 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Mon, 30 Dec 2024 14:05:05 +0100 Subject: [PATCH 27/29] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d00ddb..d961752 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Automatically maintain an up-to-date inventory of active IP addresses in your ne - Custom tag support for excluding prefixes from scanning - Tracking of last scan time for each IP address - DNS resolution support -- Tested with Python 3.12.6 and Netbox 4.1.10 +- Tested with Python 3.12.6 - 3.13.1 and Netbox 4.1.10 ## Prerequisites From 176ed529f27481a2aa98f4f8bda4c24eb3089b7e Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Mon, 30 Dec 2024 14:07:41 +0100 Subject: [PATCH 28/29] Update main.py --- main.py | 56 +++++++++++++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/main.py b/main.py index 2b2e334..7390b90 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,9 @@ +"""Script runner that executes multiple Python scripts in sequence. + +This module provides functionality to run multiple Python scripts in order, +with comprehensive logging of execution results. +""" + import subprocess import sys import os @@ -12,18 +18,16 @@ # Ensure the log directory exists os.makedirs(LOG_DIR, exist_ok=True) + def setup_logging() -> logging.Logger: - """ - Configure logging with both file and console handlers. + """Configure logging with both file and console handlers. Returns: logging.Logger: Configured logger instance """ - # Create logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) - # Create formatters file_formatter = logging.Formatter( '%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s' ) @@ -31,7 +35,6 @@ def setup_logging() -> logging.Logger: '%(asctime)s - %(levelname)s - %(message)s' ) - # Create file handlers timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') debug_handler = logging.FileHandler( os.path.join(LOG_DIR, f'script_execution_debug_{timestamp}.log') @@ -45,57 +48,53 @@ def setup_logging() -> logging.Logger: error_handler.setLevel(logging.ERROR) error_handler.setFormatter(file_formatter) - # Create console handler console_handler = logging.StreamHandler() console_handler.setLevel(logging.INFO) console_handler.setFormatter(console_formatter) - # Add handlers to logger logger.addHandler(debug_handler) logger.addHandler(error_handler) logger.addHandler(console_handler) return logger + def run_script(script_name: str, logger: logging.Logger) -> bool: - """ - Executes a Python script using the current Python interpreter. + """Execute a Python script using the current Python interpreter. Args: - script_name (str): The name of the script to run. - logger (logging.Logger): The logger instance for logging messages. + script_name: The name of the script to run + logger: The logger instance for logging messages Returns: - bool: True if the script runs successfully, False otherwise. + bool: True if the script runs successfully, False otherwise """ - logger.info(f"Running {script_name}...") + logger.info("Running %s...", script_name) try: - # Run the script using the current Python interpreter result = subprocess.run( [sys.executable, script_name], capture_output=True, text=True, check=True ) - logger.info(f"{script_name} completed successfully.") - logger.debug(f"Output:\n{result.stdout}") + logger.info("%s completed successfully.", script_name) + logger.debug("Output:\n%s", result.stdout) return True - except subprocess.CalledProcessError as e: - logger.error(f"Error running {script_name}:") - logger.error(e.stderr) # Log the error message if the script fails + except subprocess.CalledProcessError as error: + logger.error("Error running %s:", script_name) + logger.error(error.stderr) return False -def main(): - """ - Main function to run a list of Python scripts sequentially. + +def main() -> None: + """Run a list of Python scripts sequentially. The function will stop execution if any script fails, preventing subsequent scripts from running if an error is encountered. """ logger = setup_logging() - # List of scripts to execute in order scripts: List[str] = [ "netbox_export.py", "network_scan.py", @@ -103,14 +102,13 @@ def main(): "netbox_import.py" ] - # Iterate over the list of scripts and run each one for script in scripts: if not run_script(script, logger): - logger.error(f"Execution stopped due to an error in {script}") - break # Stop execution if a script fails - else: - logger.info("All scripts executed successfully.") + logger.error("Execution stopped due to an error in %s", script) + sys.exit(1) + + logger.info("All scripts executed successfully.") + if __name__ == "__main__": - # Run the main function if the script is executed directly main() From 6d8e49eb58e65d0da6896dbfdd095b99bd51cc20 Mon Sep 17 00:00:00 2001 From: LoH-lu <139378145+LoH-lu@users.noreply.github.com> Date: Mon, 30 Dec 2024 14:10:27 +0100 Subject: [PATCH 29/29] Update main.py --- main.py | 55 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/main.py b/main.py index 7390b90..30bed6e 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ -"""Script runner that executes multiple Python scripts in sequence. - +#!/usr/bin/env python3 +""" +Script runner that executes multiple Python scripts in sequence. This module provides functionality to run multiple Python scripts in order, with comprehensive logging of execution results. """ @@ -18,16 +19,18 @@ # Ensure the log directory exists os.makedirs(LOG_DIR, exist_ok=True) - def setup_logging() -> logging.Logger: - """Configure logging with both file and console handlers. + """ + Configure logging with both file and console handlers. Returns: logging.Logger: Configured logger instance """ + # Create logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) + # Create formatters file_formatter = logging.Formatter( '%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s' ) @@ -35,6 +38,7 @@ def setup_logging() -> logging.Logger: '%(asctime)s - %(levelname)s - %(message)s' ) + # Create file handlers timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') debug_handler = logging.FileHandler( os.path.join(LOG_DIR, f'script_execution_debug_{timestamp}.log') @@ -48,53 +52,57 @@ def setup_logging() -> logging.Logger: error_handler.setLevel(logging.ERROR) error_handler.setFormatter(file_formatter) + # Create console handler console_handler = logging.StreamHandler() console_handler.setLevel(logging.INFO) console_handler.setFormatter(console_formatter) + # Add handlers to logger logger.addHandler(debug_handler) logger.addHandler(error_handler) logger.addHandler(console_handler) return logger - def run_script(script_name: str, logger: logging.Logger) -> bool: - """Execute a Python script using the current Python interpreter. + """ + Executes a Python script using the current Python interpreter. Args: - script_name: The name of the script to run - logger: The logger instance for logging messages + script_name (str): The name of the script to run. + logger (logging.Logger): The logger instance for logging messages. Returns: - bool: True if the script runs successfully, False otherwise + bool: True if the script runs successfully, False otherwise. """ - logger.info("Running %s...", script_name) + logger.info(f"Running {script_name}...") try: + # Run the script using the current Python interpreter result = subprocess.run( [sys.executable, script_name], capture_output=True, text=True, check=True ) - logger.info("%s completed successfully.", script_name) - logger.debug("Output:\n%s", result.stdout) + logger.info(f"{script_name} completed successfully.") + logger.debug(f"Output:\n{result.stdout}") return True - except subprocess.CalledProcessError as error: - logger.error("Error running %s:", script_name) - logger.error(error.stderr) + except subprocess.CalledProcessError as e: + logger.error(f"Error running {script_name}:") + logger.error(e.stderr) # Log the error message if the script fails return False - -def main() -> None: - """Run a list of Python scripts sequentially. +def main(): + """ + Main function to run a list of Python scripts sequentially. The function will stop execution if any script fails, preventing subsequent scripts from running if an error is encountered. """ logger = setup_logging() + # List of scripts to execute in order scripts: List[str] = [ "netbox_export.py", "network_scan.py", @@ -102,13 +110,14 @@ def main() -> None: "netbox_import.py" ] + # Iterate over the list of scripts and run each one for script in scripts: if not run_script(script, logger): - logger.error("Execution stopped due to an error in %s", script) - sys.exit(1) - - logger.info("All scripts executed successfully.") - + logger.error(f"Execution stopped due to an error in {script}") + break # Stop execution if a script fails + else: + logger.info("All scripts executed successfully.") if __name__ == "__main__": + # Run the main function if the script is executed directly main()