-
Notifications
You must be signed in to change notification settings - Fork 2
/
net-consolidator.py
299 lines (250 loc) · 11 KB
/
net-consolidator.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
#!/usr/bin/python3
from netaddr import *
import os
import argparse
import re
from argparse import RawTextHelpFormatter
# Parse command line options
############################
description="""
_ _ _ ____ _ _ _ _
| \ | | ___| |_ / ___|___ _ __ ___ ___ | (_) __| | __ _| |_ ___ _ __
| \| |/ _ \ __|____| | / _ \| '_ \/ __|/ _ \| | |/ _` |/ _` | __/ _ \| '__|
| |\ | __/ ||_____| |__| (_) | | | \__ \ (_) | | | (_| | (_| | || (_) | |
|_| \_|\___|\__| \____\___/|_| |_|___/\___/|_|_|\__,_|\__,_|\__\___/|_|
Merge and subtract IP lists
Manually consolidating, merging or subtracting huge, overlapping IP lists is a difficult task. This tool
provides those functions and processes files with IP ranges. In addition, the tool supports the following features:
* Resulting IP networks are merged in CIDR notation while duplicates are removed
* Accepts the following IP notations: 192.168.1.1/32 | 192.168.1.1 | 192.168.1.1-192.168.1.2
EXAMPLES
--------
Following examples are given for specific scenarios:
* Consolidating of duplicates and hosts in IP lists (e.g. one subnet already includes parts of a different given subnet )
EXAMPLE: --fileIpAddresses <text_file_with_ip_addresses>
* Perform plausibility checks (e.g. the subnet size should not exceed /20)
EXAMPLE: --fileIpAddresses <text_file_with_ip_addresses> --plausibilityChecks True --subnetSize 20
* Return the IP address delta of two IP lists (e.g. ip_list_added - ip_list_already_scanned = delta)
EXAMPLE: --fileIpAddresses <text_file_with_ip_addresses_unscanned> --fileIpSubtracts <text_file_with_ip_addresses_scanned>
* Exclude specific IP addresses from the IP list (e.g. should explicitely not be scanned)
EXAMPLE: --fileIpAddresses <text_file_with_ip_addresses_main> --fileIpSubtracts <text_file_with_ip_addresses_exclude>
* Split given IP ranges in equally sized slices (e.g. 10 times 500 IP addresses)
EXAMPLE: --fileIpAddresses <text_file_with_ip_addresses> --splitIpRangeInSlices 10
PARAMETER
---------
"""
parser = argparse.ArgumentParser(description=description, formatter_class=RawTextHelpFormatter)
parser.add_argument('-f', '--fileIpAddresses', help='text file with ip addresses per line (format: 192.168.0.1, 192.168.0.1/24, 192.168.0.1-192.168.0.10)', default='')
parser.add_argument('-s', '--fileIpSubtracts', help='(optional) text file with ip addresses per line to subtract from the fileIpAddresses')
parser.add_argument('-p', '--plausibilityChecks', help='(optional) enable plausibility checks (e.g. subnet size)')
parser.add_argument('-n', '--subnetSize', help='(optional) size of the network in CIDR (e.g. 24 for /24) for plausibility checks')
parser.add_argument('-t', '--splitIpRangeInSlices', help='(optional) split IP networks in equally sized slices (e.g. 10 times 500 IP addresses')
args = parser.parse_args()
# check: parameter fileIpAddresses is a file
if args.fileIpAddresses:
if not os.path.isfile(args.fileIpAddresses):
print('[!] "' + args.fileIpAddresses + '" is not a valid file.', True)
else:
fileIpAddr = args.fileIpAddresses
# check: parameter fileIpSubtracts is a file
if args.fileIpSubtracts:
if not os.path.isfile(args.fileIpSubtracts):
print('[!] "' + args.fileIpSubtracts + '" is not a valid file.', True)
else:
fileIpSub = args.fileIpSubtracts
# check: parameter fileIpAddresses is mandatory
else:
print('[!] No "--fileIpAddresses, -f" parameter was provided')
parser.print_help()
exit()
# Merge IP ranges so that no duplicates are existing anymore
############################################################
def mergeIpRanges( fileIpAddr ):
"""
@type fileIpAddr: Filename
@param fileIpAddr: File with IP addresses
@rtype: IPSet()
@return: List of merged IP ranges
"""
ip_list = readIpFile ( fileIpAddr )
ip_list_merge = cidr_merge(ip_list)
return ip_list_merge
# Helper function: Read IP addresses from a file (one IP per line)
####################################################################
def readIpFile ( ipFile ):
"""
@type ipFile: Filename
@param ipFile: File with IP addresses
@rtype: IPSet()
@return: List of IP ranges
"""
# init return value
ip_list = IPSet()
# iterate over each line in the IP address file
with open( ipFile ) as f:
content = f.readlines()
# remove whitespace characters like `\n` at the end of each line
content = [x.strip() for x in content]
for line in content:
# update the list with fetched ip objects
ip_list.update(getValidIpObject( line ))
return ip_list
# Get valid IP based on string patterns
# Format: CIDR (192.168.0.1/24) + Ranges (192.168.1.1-192.168.1.255)
####################################################################
def getValidIpObject( line_with_ip ):
"""
@type line_with_ip: String
@param line_with_ip: IP address(es) as a String (e.g. 192.168.0.1-192.168.0.2)
@rtype: IPSet()
@return: List of IP ranges
"""
# temporary IPSet for IP objects like IP ranges
valid_ip_list = IPSet()
# Regexp definitions for parsing the ip address from a string representation
# source: https://www.regular-expressions.info/ip.html and modified to ip-ip
# regexp for case 192.168.0.1/24 | 192.168.0.1/8 | 192.168.0.1
regexp_ip = re.compile('^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:|/[0-9]{1,2})[ |]*$')
# regexp for case 192.168.1.1-192.168.1.255
regexp_ip_range = re.compile('^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)( *)-( *|)(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(|.*)$',re.DOTALL)
regexp_ip_separate = re.compile('(.*)-(.*)',re.DOTALL)
# Check: normal ip (192.168.1.1/24, 192.168.1.1)
if regexp_ip.match ( line_with_ip ):
valid_ip_list.add(IPNetwork(line_with_ip))
# Check: ip range (192.168.1.1-192.168.1.2)
elif regexp_ip_range.match( line_with_ip ):
# returns a dict
ips = regexp_ip_separate.findall(line_with_ip)
ip1 = ips[0][0]
ip2 = ips[0][1]
# cleanup empty characters
ip1 = ip1.replace(' ','')
ip2 = ip2.replace(' ','')
# append the ip range
iprange = IPRange(ip1, ip2)
for ipnetwork in iprange.cidrs():
valid_ip_list.add(ipnetwork)
else:
print("[!] Not a valid IP address in the text file:" + line_with_ip)
print("[*] Stopping now!")
exit()
return valid_ip_list
# Plausibility checks
#####################
def checkIpOnPlausibility( ip ):
"""
@type ip: String
@param ip: IP address or IP network (e.g. 192.168.0.1/24)
@rtype: Boolean
@return: If the IP passed the plausibility checks (passed, not passed)
"""
# check for parameter subnetsize
regexp_ip_subnet_check = re.compile('^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:|/([0-9]{1,2}))[ |]*$')
if args.subnetSize:
subnetSize = int(args.subnetSize)
else:
subnetSize = 22
subnetRegexGroup = regexp_ip_subnet_check.findall( str(ip) )
if int(subnetRegexGroup[0]) > subnetSize:
return True
else:
print("[!] IP did not pass the checks for subnet size (>%s): %s" % (subnetSize, ip) )
return False
return True
# Subtract IP ranges for delta between IP network A and B
#########################################################
def subtractIpRanges( fileIpAddr, fileIpSub ):
"""
@type fileIpAddr: String
@param fileIpAddr: Filename of a file with IP addresses
@type fileIpSub: String
@param fileIpSub: Filename of a file with IP addresses
@rtype: IPSet()
@return: List of IP ranges
"""
# parse the files for IP addresses
ip_list_unscanned = readIpFile ( fileIpAddr )
ip_list_scanned = readIpFile ( fileIpSub )
# Subtract the ip lists
ip_list_toscan = ip_list_unscanned - ip_list_scanned
# Merge the result ip list
ip_list_toscan_merge = cidr_merge(ip_list_toscan)
return ip_list_toscan_merge
# Split IP ranges into slices each of the same size
#####################################################
def splitIpRanges( numberOfSlices, ip_list ):
"""
@type numberOfSlices: String
@param numberOfSlices: The number of resulting slices of IP ranges
@type ip_list: IPSet()
@param ip_list: List of IP Networks
@rtype: None
@return: None
"""
# number of all IP addresses
totalCount = 0
for ip_range in ip_list:
totalCount = totalCount + ip_range.size
print("[*] Totalcount:\t", totalCount)
# slice in equal parts
slice_size = int(totalCount/int(numberOfSlices))
print("[*] Slice size:\t", slice_size)
# iterate over the ip ranges and over each ip
buffer_size = 0
bucket = IPSet()
list_of_slices = []
for ip_range in ip_list:
for ip in ip_range:
# Fill the bucket with single ip addresses - update the buffer size
bucket.add(ip)
buffer_size = buffer_size + 1
# check: slice is full
if buffer_size == slice_size:
# save this slice and reset
buffer_size = 0
list_of_slices.append(bucket)
bucket = IPSet()
# check: save the last slice (e.g. bucket not completely full)
if buffer_size != 0:
list_of_slices.append(bucket)
# print all ip ranges
for bucket_id in range(0,len(list_of_slices)):
# init
bucket = list_of_slices[bucket_id]
print("\n\n[*] Bucket # ", bucket_id, " - bucket size ", bucket.size)
for cidr in bucket.iter_cidrs():
print(cidr)
# Main
######
# Case: Substracting IP ranges in file2 from file1
if args.fileIpSubtracts:
ip_list = subtractIpRanges(fileIpAddr, fileIpSub)
# Check plausibility
if args.plausibilityChecks:
for ip in ip_list:
if not ( checkIpOnPlausibility( '%s' % ip )):
print ("[*] IP list did not pass the plausibility checks ...")
exit()
if args.splitIpRangeInSlices:
splitIpRanges( args.splitIpRangeInSlices, ip_list )
if not args.splitIpRangeInSlices:
# Print result list
for ip in ip_list:
print('%s' % ip)
exit()
# Case: Only Merging IP ranges
elif args.fileIpAddresses:
ip_list = mergeIpRanges(fileIpAddr)
# Check plausibility
if args.plausibilityChecks:
for ip in ip_list:
if not ( checkIpOnPlausibility( '%s' % ip )):
print ("[*] IP list did not pass the plausibility checks ...")
exit()
if args.splitIpRangeInSlices:
splitIpRanges( args.splitIpRangeInSlices, ip_list )
if not args.splitIpRangeInSlices:
# Print result list
for ip in ip_list:
print('%s' % ip)
exit()