From 1105e5e94f2dc6d0eddc39b851a962934b0c7f57 Mon Sep 17 00:00:00 2001 From: Neil Lathwood Date: Mon, 16 Sep 2024 22:54:42 +0100 Subject: [PATCH 1/4] Updated source files to avoid namespace duplication --- lib/oxidized/manager.rb | 2 +- lib/oxidized/source/csv.rb | 89 +++++++++++----------- lib/oxidized/source/http.rb | 128 ++++++++++++++++---------------- lib/oxidized/source/jsonfile.rb | 99 ++++++++++++------------ lib/oxidized/source/source.rb | 76 ++++++++++--------- lib/oxidized/source/sql.rb | 127 +++++++++++++++---------------- 6 files changed, 264 insertions(+), 257 deletions(-) diff --git a/lib/oxidized/manager.rb b/lib/oxidized/manager.rb index d0a040a14..ecca3edf0 100644 --- a/lib/oxidized/manager.rb +++ b/lib/oxidized/manager.rb @@ -8,7 +8,7 @@ class << self def load(dir, file) require File.join dir, file + '.rb' klass = nil - [Oxidized, Object].each do |mod| + [Oxidized, Source, Object].each do |mod| klass = mod.constants.find { |const| const.to_s.casecmp(file).zero? } klass ||= mod.constants.find { |const| const.to_s.downcase == 'oxidized' + file.downcase } klass = mod.const_get klass if klass diff --git a/lib/oxidized/source/csv.rb b/lib/oxidized/source/csv.rb index dbf79abd5..8109cd036 100644 --- a/lib/oxidized/source/csv.rb +++ b/lib/oxidized/source/csv.rb @@ -1,54 +1,55 @@ module Oxidized - class CSV < Source - def initialize - @cfg = Oxidized.config.source.csv - super - end - - def setup - if @cfg.empty? - Oxidized.asetus.user.source.csv.file = File.join(Config::ROOT, 'router.db') - Oxidized.asetus.user.source.csv.delimiter = /:/ - Oxidized.asetus.user.source.csv.map.name = 0 - Oxidized.asetus.user.source.csv.map.model = 1 - Oxidized.asetus.user.source.csv.gpg = false - Oxidized.asetus.save :user - raise NoConfig, "no source csv config, edit #{Oxidized::Config.configfile}" + module Source + class CSV < Source + def initialize + @cfg = Oxidized.config.source.csv + super end - require 'gpgme' if @cfg.gpg? - - # map.name is mandatory - return if @cfg.map.has_key?('name') - - raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}" - end - def load(_node_want = nil) - nodes = [] - open_file.each_line do |line| - next if line =~ /^\s*#/ - - data = line.chomp.split(@cfg.delimiter, -1) - next if data.empty? - - # map node parameters - keys = {} - @cfg.map.each do |key, position| - keys[key.to_sym] = node_var_interpolate data[position] + def setup + if @cfg.empty? + Oxidized.asetus.user.source.csv.file = File.join(Config::ROOT, 'router.db') + Oxidized.asetus.user.source.csv.delimiter = /:/ + Oxidized.asetus.user.source.csv.map.name = 0 + Oxidized.asetus.user.source.csv.map.model = 1 + Oxidized.asetus.user.source.csv.gpg = false + Oxidized.asetus.save :user + raise NoConfig, "no source csv config, edit #{Oxidized::Config.configfile}" end - keys[:model] = map_model keys[:model] if keys.has_key? :model - keys[:group] = map_group keys[:group] if keys.has_key? :group + require 'gpgme' if @cfg.gpg? - # map node specific vars - vars = {} - @cfg.vars_map.each do |key, position| - vars[key.to_sym] = node_var_interpolate data[position] - end - keys[:vars] = vars unless vars.empty? + # map.name is mandatory + return if @cfg.map.has_key?('name') + raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}" + end - nodes << keys + def load(_node_want = nil) + nodes = [] + open_file.each_line do |line| + next if line =~ /^\s*#/ + + data = line.chomp.split(@cfg.delimiter, -1) + next if data.empty? + + # map node parameters + keys = {} + @cfg.map.each do |key, position| + keys[key.to_sym] = node_var_interpolate data[position] + end + keys[:model] = map_model keys[:model] if keys.has_key? :model + keys[:group] = map_group keys[:group] if keys.has_key? :group + + # map node specific vars + vars = {} + @cfg.vars_map.each do |key, position| + vars[key.to_sym] = node_var_interpolate data[position] + end + keys[:vars] = vars unless vars.empty? + + nodes << keys + end + nodes end - nodes end end end diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index e67cfeaaf..4b9c68610 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -1,82 +1,84 @@ module Oxidized - require "oxidized/source/jsonfile" - class HTTP < JSONFile - def initialize - super - @cfg = Oxidized.config.source.http - end - - def setup - Oxidized.setup_logger - if @cfg.empty? - Oxidized.asetus.user.source.http.url = 'https://url/api' - Oxidized.asetus.user.source.http.map.name = 'name' - Oxidized.asetus.user.source.http.map.model = 'model' - Oxidized.asetus.save :user - - raise NoConfig, "No source http config, edit #{Oxidized::Config.configfile}" + module Source + require "oxidized/source/jsonfile" + class HTTP < JSONFile + def initialize + super + @cfg = Oxidized.config.source.http end - # check for mandatory attributes - if !@cfg.has_key?('url') - raise InvalidConfig, "url is a mandatory http source attribute, edit #{Oxidized::Config.configfile}" - elsif !@cfg.map.has_key?('name') - raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}" + def setup + Oxidized.setup_logger + if @cfg.empty? + Oxidized.asetus.user.source.http.url = 'https://url/api' + Oxidized.asetus.user.source.http.map.name = 'name' + Oxidized.asetus.user.source.http.map.model = 'model' + Oxidized.asetus.save :user + + raise NoConfig, "No source http config, edit #{Oxidized::Config.configfile}" + end + + # check for mandatory attributes + if !@cfg.has_key?('url') + raise InvalidConfig, "url is a mandatory http source attribute, edit #{Oxidized::Config.configfile}" + elsif !@cfg.map.has_key?('name') + raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}" + end end - end - require "net/http" - require "net/https" - require "uri" - require "json" + require "net/http" + require "net/https" + require "uri" + require "json" - def load(node_want = nil) - uri = URI.parse(@cfg.url) - data = JSON.parse(read_http(uri, node_want)) - node_data = data - node_data = string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location? - node_data = pagination(data, node_want) if @cfg.pagination? + def load(node_want = nil) + uri = URI.parse(@cfg.url) + data = JSON.parse(read_http(uri, node_want)) + node_data = data + node_data = string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location? + node_data = pagination(data, node_want) if @cfg.pagination? - transform_json(node_data) - end + transform_json(node_data) + end - private + private - def pagination(data, node_want) - node_data = [] - raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" unless @cfg.pagination_key_name? + def pagination(data, node_want) + node_data = [] + raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" unless @cfg.pagination_key_name? - next_key = @cfg.pagination_key_name - loop do - node_data += string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location? - break if data[next_key].nil? + next_key = @cfg.pagination_key_name + loop do + node_data += string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location? + break if data[next_key].nil? - new_uri = URI.parse(data[next_key]) if data.has_key?(next_key) - data = JSON.parse(read_http(new_uri, node_want)) - node_data += string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location? + new_uri = URI.parse(data[next_key]) if data.has_key?(next_key) + data = JSON.parse(read_http(new_uri, node_want)) + node_data += string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location? + end + node_data end - node_data - end - def read_http(uri, node_want) - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = true if uri.scheme == 'https' - http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure + def read_http(uri, node_want) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true if uri.scheme == 'https' + http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure - # Add read_timeout to handle case of big list of nodes (default value is 60 seconds) - http.read_timeout = Integer(@cfg.read_timeout) if @cfg.has_key? "read_timeout" + # Add read_timeout to handle case of big list of nodes (default value is 60 seconds) + http.read_timeout = Integer(@cfg.read_timeout) if @cfg.has_key? "read_timeout" - # map headers - headers = {} - @cfg.headers.each do |header, value| - headers[header] = value - end + # map headers + headers = {} + @cfg.headers.each do |header, value| + headers[header] = value + end - req_uri = uri.request_uri - req_uri = "#{req_uri}/#{node_want}" if node_want - request = Net::HTTP::Get.new(req_uri, headers) - request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass? - http.request(request).body + req_uri = uri.request_uri + req_uri = "#{req_uri}/#{node_want}" if node_want + request = Net::HTTP::Get.new(req_uri, headers) + request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass? + http.request(request).body + end end end end diff --git a/lib/oxidized/source/jsonfile.rb b/lib/oxidized/source/jsonfile.rb index b0cd96def..2a9565625 100644 --- a/lib/oxidized/source/jsonfile.rb +++ b/lib/oxidized/source/jsonfile.rb @@ -1,61 +1,62 @@ module Oxidized - class JSONFile < Source - require "json" - def initialize - @cfg = Oxidized.config.source.jsonfile - super - end - - def setup - if @cfg.empty? - Oxidized.asetus.user.source.jsonfile.file = File.join(Oxidized::Config::ROOT, - 'router.json') - Oxidized.asetus.user.source.jsonfile.map.name = "name" - Oxidized.asetus.user.source.jsonfile.map.model = "model" - Oxidized.asetus.user.source.jsonfile.gpg = false - Oxidized.asetus.save :user - raise NoConfig, "No source json config, edit #{Oxidized::Config.configfile}" + module Source + class JSONFile < Source + require "json" + def initialize + @cfg = Oxidized.config.source.jsonfile + super end - require 'gpgme' if @cfg.gpg? - - # map.name is mandatory - return if @cfg.map.has_key?('name') - - raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}" - end - - def load(*) - data = JSON.parse(open_file.read) - data = string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location? - transform_json(data) - end + def setup + if @cfg.empty? + Oxidized.asetus.user.source.jsonfile.file = File.join(Oxidized::Config::ROOT, + 'router.json') + Oxidized.asetus.user.source.jsonfile.map.name = "name" + Oxidized.asetus.user.source.jsonfile.map.model = "model" + Oxidized.asetus.user.source.jsonfile.gpg = false + Oxidized.asetus.save :user + raise NoConfig, "No source json config, edit #{Oxidized::Config.configfile}" + end + require 'gpgme' if @cfg.gpg? - private + # map.name is mandatory + return if @cfg.map.has_key?('name') + raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}" + end - def transform_json(data) - nodes = [] - data.each do |node| - next if node.empty? + def load(*) + data = JSON.parse(open_file.read) + data = string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location? - # map node parameters - keys = {} - @cfg.map.each do |key, want_position| - keys[key.to_sym] = node_var_interpolate string_navigate_object(node, want_position) - end - keys[:model] = map_model keys[:model] if keys.has_key? :model - keys[:group] = map_group keys[:group] if keys.has_key? :group + transform_json(data) + end - # map node specific vars - vars = {} - @cfg.vars_map.each do |key, want_position| - vars[key.to_sym] = node_var_interpolate string_navigate_object(node, want_position) + private + + def transform_json(data) + nodes = [] + data.each do |node| + next if node.empty? + + # map node parameters + keys = {} + @cfg.map.each do |key, want_position| + keys[key.to_sym] = node_var_interpolate string_navigate_object(node, want_position) + end + keys[:model] = map_model keys[:model] if keys.has_key? :model + keys[:group] = map_group keys[:group] if keys.has_key? :group + + # map node specific vars + vars = {} + @cfg.vars_map.each do |key, want_position| + vars[key.to_sym] = node_var_interpolate string_navigate_object(node, want_position) + end + keys[:vars] = vars unless vars.empty? + + nodes << keys end - keys[:vars] = vars unless vars.empty? - - nodes << keys + nodes end - nodes end end end diff --git a/lib/oxidized/source/source.rb b/lib/oxidized/source/source.rb index 713dbca7d..71fbaa97a 100644 --- a/lib/oxidized/source/source.rb +++ b/lib/oxidized/source/source.rb @@ -1,50 +1,52 @@ module Oxidized - class Source - class NoConfig < OxidizedError; end + module Source + class Source + class NoConfig < OxidizedError; end - def initialize - @model_map = Oxidized.config.model_map || {} - @group_map = Oxidized.config.group_map || {} - end + def initialize + @model_map = Oxidized.config.model_map || {} + @group_map = Oxidized.config.group_map || {} + end - def map_model(model) - @model_map.has_key?(model) ? @model_map[model] : model - end + def map_model(model) + @model_map.has_key?(model) ? @model_map[model] : model + end - def map_group(group) - @group_map.has_key?(group) ? @group_map[group] : group - end + def map_group(group) + @group_map.has_key?(group) ? @group_map[group] : group + end - def node_var_interpolate(var) - case var - when "nil" then nil - when "false" then false - when "true" then true - else var + def node_var_interpolate(var) + case var + when "nil" then nil + when "false" then false + when "true" then true + else var + end end - end - private + private - def open_file - file = File.expand_path(@cfg.file) - if @cfg.gpg? - crypto = GPGME::Crypto.new password: @cfg.gpg_password - crypto.decrypt(File.open(file)).to_s - else - File.open(file) + def open_file + file = File.expand_path(@cfg.file) + if @cfg.gpg? + crypto = GPGME::Crypto.new password: @cfg.gpg_password + crypto.decrypt(File.open(file)).to_s + else + File.open(file) + end end - end - def string_navigate_object(object, wants) - wants = wants.split(".").map do |want| - head, match, _tail = want.partition(/\[\d+\]/) - match.empty? ? head : [head, match[1..-2].to_i] - end - wants.flatten.each do |want| - object = object[want] if object.respond_to? :each + def string_navigate_object(object, wants) + wants = wants.split(".").map do |want| + head, match, _tail = want.partition(/\[\d+\]/) + match.empty? ? head : [head, match[1..-2].to_i] + end + wants.flatten.each do |want| + object = object[want] if object.respond_to? :each + end + object end - object end end -end +end \ No newline at end of file diff --git a/lib/oxidized/source/sql.rb b/lib/oxidized/source/sql.rb index c312aa881..b87158e14 100644 --- a/lib/oxidized/source/sql.rb +++ b/lib/oxidized/source/sql.rb @@ -1,80 +1,81 @@ module Oxidized - class SQL < Source - begin - require 'sequel' - rescue LoadError - raise OxidizedError, 'sequel not found: sudo gem install sequel' - end - - def setup - if @cfg.empty? - Oxidized.asetus.user.source.sql.adapter = 'sqlite' - Oxidized.asetus.user.source.sql.database = File.join(Config::ROOT, 'sqlite.db') - Oxidized.asetus.user.source.sql.table = 'devices' - Oxidized.asetus.user.source.sql.map.name = 'name' - Oxidized.asetus.user.source.sql.map.model = 'rancid' - Oxidized.asetus.save :user - raise NoConfig, "No source sql config, edit #{Oxidized::Config.configfile}" + module Source + class SQL < Source + begin + require 'sequel' + rescue LoadError + raise OxidizedError, 'sequel not found: sudo gem install sequel' end - # map.name is mandatory - return if @cfg.map.has_key?('name') + def setup + if @cfg.empty? + Oxidized.asetus.user.source.sql.adapter = 'sqlite' + Oxidized.asetus.user.source.sql.database = File.join(Config::ROOT, 'sqlite.db') + Oxidized.asetus.user.source.sql.table = 'devices' + Oxidized.asetus.user.source.sql.map.name = 'name' + Oxidized.asetus.user.source.sql.map.model = 'rancid' + Oxidized.asetus.save :user + raise NoConfig, "No source sql config, edit #{Oxidized::Config.configfile}" + end - raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}" - end + # map.name is mandatory + return if @cfg.map.has_key?('name') - def load(node_want = nil) - nodes = [] - db = connect - query = db[@cfg.table.to_sym] - query = query.with_sql(@cfg.query) if @cfg.query? + end - query = query.where(@cfg.map.name.to_sym => node_want) if node_want + def load(node_want = nil) + nodes = [] + db = connect + query = db[@cfg.table.to_sym] + query = query.with_sql(@cfg.query) if @cfg.query? - query.each do |node| - # map node parameters - keys = {} - @cfg.map.each { |key, sql_column| keys[key.to_sym] = node_var_interpolate node[sql_column.to_sym] } - keys[:model] = map_model keys[:model] if keys.has_key? :model - keys[:group] = map_group keys[:group] if keys.has_key? :group + query = query.where(@cfg.map.name.to_sym => node_want) if node_want - # map node specific vars - vars = {} - @cfg.vars_map.each do |key, sql_column| - vars[key.to_sym] = node_var_interpolate node[sql_column.to_sym] - end - keys[:vars] = vars unless vars.empty? + query.each do |node| + # map node parameters + keys = {} + @cfg.map.each { |key, sql_column| keys[key.to_sym] = node_var_interpolate node[sql_column.to_sym] } + keys[:model] = map_model keys[:model] if keys.has_key? :model + keys[:group] = map_group keys[:group] if keys.has_key? :group - nodes << keys + # map node specific vars + vars = {} + @cfg.vars_map.each do |key, sql_column| + vars[key.to_sym] = node_var_interpolate node[sql_column.to_sym] + end + keys[:vars] = vars unless vars.empty? + + nodes << keys + end + db.disconnect + nodes end - db.disconnect - nodes - end - private + private - def initialize - super - @cfg = Oxidized.config.source.sql - end + def initialize + super + @cfg = Oxidized.config.source.sql + end - def connect - options = { - adapter: @cfg.adapter, - host: @cfg.host?, - user: @cfg.user?, - password: @cfg.password?, - database: @cfg.database, - ssl_mode: @cfg.ssl_mode? - } - if @cfg.with_ssl? - options.merge!(sslca: @cfg.ssl_ca?, - sslcert: @cfg.ssl_cert?, - sslkey: @cfg.ssl_key?) + def connect + options = { + adapter: @cfg.adapter, + host: @cfg.host?, + user: @cfg.user?, + password: @cfg.password?, + database: @cfg.database, + ssl_mode: @cfg.ssl_mode? + } + if @cfg.with_ssl? + options.merge!(sslca: @cfg.ssl_ca?, + sslcert: @cfg.ssl_cert?, + sslkey: @cfg.ssl_key?) + end + Sequel.connect(options) + rescue Sequel::AdapterNotFound => e + raise OxidizedError, "SQL adapter gem not installed: " + e.message end - Sequel.connect(options) - rescue Sequel::AdapterNotFound => e - raise OxidizedError, "SQL adapter gem not installed: " + e.message end end end From ef3c94daa7af102f6c4243b2bddffde3b8751e9f Mon Sep 17 00:00:00 2001 From: Neil Lathwood Date: Tue, 17 Sep 2024 09:22:17 +0100 Subject: [PATCH 2/4] Added blank lines to end of files --- lib/oxidized/source/source.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oxidized/source/source.rb b/lib/oxidized/source/source.rb index 71fbaa97a..c4d59f320 100644 --- a/lib/oxidized/source/source.rb +++ b/lib/oxidized/source/source.rb @@ -49,4 +49,4 @@ def string_navigate_object(object, wants) end end end -end \ No newline at end of file +end From 6da869a29d0dae4bf1350734dd55131d14d2af92 Mon Sep 17 00:00:00 2001 From: Neil Lathwood Date: Tue, 17 Sep 2024 09:24:57 +0100 Subject: [PATCH 3/4] Updated tests --- spec/source/http_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/source/http_spec.rb b/spec/source/http_spec.rb index 883e4050a..896723b25 100644 --- a/spec/source/http_spec.rb +++ b/spec/source/http_spec.rb @@ -63,16 +63,16 @@ _(http.send(:string_navigate_object, h1, "jotain.2")).must_equal "jotain" end it "should be able to navigate multilevel-hash" do - _(Oxidized::HTTP.new.send(:string_navigate_object, h1, "jotain.2")).must_equal "jotain" + _(Oxidized::Source::HTTP.new.send(:string_navigate_object, h1, "jotain.2")).must_equal "jotain" end it "should be able to navigate hash/array combination" do - _(Oxidized::HTTP.new.send(:string_navigate_object, h1, "inventory[0].ip")).must_equal "10.10.10.10" + _(Oxidized::Source::HTTP.new.send(:string_navigate_object, h1, "inventory[0].ip")).must_equal "10.10.10.10" end it "should return nil on non-existing string key" do - _(Oxidized::HTTP.new.send(:string_navigate_object, h1, "jotain.3")).must_be_nil + _(Oxidized::Source::HTTP.new.send(:string_navigate_object, h1, "jotain.3")).must_be_nil end it "should return nil on non-existing array index" do - _(Oxidized::HTTP.new.send(:string_navigate_object, h1, "inventory[3]")).must_be_nil + _(Oxidized::Source::HTTP.new.send(:string_navigate_object, h1, "inventory[3]")).must_be_nil end end end From 10b2463279f5f2b2d3dca0a4f131ddaeedbab0f8 Mon Sep 17 00:00:00 2001 From: Neil Lathwood Date: Tue, 17 Sep 2024 13:15:21 +0100 Subject: [PATCH 4/4] Updated tests file --- spec/source/http_spec.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/source/http_spec.rb b/spec/source/http_spec.rb index 896723b25..faf2d67ee 100644 --- a/spec/source/http_spec.rb +++ b/spec/source/http_spec.rb @@ -1,7 +1,7 @@ require_relative '../spec_helper' require 'oxidized/source/http' -describe Oxidized::HTTP do +describe Oxidized::Source::HTTP do before(:each) do Oxidized.asetus = Asetus.new Oxidized.setup_logger @@ -58,8 +58,8 @@ h1["inventory"] = [{ "ip" => "10.10.10.10" }] h1["jotain"] = { "2" => "jotain" } it "should be able to navigate multilevel-hash" do - http = Oxidized::HTTP.new - _(http.class).must_equal Oxidized::HTTP + http = Oxidized::Source::HTTP.new + _(http.class).must_equal Oxidized::Source::HTTP _(http.send(:string_navigate_object, h1, "jotain.2")).must_equal "jotain" end it "should be able to navigate multilevel-hash" do