Skip to content

Commit

Permalink
repair nwo ingestor (#1075)
Browse files Browse the repository at this point in the history
  • Loading branch information
mikesndrs authored Jan 15, 2025
1 parent 2e8b69c commit fdea694
Show file tree
Hide file tree
Showing 3 changed files with 4,313 additions and 43 deletions.
4 changes: 2 additions & 2 deletions lib/ingestors/nwo_ingestor.rb
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def process_nwo(url)
unless Rails.env.test? and File.exist?('test/vcr_cassettes/ingestors/nwo.yml')
sleep(1)
end
event_page = Nokogiri::HTML5.parse(open_url("#{url}?page=#{i}", raise: true)).css(".overviewContent > .listing-cards > li.list-item > a")
event_page = Nokogiri::HTML5.parse(open_url("#{url}?page=#{i}", raise: true)).css(".overviewContent > .listing-cards > li.list-item")
event_page.each do |event_data|
event = OpenStruct.new

Expand All @@ -43,7 +43,7 @@ def process_nwo(url)
event.keywords = []
event.description = convert_description event_data.css('.card__intro').inner_html

event.url = "https://www.nwo.nl#{event_data['href']}"
event.url = "https://www.nwo.nl#{event_data.css('h3.card__title > a').attribute('href').value}"

event.source = 'NWO'

Expand Down
16 changes: 8 additions & 8 deletions test/unit/ingestors/nwo_ingestor_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ class NwoIngestorTest < ActiveSupport::TestCase
ingestor = Ingestors::NwoIngestor.new

# check event doesn't
new_title = 'NWO Biophysics'
new_url = 'https://www.nwo.nl/en/meetings/biophysics'
new_title = 'NL Polar day 2025'
new_url = 'https://www.nwo.nl/en/meetings/nl-polar-day-2025'
refute Event.where(title: new_title, url: new_url).any?

# run task
assert_difference 'Event.count', 24 do
assert_difference 'Event.count', 14 do
freeze_time(2019) do
VCR.use_cassette('ingestors/nwo') do
ingestor.read(source.url)
Expand All @@ -36,9 +36,9 @@ class NwoIngestorTest < ActiveSupport::TestCase
end
end

assert_equal 24, ingestor.events.count
assert_equal 14, ingestor.events.count
assert ingestor.materials.empty?
assert_equal 24, ingestor.stats[:events][:added]
assert_equal 14, ingestor.stats[:events][:added]
assert_equal 0, ingestor.stats[:events][:updated]
assert_equal 0, ingestor.stats[:events][:rejected]

Expand All @@ -49,10 +49,10 @@ class NwoIngestorTest < ActiveSupport::TestCase
assert_equal new_url, event.url

# check other fields
assert_equal 'NWO Biophysics', event.title
assert_equal 'NL Polar day 2025', event.title
assert_equal 'Amsterdam', event.timezone
assert_equal 'NWO', event.source
assert_equal Time.zone.parse('Mon, 09 Oct 2023 09:00:00.000000000 UTC +00:00'), event.start
assert_equal Time.zone.parse('Tue, 10 Oct 2023 17:00:00.000000000 UTC +00:00'), event.end
assert_equal Time.zone.parse('Mon, 8 Apr 2025 09:00:00.000000000 UTC +00:00'), event.start
assert_equal Time.zone.parse('Mon, 8 Apr 2025 17:00:00.000000000 UTC +00:00'), event.end
end
end
4,336 changes: 4,303 additions & 33 deletions test/vcr_cassettes/ingestors/nwo.yml

Large diffs are not rendered by default.

0 comments on commit fdea694

Please sign in to comment.