Get future calendar working again.

classes/Utility/Calendar.php

@@ -70,22 +70,17 @@ public static function displayEntry($e) {
 
         if ($e['witnesses']) {
             print "<dd>";
-            print '<a href=" $e[link_calendar] "></a>';
-            print '<a href=" $e[link_external] "></a>';
-            print 'Witnesses: ' . $e['witnesses'];
+            print 'Witnesses: <ul><li>' . str_replace("\n", '<li>', $e['witnesses']) . '</ul>';
             print "</dd>\n";
         }
     }
 
     public static function meta($e) {
-        $private = false;
         if ($e['committee_name']) {
             $title = $e['committee_name'];
             if ($e['title'] == 'to consider the Bill') {
-            } elseif ($e['title'] && $e['title'] != 'This is a private meeting.') {
+            } elseif ($e['title']) {
                 $title .= ': ' . $e['title'];
-            } else {
-                $private = true;
             }
         } else {
             $title = $e['title'];
@@ -125,10 +120,6 @@ public static function meta($e) {
             }
         }
 
-        if ($private) {
-            $meta[] = 'Private meeting';
-        }
-
         return array($title, $meta);
     }
 

scripts/future-fetch.py

@@ -1,9 +1,11 @@
 #!/usr/bin/python
+# encoding: utf-8
 
+import json
 import os
 import sys
 import re
-from lxml import objectify
+import urllib2
 import MySQLdb
 
 import datetime
@@ -22,17 +24,42 @@
 from resolvemembernames import memberList
 from lords.resolvenames import lordsList
 
-CALENDAR_URL = 'http://services.parliament.uk/calendar/all.rss'
+CALENDAR_LINK = 'https://calendar.parliament.uk/calendar/%(house)s/All/%(y)d/%(m)d/%(d)d/Daily'
+CALENDAR_BASE = 'https://calendar.parliament.uk/Calendar/Refresh'
+CALENDAR_DATA = 'StartDate=%(date)s&ViewBy=Daily&House=%(house)s&EventGrouping=All'
 
 positions = {}
 
 
+def fetch_url(date, house):
+    data = CALENDAR_DATA % {'date': date, 'house': house}
+    data = urllib2.urlopen(CALENDAR_BASE, data)
+    data = json.load(data)
+    data = data['Results']
+    return data
+
+def get_calendar_events():
+    start_date = datetime.date.today()
+    WEEKS = 8
+    for date in (start_date + datetime.timedelta(n) for n in range(WEEKS*7)):
+        if date.weekday() >= 5: continue
+        for house in ('Commons', 'Lords'):
+            data = fetch_url(date, house)
+            for group in data['Groupings']:
+                chamber = group['Name']
+                if house == 'Lords' and chamber == 'Joint Committees':
+                    # Duplicated in Lords and Commons
+                    continue
+                for event in group['Events']:
+                    yield Entry(date, house, chamber, event)
+
+
 class Entry(object):
     id = None
     modified = None
     deleted = 0
     link_calendar = None
-    link_external = None
+    link_external = ''
     body = 'uk'
     chamber = None
     event_date = None
@@ -45,78 +72,67 @@ class Entry(object):
     witnesses_str = ''
     location = ''
 
-    def __init__(self, entry):
-        event = entry['{http://services.parliament.uk/ns/calendar/feeds}event']
-        self.id = event.attrib['id']
+    def __init__(self, date, house, chamber, event):
+        self.id = event['Id'] + 100000
         self.deleted = 0
-        self.link_calendar = entry.guid
-        self.link_external = entry.link
-        chamber = event.chamber.text.strip()
-        self.chamber = '%s: %s' % (event.house.text.strip(), chamber)
-        self.event_date = event.date.text
-        self.time_start = getattr(event, 'startTime', None)
-        self.time_end = getattr(event, 'endTime', None)
-
-        committee_text = event.comittee.text
-        if committee_text:
-            committee_text = committee_text.strip()
-            if chamber in ('Select Committee', 'General Committee'):
-                self.committee_name = committee_text
-            elif committee_text != "Prime Minister's Question Time":
-                self.debate_type = committee_text
+        self.link_calendar = CALENDAR_LINK % {'house': house, 'y': date.year, 'm': date.month, 'd': date.day}
+        if chamber == 'Joint Committees':
+            self.chamber = chamber
+        else:
+            self.chamber = '%s: %s' % (house, chamber)
+
+        self.event_date = date.isoformat()
+        self.time_start = event['StartDateTime'][11:]
+        self.time_end = event['EndDateTime'][11:]
+        if not event['DisplayStartTime']:
+            self.time_start = None
+        if not event['DisplayEndTime']:
+            self.time_end = None
+
+        self.committee_name = event['CommitteeName'] or ''
+        if event['Title'] == "Prime Minister's Question Time":
+            self.debate_type = 'Oral questions'
+            self.title = event['Title']
+        else:
+            self.debate_type = event['Title']
+            self.title = event['Description'] or ''
+        subject = event['InquirySubject']
+        if subject and self.title:
+            self.title += ': ' + subject
+        elif subject:
+            self.title = subject
 
         self.people = []
+        for member in event['Members']:
+            id = str(member['Id'])
+            match = memberList.match_by_mnis(id, self.event_date)
+            if not match:
+                match = lordsList.match_by_mnis(id, self.event_date)
+            if match:
+		self.people.append(
+		    int(match['id'].replace('uk.org.publicwhip/person/', ''))
+		    )
 
-        title_text = event.inquiry.text
-        if title_text:
-            m = re.search(' - ([^-]*)$', title_text)
-            if m:
-                person_texts = [x.strip() for x in m.group(1).split('/')]
-
-                for person_text in person_texts:
+        self.witnesses = []
+        witnesses_str = []
+        for activity in event['Activities']:
+            for attendee in activity['Attendees']:
+                m = re.match(r'\b(\w+ \w+ MP)', attendee)
+                if m:
+                    mp = m.group(1)
                     id, name, cons = memberList.matchfullnamecons(
-                        person_text, None, self.event_date
-                        )
-                    if not id:
-                        try:
-                            id = lordsList.GetLordIDfname(
-                                person_text, None, self.event_date
-                                )
-                        except:
-                            pass
+                        mp, None, self.event_date
+                    )
                     if id:
-                        self.people.append(
-                            int(id.replace('uk.org.publicwhip/person/', ''))
-                            )
-
-                if len(self.people) == len(person_texts):
-                    title_text = title_text.replace(' - ' + m.group(1), '')
+                        pid = int(id.replace('uk.org.publicwhip/person/', ''))
+                        mp_link = '<a href="/mp/?p=%d">%s</a>' % (pid, mp)
+                        self.witnesses.append(pid)
+                        witnesses_str.append(attendee.replace(mp, mp_link))
+                        continue
+                witnesses_str.append(attendee)
+        self.witnesses_str = '\n'.join(witnesses_str)
 
-            self.title = title_text.strip()
-        elif committee_text == "Prime Minister's Question Time":
-            self.title = committee_text
-
-        self.witnesses = []
-        witness_text = event.witnesses.text
-        if witness_text == 'This is a private meeting.':
-            self.title = witness_text
-        elif witness_text:
-            self.witnesses_str = witness_text.strip()
-            m = re.findall(r'\b(\w+ \w+ MP)', self.witnesses_str)
-            for mp in m:
-                id, name, cons = memberList.matchfullnamecons(
-                    mp, None, self.event_date
-                )
-                if not id:
-                    continue
-                pid = int(id.replace('uk.org.publicwhip/person/', ''))
-                mp_link = '<a href="/mp/?p=%d">%s</a>' % (pid, mp)
-                self.witnesses.append(pid)
-                self.witnesses_str = self.witnesses_str.replace(mp, mp_link)
-
-        location_text = event.location.text
-        if location_text:
-            self.location = location_text.strip()
+        self.location = event['Location'] or ''
 
     def get_tuple(self):
         return (
@@ -207,18 +223,14 @@ def update(self):
 db_cursor = db_connection.cursor()
 
 
-parsed = objectify.parse(CALENDAR_URL)
-root = parsed.getroot()
 
 # Get the id's of entries from the future as the database sees it.
 # We'll delete ids from here as we go, and what's left will be things
 # which are no longer in Future Business.
 db_cursor.execute('select id from future where event_date > CURRENT_DATE()')
 old_entries = set(db_cursor.fetchall())
 
-entries = root.channel.findall('item')
-for entry in entries:
-    new_entry = Entry(entry)
+for new_entry in get_calendar_events():
     id = new_entry.id
     event_date = new_entry.event_date
     positions[event_date] = positions.setdefault(event_date, 0) + 1
@@ -242,16 +254,16 @@ def update(self):
 
         # For some reason the time fields come out as timedelta rather that
         # time, so need converting.
-        old_tuple = (str(old_row[0]),) + \
-            old_row[1:6] + \
+        old_tuple = \
+            old_row[0:6] + \
             (old_row[6].isoformat(), ) + \
             ((datetime.datetime.min + old_row[7]).time().isoformat() if old_row[7] is not None else None,) + \
             ((datetime.datetime.min + old_row[8]).time().isoformat() if old_row[8] is not None else None,) + \
             old_row[9:]
 
         new_tuple = new_entry.get_tuple()
 
-        if old_tuple != new_entry.get_tuple():
+        if old_tuple != new_tuple:
             new_entry.update()
 
         old_entries.discard((long(id),))
@@ -265,3 +277,5 @@ def update(self):
 db_cursor.executemany(
     'UPDATE future SET deleted=1 WHERE id=%s', tuple(old_entries)
     )
+
+db_connection.commit()

www/docs/style/sass/app.scss

@@ -229,6 +229,7 @@ form {
 @import "pages/admin";
 @import "pages/login";
 @import "pages/divisions";
+@import "pages/future";
 
 .content-page {
     p, ol, ul {

www/docs/style/sass/pages/_business.scss

@@ -405,4 +405,4 @@ input.search-section__submit {
     @media (min-height: 700px) {
         margin-top: 3em;
     }
-}
+}

www/docs/style/sass/pages/_future.scss

@@ -0,0 +1,29 @@
+ul.future {
+    list-style-type: disc;
+    padding-left: 1.5em;
+}
+
+dl.future dt {
+    font-weight: normal;
+    margin-top: 0.5em;
+}
+dl.future dt.sc {
+    font-weight: bold;
+}
+
+dl.future dt span, ul.future li span, span.future_meta {
+    white-space: nowrap;
+    color: #666666;
+    font-size: 0.85em;
+    font-weight: normal;
+}
+
+dl.future dt span.type {
+}
+dl.future dt span.meta {
+}
+
+dl.future dd {
+    margin: 0 0 0.5em 1.5em;
+    font-size: 0.85em;
+}

www/includes/easyparliament/templates/html/calendar_date.php

@@ -7,29 +7,28 @@
 $order = array(
     'Commons: Main Chamber', 'Lords: Main Chamber',
     'Commons: Westminster Hall',
-    'Commons: General Committee',
-    'Commons: Select Committee', 'Lords: Select Committee',
-    'Lords: Grand Committee',
+    'Commons: General Committees',
+    'Commons: Select Committees', 'Lords: Select Committees',
+    'Lords: Grand Committees',
+    'Joint Committees',
 );
-$plural = array(0, 0, 0, 1, 1, 1, 0);
-$list   = array(1, 1, 1, 1, 0, 0, 1);
-$major  = array(1, 101, 2, 0, 0, 0, 0);
+$list   = array(1, 1, 1, 1, 0, 0, 1, 1);
+$major  = array(1, 101, 2, 6, 0, 0, 0, 0);
 
 # Content goes here
 foreach ($data['dates'] as $date => $day_events) {
     foreach ($order as $i => $chamber) {
         if (!array_key_exists($chamber, $day_events))
             continue;
         $events = $day_events[$chamber];
-        if ($plural[$i]) $chamber .= 's';
-        print "<h2 class='calendar'>$chamber";
+        print "<h2>$chamber";
         if (in_array($major[$i], $data['majors'])) {
             $URL = new \MySociety\TheyWorkForYou\Url($hansardmajors[$major[$i]]['page_all']);
             $URL->insert( array( 'd' => $date ) );
             print ' &nbsp; <a href="' . $URL->generate() . '">See this day &rarr;</a>';
         }
         print "</h2>\n";
-        print $list[$i] ? "<ul class='calendar'>\n" : "<dl class='calendar'>\n";
+        print $list[$i] ? "<ul class='future'>\n" : "<dl class='future'>\n";
         foreach ($events as $event) {
             \MySociety\TheyWorkForYou\Utility\Calendar::displayEntry($event);
         }

www/includes/easyparliament/templates/html/index.php

@@ -144,23 +144,12 @@
                                     <h3><?= $place ?></h3>
                                     <ul class="upcoming__list">
                                         <?php for ( $i = 0; $i < 3; $i++ ) {
-                                            if ( isset( $events[$i] ) ) { ?>
+                                            if ( isset( $events[$i] ) ) {
+                                                list($event_title, $meta_items) = MySociety\TheyWorkForYou\Utility\Calendar::meta($events[$i]);
+                                        ?>
                                         <li>
-                                            <h4 class="upcoming__title"><a href="<?= $events[$i]['link_external'] ?>"><?= $events[$i]['title'] ?></a></h4>
-                                            <p class="meta"><?php
-                                                $meta_items = array();
-                                                $meta_items[] = $events[$i]['debate_type'];
-                                                if( isset($events[$i]['time_start']) && $events[$i]['time_start'] != '00:00:00' ){
-                                                    $times = format_time($events[$i]['time_start'], 'g:i a');
-
-                                                    if( isset($events[$i]['time_end']) && $events[$i]['time_end'] != '00:00:00' ){
-                                                        $times = $times . ' - ' . format_time($events[$i]['time_end'], 'g:i a');
-                                                    }
-                                                    $meta_items[] = $times;
-                                                }
-                                                // Removes "empty" items, and joins them with a semicolon
-                                                echo implode('; ', array_filter($meta_items));
-                                            ?></p>
+                                            <h4 class="upcoming__title"><a href="<?= $events[$i]['link_calendar'] ?>"><?= $event_title ?></a></h4>
+                                            <p class="meta"><?= implode('; ', array_filter($meta_items)) ?></p>
                                         </li>
                                         <?php } ?>
                                         <?php } ?>