diff --git a/NOTES.add_event_type b/NOTES.add_event_type index 12721424..48734630 100644 --- a/NOTES.add_event_type +++ b/NOTES.add_event_type @@ -21,7 +21,7 @@ export ModelClassName= cp $ska/data/kadi/events.db3 ./ export KADI=$PWD ./manage.py syncdb -./update_events --start=2000:001 --stop=2001:001 --model=${ModelClassName} [--delete-from-start] +./update_events --start=1999:200 --stop=2001:001 --model=${ModelClassName} [--delete-from-start] ./update_events --start=2001:001 --model=${ModelClassName} %% diff --git a/docs/event_descriptions.rst b/docs/event_descriptions.rst index cb9161a4..b2096c45 100644 --- a/docs/event_descriptions.rst +++ b/docs/event_descriptions.rst @@ -261,6 +261,7 @@ LTT bad intervals ======== ========== ================================ Field Type Description ======== ========== ================================ + key Char(38) Unique key for this event start Char(21) Start time (YYYY:DDD:HH:MM:SS) stop Char(21) Stop time (YYYY:DDD:HH:MM:SS) tstart Float Start time (CXC secs) diff --git a/docs/index.rst b/docs/index.rst index 7c5525c7..3836b45f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -557,8 +557,18 @@ The following example selects all Kalman mode dwells, but removes times that cou affected by a SIM TSC move or a momentum dump. >>> dwells = events.dwells(pad=-100) - >>> good_times = dwells & ~disturbances # Dwells and NOT disturbances + +You can examine the composite ``good_times`` event query and see how it is constructed of +boolean combinations of the underlying base event query objects:: + + >>> good_times + ( AND NOT (( + OR ) OR )) + +Finally you can you this composite event query to define times for selecting telemetry +of interest:: + >>> dat = fetch.Msid('aoattqt1', '2012:001', '2012:002') >>> dat.plot() >>> dat_good = dat.select_intervals(good_times, copy=True) @@ -567,6 +577,102 @@ affected by a SIM TSC move or a momentum dump. .. image:: complex_event_filter.png +Filtering the interval events +"""""""""""""""""""""""""""""" + +The examples shown above share the feature that the selected intervals were defined +using *all* of the events for a particular type. However, it is also possible to +select only a subset of the available events based on other filter criteria. For +example, if you wanted to examine telemetry during HETG insertions. This would be +a snap with the following, which defines a new event query which is the subset +of HETG insertion grating moves:: + + >>> hetg_insert = events.grating_moves(pad=50, grating='HETG', direction='INSR') + +This new event query object can be used just like the original ``events.grating_moves`` +except that now it only has HETG insertion events. +:: + + >>> events.grating_moves + + >>> hetg_insert + + +To overplot the grating angle as a function of time since the grating move start you might +do:: + + >>> intervals = hetg_insert.intervals('2010:001', '2010:030') + >>> intervals + >>> print intervals # This is a list of (start, stop) pairs + [('2010:002:15:25:39.725', '2010:002:15:28:15.013'), + ('2010:004:09:41:29.708', '2010:004:09:44:04.996'), + ... + ('2010:018:13:36:14.745', '2010:018:13:38:50.033'), + ('2010:021:04:47:00.207', '2010:021:04:49:35.494')] + >>> for start, stop in intervals: + ... dat = fetch.Msid('4hposaro', start, stop) + ... plot(dat.times - dat.times[0], dat.vals) + +LTT bad times +@@@@@@@@@@@@@@ + +Another example of practical interest is using the LTT bad times event to remove bad times +for long-term trending plots by MSID. Before explaining more about what is going on, here is +an example of filtering out LTT bad times (for the impatient):: + + >>> dat = fetch.Msid('AIRU2BT', '2011:001', '2013:001', stat='daily') + >>> dat_good = dat.remove_intervals(events.ltt_bads(msid='AIRU2BT'), copy=True) + >>> dat.plot('r', label='All') + >>> dat_good.plot(label='Good') + >>> plt.ylim(96, 103) + >>> plt.grid() + >>> plt.legend(fontsize='small') + >>> plt.title('AIRU2BT') + +.. image:: ltt_bads_airu2bt.png + +Now let's look more closely at the LTT bad times events, which are derived from a +FOT-supplied file that has the dates when particular trending items (the ``msid`` column) +are bad for some reason:: + + >>> print events.ltt_bads.filter('2000:001', '2001:001').table + start stop tstart tstop dur msid flag + --------------------- --------------------- ------------ ------------ ------- --------------- ---- + 2000:001:00:00:00.000 2000:002:00:00:00.000 63072064.184 63158464.184 86400.0 PITCH_STAB_PERF J + 2000:001:00:00:00.000 2000:002:00:00:00.000 63072064.184 63158464.184 86400.0 YAW_STAB_PERF J + 2000:004:00:00:00.000 2000:005:00:00:00.000 63331264.184 63417664.184 86400.0 GCM_TSCACC 1 + ... ... ... ... ... ... ... + 2000:358:00:00:00.000 2000:359:00:00:00.000 93916864.184 94003264.184 86400.0 3SDM15V 1 + 2000:358:00:00:00.000 2000:359:00:00:00.000 93916864.184 94003264.184 86400.0 3SDP5V 1 + 2000:366:00:00:00.000 2001:001:00:00:00.000 94608064.184 94694464.184 86400.0 3SDM15V 1 + +Some of the ``msid`` values correspond to like-named MSIDs in the engineering archive, but +many (including all those shown here) do not. You can find the non-matches with:: + + >>> print sorted(set(x.msid for x in events.ltt_bads.filter('1999:001') + if not (x.msid in fetch.content or 'DP_' + x.msid in fetch.content))) + [u'*', u'3SDAGV', u'3SDFATSV', u'3SDM15V', u'3SDP15V', u'3SDP5V', u'3SDTSTSV', u'5EHSE300', u'ABIASZ', + ... + u'ROLL_BIAS_DIFF', u'SAMYTEMDEL', u'SAPYTEMDEL', u'TFCAG', u'TFCDG', u'VECANGLE_DIFF', + u'YAW_BIAS_DIFF', u'YAW_CTRL', u'YAW_STAB', u'YAW_STAB_PERF'] + +There is a special ``msid`` value of ``'*'`` which corresponds to times that are bad for ALL +MSIDs. The bad intervals with ``msid == '*'`` are always included in query results:: + + >>> events.ltt_bads(msid='AACCCDPT').all() + + + + ... + + + + ... + + + + + Get commands ^^^^^^^^^^^^^^^^ diff --git a/docs/ltt_bads_airu2bt.png b/docs/ltt_bads_airu2bt.png new file mode 100644 index 00000000..b495955e Binary files /dev/null and b/docs/ltt_bads_airu2bt.png differ diff --git a/kadi/events/__init__.py b/kadi/events/__init__.py index 5f75f443..ed7cef91 100644 --- a/kadi/events/__init__.py +++ b/kadi/events/__init__.py @@ -16,6 +16,7 @@ fa_moves SIM FA translation FaMove grating_moves Grating movement (HETG or LETG) GratingMove load_segments Load segment from iFOT database LoadSegment + ltt_bads LTT bad intervals LttBad major_events Major event MajorEvent manvrs Maneuver Manvr manvr_seqs Maneuver sequence event ManvrSeq diff --git a/kadi/events/models.py b/kadi/events/models.py index 9f8ca518..db59d19d 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -342,7 +342,7 @@ def model_name(self): @classmethod @import_ska - def get_date_intervals(cls, start, stop, pad=None): + def get_date_intervals(cls, start, stop, pad=None, **filter_kwargs): # OPTIMIZE ME! # Initially get events within padded date range. Filter on only @@ -355,7 +355,7 @@ def get_date_intervals(cls, start, stop, pad=None): datestart = (DateTime(start) - cls.lookback).date datestop = (DateTime(stop) + cls.lookback).date - events = cls.objects.filter(start__gte=datestart, start__lte=datestop) + events = cls.objects.filter(start__gte=datestart, start__lte=datestop, **filter_kwargs) datestart = DateTime(start).date datestop = DateTime(stop).date @@ -2121,7 +2121,7 @@ def __unicode__(self): self.dur / 1000)) -class AsciiTableEvent(Event): +class AsciiTableEvent(BaseEvent): """ Base class for events defined by a simple quasi-static text table file. Subclasses need to define the file name (lives in DATA_DIR()/) @@ -2192,6 +2192,7 @@ class LttBad(AsciiTableEvent): ======== ========== ================================ Field Type Description ======== ========== ================================ + key Char(38) Unique key for this event start Char(21) Start time (YYYY:DDD:HH:MM:SS) stop Char(21) Stop time (YYYY:DDD:HH:MM:SS) tstart Float Start time (CXC secs) @@ -2201,9 +2202,19 @@ class LttBad(AsciiTableEvent): flag Char(2) Flag ======== ========== ================================ """ + key = models.CharField(max_length=38, primary_key=True, + help_text='Unique key for this event') + start = models.CharField(max_length=21, help_text='Start time (YYYY:DDD:HH:MM:SS)') + stop = models.CharField(max_length=21, help_text='Stop time (YYYY:DDD:HH:MM:SS)') + tstart = models.FloatField(db_index=True, help_text='Start time (CXC secs)') + tstop = models.FloatField(help_text='Stop time (CXC secs)') + dur = models.FloatField(help_text='Duration (secs)') msid = models.CharField(max_length=20, help_text='MSID') flag = models.CharField(max_length=2, help_text='Flag') + key._kadi_hidden = True + dur._kadi_format = '{:.1f}' + intervals_file = 'ltt_bads.dat' # Table.read keyword args table_read_kwargs = dict(format='ascii', data_start=2, delimiter='|', guess=False, @@ -2215,12 +2226,14 @@ class LttBad(AsciiTableEvent): def process_intervals(cls, intervals): intervals['start'] = DateTime(intervals['tstart']).date intervals['stop'] = (DateTime(intervals['tstart']) + 1).date + intervals.sort('start') @classmethod def get_extras(cls, event, interval): out = {} for key in ('msid', 'flag'): out[key] = interval[key].tolist() + out['key'] = event['start'][:17] + out['msid'] return out def __unicode__(self): diff --git a/kadi/events/query.py b/kadi/events/query.py index dd6e2460..69ca8e80 100644 --- a/kadi/events/query.py +++ b/kadi/events/query.py @@ -127,6 +127,11 @@ class EventQuery(object): - filter() : filter events matching criteria and return Django query set - intervals(): return time intervals between event start/stop times + An EventQuery object can be pre-filtered via any of the expressions + described in the ``filter()`` doc string. In this way the corresponding + ``intervals()`` and fetch ``remove_intervals`` / ``select_intervals`` + outputs can be likewise filtered. + A key feature is that EventQuery objects can be combined with boolean and, or, and not logic to generate composite EventQuery objects. From there the intervals() output can be used to select or remove the intervals @@ -135,18 +140,41 @@ class EventQuery(object): interval_pad = IntervalPad() # descriptor defining a Pad for intervals - def __init__(self, cls=None, left=None, right=None, op=None, pad=None): + def __init__(self, cls=None, left=None, right=None, op=None, pad=None, **filter_kwargs): self.cls = cls self.left = left self.right = right self.op = op self.interval_pad = pad + self.filter_kwargs = filter_kwargs - def __call__(self, pad=None): + def __repr__(self): + if self.cls is None: + op_name = {'and_': 'AND', + 'or_': 'OR'}.get(self.op.__name__, 'UNKNOWN_OP') + if self.right is None: + # This assumes any unary operator is ~. FIX ME! + return 'NOT {}'.format(self.left) + else: + return '({} {} {})'.format(self.left, op_name, self.right) + else: + bits = ['') + return ''.join(bits) + + def __call__(self, pad=None, **filter_kwargs): """ Generate new EventQuery event for the same model class but with different pad. """ - return EventQuery(cls=self.cls, pad=pad) + return EventQuery(cls=self.cls, pad=pad, **filter_kwargs) @property def name(self): @@ -172,7 +200,8 @@ def intervals(self, start, stop): intervals1 = self.right.intervals(start, stop) return combine_intervals(self.op, intervals0, intervals1, start, stop) else: - date_intervals = self.cls.get_date_intervals(start, stop, self.interval_pad) + date_intervals = self.cls.get_date_intervals(start, stop, self.interval_pad, + **self.filter_kwargs) return date_intervals @property @@ -224,6 +253,11 @@ def filter(self, start=None, stop=None, obsid=None, subset=None, **kwargs): cls = self.cls objs = cls.objects.all() + # Start from self.filter_kwargs as the default and update with kwargs + new_kwargs = self.filter_kwargs.copy() + new_kwargs.update(kwargs) + kwargs = new_kwargs + if obsid is not None: if start or stop: raise ValueError('Cannot set both obsid and start or stop') @@ -270,29 +304,41 @@ def all(self): >>> from kadi import events >>> print events.safe_suns.all() - - - - - + + + + + >>> print events.safe_suns.all().table - start stop tstart tstop dur notes - --------------------- --------------------- ------------- ------------- ------------- ----- - 1999:229:20:18:22.688 1999:231:01:29:05.885 51308366.8723 51413410.069 105043.196657 - 1999:250:16:31:46.461 1999:270:08:10:11.850 53109170.6451 54807076.0338 1697905.38868 - 2000:048:08:09:30.216 2000:049:03:14:19.260 67162234.4001 67230923.4436 68689.0435828 - 2011:187:12:29:22.579 2011:190:20:37:38.914 426342628.763 426631125.098 288496.334723 - 2012:150:03:33:45.816 2012:151:12:30:03.213 454649692.0 454768269.397 118577.396626 + start stop tstart tstop dur notes + --------------------- --------------------- ----------- ----------- -------- ----- + 1999:229:20:17:50.616 1999:231:01:29:21.816 51308334.8 51413426.0 105091.2 + 1999:269:20:22:50.616 1999:270:08:22:15.416 54764634.8 54807799.6 43164.8 + 2000:048:08:08:54.216 2000:049:03:15:32.216 67162198.4 67230996.4 68798.0 + 2011:187:12:28:53.816 2011:190:20:39:17.416 426342600.0 426631223.6 288623.6 + 2012:150:03:33:09.816 2012:151:12:31:49.416 454649656.0 454768375.6 118719.6 """ return self.filter() +class LttBadEventQuery(EventQuery): + def __call__(self, pad=None, **filter_kwargs): + """ + Generate new EventQuery event for the same model class but with different pad. + """ + if 'msid' in filter_kwargs: + filter_kwargs['msid__in'] = ['*', filter_kwargs.pop('msid')] + + return EventQuery(cls=self.cls, pad=pad, **filter_kwargs) + + # Put EventQuery objects for each query-able model class into module globals event_models = models.get_event_models() for model_name, model_class in event_models.items(): query_name = model_name + 's' # simple pluralization - query_instance = EventQuery(cls=model_class) + event_query_class = LttBadEventQuery if model_name == 'ltt_bad' else EventQuery + query_instance = event_query_class(cls=model_class) query_instance.__doc__ = model_class.__doc__ globals()[query_name] = query_instance __all__.append(query_name) diff --git a/kadi/tests/test_events.py b/kadi/tests/test_events.py index d0373c1a..c52a3433 100644 --- a/kadi/tests/test_events.py +++ b/kadi/tests/test_events.py @@ -108,3 +108,30 @@ def test_get_obsid(): assert len(query_events) >= 1 for query_event in query_events: assert query_event.get_obsid() == obsid + + +def test_intervals_filter(): + """ + Test setting filter keywords in the EventQuery object itself. + """ + ltt_bads = events.ltt_bads + start, stop = '2011:295', '2011:305' + + assert (str(ltt_bads().filter('2011:298', '2011:305')).splitlines() == + ['', + '', + '', + '']) + + # No filter + assert (ltt_bads.intervals(start, stop) == + [('2011:299:04:58:39.000', '2011:303:00:00:00.000')]) + + assert (ltt_bads(flag='M').intervals(start, stop) == + [('2011:299:04:58:39.000', '2011:300:04:58:39.000')]) + + assert (ltt_bads(msid='AACCCDPT', flag='A').intervals(start, stop) == + [('2011:300:00:00:00.000', '2011:303:00:00:00.000')]) + + assert (ltt_bads(msid='AACCCDPT', flag='A', start__gt='2011:301:12').intervals(start, stop) == + [('2011:302:00:00:00.000', '2011:303:00:00:00.000')]) diff --git a/kadi/version.py b/kadi/version.py index 74e683eb..0ad0599b 100644 --- a/kadi/version.py +++ b/kadi/version.py @@ -33,7 +33,7 @@ ### SET THESE VALUES ############################ # Major, Minor, Bugfix, Dev -VERSION = (0, 9, None, True) +VERSION = (0, 9, None, False) class SemanticVersion(object):