Skip to content

Commit

Permalink
Add mmsi-transit finding code
Browse files Browse the repository at this point in the history 
Now the program outputs mmsi_transit values on positional messages,
which identify sequences of continuous movement by individual
vessels. See src/aisutil/transits.d for a spec.
  • Loading branch information
@jamtho
jamtho committed Nov 14, 2018
1 parent a2210ee commit 1649a9e
Show file tree
Hide file tree
Showing 6 changed files with 573 additions and 75 deletions.
38 changes: 38 additions & 0 deletions src/aisutil/ais.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,44 @@ struct AnyAisMsgPossTS {
}


// ----------------------------------------------------------------------
// Utils for investigating whether an AnyAisMsg has a speed val, and getting it

bool hasSpeed (AnyAisMsg msg) {
return msg.visit!(
(AisMsg1n2n3 m) => true,
(AisMsg5 m) => false,
(AisMsg18 m) => true,
(AisMsg19 m) => true,
(AisMsg24 m) => false,
(AisMsg27 m) => true,
);
}

double speed (AnyAisMsg msg) {
import std.exception, core.exception;
assert (msg.hasSpeed);
return msg.visit!(
(AisMsg1n2n3 m) => m.speed,
(AisMsg5 m) {
enforce!AssertError (false, "AisMsg5 does not have speed field");
return -1; }, // placate compiler
(AisMsg18 m) => m.speed,
(AisMsg19 m) => m.speed,
(AisMsg24 m) {
enforce!AssertError (false, "AisMsg24 does not have speed field");
return -1; }, // placate compiler
(AisMsg27 m) => m.speed,
);
}

unittest {
auto msg = AnyAisMsg (AisMsg1n2n3 ("177KQJ5000G?tO`K>RA1wUbN0TKH", 0));
assert (msg.hasSpeed ());
assert (msg.speed == 0.0);
}


// --------------------------------------------------------------------------
// Parser for AnyAisMsg (if you want it)

Expand Down
44 changes: 27 additions & 17 deletions src/aisutil/csv.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
module aisutil.csv;
import std.string, std.range, std.algorithm, std.conv, std.typecons, std.variant;
import aisutil.ext.libaiswrap, aisutil.ais, aisutil.dlibaiswrap,
aisutil.geotracks;
aisutil.geotracks, aisutil.transits, aisutil.filewriting;


// Functions to generate CSV strings, particularly for AIS messages
Expand Down Expand Up @@ -76,8 +76,9 @@ unittest {

private immutable string[] ignoredFields = ["parse_error", "turn_valid"];

private immutable string[] aisCsvCols = ["tagblock_timestamp"] ~
"mmsi_geotrack"
private immutable string[] aisCsvCols = ["tagblock_timestamp",
"mmsi_geotrack",
"mmsi_transit"]
~
( [__traits(allMembers, C_AisMsg1n2n3)]
~ [__traits(allMembers, C_AisMsg5)]
Expand Down Expand Up @@ -105,26 +106,26 @@ string csvHeader() {
// -- AnyAisMsg wrapper

string toCsvRow (in AnyAisMsg msg, Nullable!int tagblockTimestamp,
Nullable!GeoTrackID gtid) {
SubtrackData stData) {
return msg.visit!(
(in ref AisMsg1n2n3 m) => toCsvRow (m, tagblockTimestamp, gtid),
(in ref AisMsg5 m) => toCsvRow (m, tagblockTimestamp, gtid),
(in ref AisMsg18 m) => toCsvRow (m, tagblockTimestamp, gtid),
(in ref AisMsg19 m) => toCsvRow (m, tagblockTimestamp, gtid),
(in ref AisMsg24 m) => toCsvRow (m, tagblockTimestamp, gtid),
(in ref AisMsg27 m) => toCsvRow (m, tagblockTimestamp, gtid)
(in ref AisMsg1n2n3 m) => toCsvRow (m, tagblockTimestamp, stData),
(in ref AisMsg5 m) => toCsvRow (m, tagblockTimestamp, stData),
(in ref AisMsg18 m) => toCsvRow (m, tagblockTimestamp, stData),
(in ref AisMsg19 m) => toCsvRow (m, tagblockTimestamp, stData),
(in ref AisMsg24 m) => toCsvRow (m, tagblockTimestamp, stData),
(in ref AisMsg27 m) => toCsvRow (m, tagblockTimestamp, stData)
);
}

// -- Distinct message type handlers

// This is just used in unit tests
private string toCsvRow(T)(in T obj) if(isAisMsg!T) {
return toCsvRow!T(obj, Nullable!int.init, Nullable!GeoTrackID.init);
return toCsvRow!T(obj, Nullable!int.init, SubtrackData ());
}

string toCsvRow(T)(in T obj, Nullable!int tagblockTimestamp,
Nullable!GeoTrackID gtid)
SubtrackData stData)
if(isAisMsg!T)
{
import std.conv;
Expand All @@ -147,10 +148,18 @@ string toCsvRow(T)(in T obj, Nullable!int tagblockTimestamp,
} else
// as is geotrack
static if (cn == "mmsi_geotrack") {
if (gtid.isNull) {
if (stData.geoTrackID.isNull) {
// pass
} else {
res ~= gtid.get().value.csvValStr();
res ~= stData.geoTrackID.value.csvValStr();
}
} else
// as is transit
static if (cn == "mmsi_transit") {
if (stData.transitID.isNull) {
// pass
} else {
res ~= stData.transitID.value.csvValStr();
}
} else
// Every other present field is handled the same way
Expand Down Expand Up @@ -183,7 +192,6 @@ unittest {

// Since we know the fields in type 24 B...
auto nonEmptyCells = cells.filter !(c => c.length > 0) .array;
//assert (nonEmptyCells.length == 6);
assert (nonEmptyCells.length == 5);

// Get the member of 'cells' matching col header named 'colName'
Expand Down Expand Up @@ -233,12 +241,13 @@ unittest {
assert (colVal("turn") == "0");
}

// Check timestamp and gtid
// Check timestamp, gtid and transitid
{
auto msg = AisMsg1n2n3("177KQJ5000G?tO`K>RA1wUbN0TKH", 0);

auto csvRow = toCsvRow (msg, Nullable!int(123),
Nullable!GeoTrackID(GeoTrackID(999)));
SubtrackData (nullable (GeoTrackID (999)),
nullable (TransitID (555))));

string[] cells = csvRow.split(",");
assert (cells.length == aisCsvCols.length);
Expand All @@ -250,6 +259,7 @@ unittest {

assert (colVal("tagblock_timestamp") == "123");
assert (colVal("mmsi_geotrack") == "999");
assert (colVal("mmsi_transit") == "555");
}

}
21 changes: 15 additions & 6 deletions src/aisutil/decodeprocess.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ import aisutil.filewriting, aisutil.mmsistats, aisutil.ais,
aisutil.decodeprocessdef, aisutil.geo, aisutil.decprocfinstats,
aisutil.geoheatmap, aisutil.dlibaiswrap, aisutil.simpleshiptypes,
aisutil.shiplengths, aisutil.daisnmea, aisutil.backlog,
aisutil.aisnmeagrouping, aisutil.filereading, aisutil.geotracks;
aisutil.aisnmeagrouping, aisutil.filereading, aisutil.geotracks,
aisutil.transits;


// ==========================================================================
Expand Down Expand Up @@ -296,6 +297,7 @@ DecProcFinStats executeDecodeProcess (DecodeProcessDef procDef,
auto mmsiStats = MmsiStatsBucket ();
auto backlog = MmsiBacklog ();
auto geoTracker = GeoTrackFinder ();
auto transFinder = TransitFinder ();
immutable totalBytesInInput = procDef.totalBytesInInput();

// Choose the variant-specific message file writer
Expand Down Expand Up @@ -351,17 +353,24 @@ DecProcFinStats executeDecodeProcess (DecodeProcessDef procDef,
// All msg->file writing goes through these
auto emitMessage_h = delegate void (AnyAisMsgPossTS msg,
Nullable!MmsiStats stats) {
Nullable!GeoTrackID possGtid;
if (msg.msg.isPositional)
possGtid = geoTracker.put (msg.msg, msg.possTS);
SubtrackData stData;
if (! msg.possTS.isNull()) {
if (msg.msg.isPositional)
stData.geoTrackID = geoTracker.put (msg.msg, msg.possTS);
if (msg.msg.hasSpeed) {
auto res = transFinder.put (msg.msg, msg.possTS);
if (res.isInTransit)
stData.transitID = res.transitID;
}
}

geoHeatmap.markLatLon_ifPositional (msg.msg);
statsBuilder.notifyParsedMsgWritten ();

if (stats.isNull)
msgWriter.writeMsg_noStats (msg.msg, msg.possTS, possGtid);
msgWriter.writeMsg_noStats (msg.msg, msg.possTS, stData);
else
msgWriter.writeMsg (msg.msg, msg.possTS, possGtid, stats);
msgWriter.writeMsg (msg.msg, msg.possTS, stData, stats);
};
auto emitMessage_noStats = delegate void (AnyAisMsgPossTS msg) {
emitMessage_h (msg, Nullable!MmsiStats.init);
Expand Down
73 changes: 42 additions & 31 deletions src/aisutil/filewriting.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
module aisutil.filewriting;
import aisutil.mmsistats, aisutil.csv, aisutil.json, aisutil.geo,
aisutil.dlibaiswrap, aisutil.simpleshiptypes, aisutil.shiplengths,
aisutil.ais, aisutil.geotracks;
aisutil.ais, aisutil.geotracks, aisutil.transits;
import std.stdio, std.range, std.algorithm, std.typecons;


Expand Down Expand Up @@ -44,6 +44,16 @@ enum MessageOutputFormat {
}


// ----------------------------------------------------------------------
// Each message may or may not be part of an MMSI geotrack and/or a transit,
// which together make up it's 'subtrack membership'

struct SubtrackData {
Nullable!GeoTrackID geoTrackID;
Nullable!TransitID transitID;
}


// --------------------------------------------------------------------------
// Clients pass in a lot of possibly-present timestamps and GeoTrackIDs
// (and often deal with them themselves)
Expand All @@ -56,9 +66,9 @@ alias PossGeoTrackID = Nullable!GeoTrackID;
// Trivial json making

private string msgJsonStr (in ref AnyAisMsg msg, PossTimestamp possTS,
PossGeoTrackID possGtid) {
SubtrackData stData) {
import std.json;
auto jsval = toJsonVal (msg, possTS, possGtid);
auto jsval = toJsonVal (msg, possTS, stData);
return jsval.toJSON();
}

Expand All @@ -72,11 +82,11 @@ interface MsgFileWriter {

// Write to file - normal control path
void writeMsg (in ref AnyAisMsg msg, PossTimestamp possTS,
PossGeoTrackID possGtid, in ref MmsiStats mmstats);
SubtrackData stData, in ref MmsiStats mmstats);

// When we really don't have useful mmsi stats, write the message somehow
void writeMsg_noStats (in ref AnyAisMsg msg, PossTimestamp possTS,
PossGeoTrackID possGtid);
SubtrackData stData);

// Close all open files. MUST be called in class dtr
void close();
Expand Down Expand Up @@ -113,20 +123,20 @@ class SimpleMsgFileWriter : MsgFileWriter {
override bool canWriteWithStats (in ref MmsiStats mmstats) {return true;}

override void writeMsg (in ref AnyAisMsg msg, PossTimestamp possTS,
PossGeoTrackID possGtid,
SubtrackData stData,
in ref MmsiStats stats) {
assert (canWriteWithStats (stats));
writeMsg_noStats (msg, possTS, possGtid);
writeMsg_noStats (msg, possTS, stData);
}

override void writeMsg_noStats (in ref AnyAisMsg msg, PossTimestamp possTS,
PossGeoTrackID possGtid) {
SubtrackData stData) {
final switch (_format) {
case MessageOutputFormat.CSV:
_file.writeln (toCsvRow (msg, possTS, possGtid));
_file.writeln (toCsvRow (msg, possTS, stData));
break;
case MessageOutputFormat.NDJSON:
auto jsval = aisutil.json.toJsonVal (msg, possTS, possGtid);
auto jsval = aisutil.json.toJsonVal (msg, possTS, stData);
import std.json;
_file.writeln (jsval.toJSON());
break;
Expand Down Expand Up @@ -184,28 +194,28 @@ class MsgFileWriter_SplittingSimpleShiptypes : MsgFileWriter {
}

override void writeMsg (in ref AnyAisMsg msg, PossTimestamp possTS,
PossGeoTrackID possGtid,
SubtrackData stData,
in ref MmsiStats stats) {
assert (canWriteWithStats (stats));
auto sst = simplifyShiptype (stats.shiptype);
auto file = _files [sst];
doWriteMsg (file, msg, possTS, possGtid);
doWriteMsg (file, msg, possTS, stData);
}

override void writeMsg_noStats (in ref AnyAisMsg msg, PossTimestamp possTS,
PossGeoTrackID possGtid) {
SubtrackData stData) {
auto file = _files [SimpleShiptype.NotBroadcast];
doWriteMsg (file, msg, possTS, possGtid);
doWriteMsg (file, msg, possTS, stData);
}

private void doWriteMsg (File file, in ref AnyAisMsg msg,
PossTimestamp possTS, PossGeoTrackID possGtid) {
PossTimestamp possTS, SubtrackData stData) {
final switch (_format) {
case MessageOutputFormat.CSV:
file.writeln (toCsvRow (msg, possTS, possGtid));
file.writeln (toCsvRow (msg, possTS, stData));
break;
case MessageOutputFormat.NDJSON:
auto jsval = aisutil.json.toJsonVal (msg, possTS, possGtid);
auto jsval = aisutil.json.toJsonVal (msg, possTS, stData);
import std.json;
file.writeln (jsval.toJSON());
break;
Expand Down Expand Up @@ -252,27 +262,28 @@ class MsgFileWriter_SplitShipLenCat : MsgFileWriter {
return stats.hasShiplen; }

override void writeMsg (in ref AnyAisMsg msg, PossTimestamp possTS,
PossGeoTrackID possGtid, in ref MmsiStats stats) {
SubtrackData stData, in ref MmsiStats stats) {
assert (canWriteWithStats (stats));
auto lenCat = stats.shiplen.shipLenCatForLen ();
auto file = _files [lenCat];
doWriteMsg (file, msg, possTS, possGtid);
doWriteMsg (file, msg, possTS, stData);
}

override void writeMsg_noStats (in ref AnyAisMsg msg, PossTimestamp possTS,
PossGeoTrackID possGtid) {
SubtrackData stData) {
//PossGeoTrackID possGtid) {
auto file = _files [ShipLenCat.NotBroadcast];
doWriteMsg (file, msg, possTS, possGtid);
doWriteMsg (file, msg, possTS, stData);
}

void doWriteMsg (File file, in ref AnyAisMsg msg,
PossTimestamp possTS, PossGeoTrackID possGtid) {
PossTimestamp possTS, SubtrackData stData) {
final switch (_format) {
case MessageOutputFormat.CSV:
file.writeln (toCsvRow (msg, possTS, possGtid));
file.writeln (toCsvRow (msg, possTS, stData));
break;
case MessageOutputFormat.NDJSON:
file.writeln (msgJsonStr (msg, possTS, possGtid));
file.writeln (msgJsonStr (msg, possTS, stData));
break;
}
}
Expand Down Expand Up @@ -353,30 +364,30 @@ class MsgFileWriter_SplitTimestampDay : MsgFileWriter {
override bool canWriteWithStats (in ref MmsiStats stats) const {return true;}

override void writeMsg (in ref AnyAisMsg msg, PossTimestamp possTS,
PossGeoTrackID possGtid, in ref MmsiStats stats) {
SubtrackData stData, in ref MmsiStats stats) {
assert (canWriteWithStats (stats));
writeMsg_noStats (msg, possTS, possGtid);
writeMsg_noStats (msg, possTS, stData);
}

override void writeMsg_noStats (in ref AnyAisMsg msg, PossTimestamp possTS,
PossGeoTrackID possGtid) {
SubtrackData stData) {
if (possTS.isNull)
openFileWithDay (Nullable!DayID.init);
else
openFileWithDay (Nullable!DayID (DayID.forTimestamp (possTS)));

writeMsg_intoCurFile (msg, possTS, possGtid);
writeMsg_intoCurFile (msg, possTS, stData);
}

private void writeMsg_intoCurFile (in ref AnyAisMsg msg,
PossTimestamp possTS,
PossGeoTrackID possGtid) {
SubtrackData stData) {
final switch (_format) with (MessageOutputFormat) {
case CSV:
_curFile.writeln (toCsvRow (msg, possTS, possGtid));
_curFile.writeln (toCsvRow (msg, possTS, stData));
break;
case NDJSON:
_curFile.writeln (msgJsonStr (msg, possTS, possGtid));
_curFile.writeln (msgJsonStr (msg, possTS, stData));
break;
}
}
Expand Down
Loading

0 comments on commit 1649a9e

Please sign in to comment.