From ce8684272ea9505b3b64622931b75830f2fdfd40 Mon Sep 17 00:00:00 2001 From: Jannick Heisch Date: Fri, 8 Dec 2023 11:58:49 +0100 Subject: [PATCH 1/6] Add in-order delivery api for frontend --- .../app/src/main/assets/web/tremola.js | 18 +++++++++ .../tremolavossbol/WebAppInterface.kt | 37 +++++++++++++------ 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/android/tinySSB/app/src/main/assets/web/tremola.js b/android/tinySSB/app/src/main/assets/web/tremola.js index 89d229c..bc30613 100644 --- a/android/tinySSB/app/src/main/assets/web/tremola.js +++ b/android/tinySSB/app/src/main/assets/web/tremola.js @@ -979,6 +979,24 @@ function b2f_local_peer(type, identifier, displayname, status) { refresh_connection_entry(identifier) } +/** + * This function is called, when the backend received a new log entry and successfully completed the corresponding sidechain. + * The backend assures, that the log entries are sent to the frontend in the same sequential order as in the append-only log. + * + * @param {Object} e Object containing all information of the log_entry. + * @param {Object} e.hdr Contains basic information about the log entry.{tst: 1700234500672, ref: 417869, fid: '@AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=.ed25519'} + * @param {number} e.hdr.tst Timestamp at which the message was created. (Number of milliseconds elapsed since midnight at the beginning of January 1970 00:00 UTC) + * @param {string} e.hdr.ref The message ID of this log entry. + * @param {string} e.hdr.fid The public key of the author encoded in base64. + * @param {[]} e.public The payload of the message + * + */ +function b2f_new_in_order_event(e) { +} + +function b2f_new_incomplete_event(e) { +} + function b2f_new_event(e) { // incoming SSB log event: we get map with three entries // console.log('hdr', JSON.stringify(e.header)) console.log('pub', JSON.stringify(e.public)) diff --git a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt index 4c67476..20ea60b 100644 --- a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt +++ b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt @@ -3,6 +3,7 @@ package nz.scuttlebutt.tremolavossbol import android.Manifest import android.content.ClipData import android.content.ClipboardManager +import android.content.Context import android.content.Intent import android.content.pm.PackageManager import android.util.Base64 @@ -30,6 +31,7 @@ import org.json.JSONArray class WebAppInterface(val act: MainActivity, val webView: WebView) { var frontend_ready = false + private val frontend_frontier = act.getSharedPreferences("frontend_frontier", Context.MODE_PRIVATE) @JavascriptInterface fun onFrontendRequest(s: String) { @@ -63,7 +65,7 @@ class WebAppInterface(val act: MainActivity, val webView: WebView) { val mid = r.get_mid(i) if (payload == null || mid == null) break Log.d("restream", "${i}, ${payload.size} Bytes") - sendToFrontend(fid, i, mid, payload) + sendTinyEventToFrontend(fid, i, mid, payload) i++ } } @@ -352,26 +354,39 @@ class WebAppInterface(val act: MainActivity, val webView: WebView) { fun sendTinyEventToFrontend(fid: ByteArray, seq: Int, mid:ByteArray, body: ByteArray) { Log.d("wai","sendTinyEvent ${body.toHex()}") - sendToFrontend(fid, seq, mid, body) + var e = toFrontendObject(fid, seq, mid, body) + if (e != null) + eval("b2f_new_event($e)") + + // in-order api + val replica = act.tinyRepo.fid2replica(fid) + + if (frontend_frontier.getInt(fid.toHex(), 0) == seq && replica != null) { + for (i in seq .. replica.state.max_seq ) { + val content = replica.read(i) + val message_id= replica.get_mid(seq) + if(content == null || message_id == null) + break + e = toFrontendObject(fid, i, message_id, content) + if (e != null) + eval("b2f_new_in_order_event($e)") + frontend_frontier.edit().putInt(fid.toHex(), i + 1).apply() + } + } } - fun sendToFrontend(fid: ByteArray, seq: Int, mid: ByteArray, payload: ByteArray) { - Log.d("wai", "sendToFrontend seq=${seq} ${payload.toHex()}") + fun toFrontendObject(fid: ByteArray, seq: Int, mid: ByteArray, payload: ByteArray): String? { val bodyList = Bipf.decode(payload) if (bodyList == null || bodyList.typ != BIPF_LIST) { - Log.d("sendToFrontend", "decoded payload == null") - return + Log.d("toFrontendObject", "decoded payload == null") + return null } val param = Bipf.bipf_list2JSON(bodyList) var hdr = JSONObject() hdr.put("fid", "@" + fid.toBase64() + ".ed25519") hdr.put("ref", mid.toBase64()) hdr.put("seq", seq) - var cmd = "b2f_new_event({header:${hdr.toString()}," - cmd += "public:${param.toString()}" - cmd += "});" - Log.d("CMD", cmd) - eval(cmd) + return "{header:${hdr.toString()}, public:${param.toString()}}" } From c7c23215223c7b7d0c702ed598062f106bc1bb1b Mon Sep 17 00:00:00 2001 From: Jannick Heisch Date: Fri, 8 Dec 2023 12:26:57 +0100 Subject: [PATCH 2/6] Add incomplete log entry delivery to frontend --- .../nz/scuttlebutt/tremolavossbol/WebAppInterface.kt | 9 ++++++++- .../java/nz/scuttlebutt/tremolavossbol/tssb/Replica.kt | 5 +++++ .../main/java/nz/scuttlebutt/tremolavossbol/tssb/Repo.kt | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt index 20ea60b..9868bea 100644 --- a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt +++ b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt @@ -31,7 +31,7 @@ import org.json.JSONArray class WebAppInterface(val act: MainActivity, val webView: WebView) { var frontend_ready = false - private val frontend_frontier = act.getSharedPreferences("frontend_frontier", Context.MODE_PRIVATE) + val frontend_frontier = act.getSharedPreferences("frontend_frontier", Context.MODE_PRIVATE) @JavascriptInterface fun onFrontendRequest(s: String) { @@ -352,6 +352,13 @@ class WebAppInterface(val act: MainActivity, val webView: WebView) { eval(cmd) } + fun sendIncompleteEntryToFrontend(fid: ByteArray, seq: Int, mid:ByteArray, body: ByteArray) { + val e = toFrontendObject(fid, seq, mid, body) + if (e != null) + eval("b2f_new_incomplete_event($e)") + + } + fun sendTinyEventToFrontend(fid: ByteArray, seq: Int, mid:ByteArray, body: ByteArray) { Log.d("wai","sendTinyEvent ${body.toHex()}") var e = toFrontendObject(fid, seq, mid, body) diff --git a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Replica.kt b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Replica.kt index ee7c01a..2fa0800 100644 --- a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Replica.kt +++ b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Replica.kt @@ -185,6 +185,11 @@ class Replica(val context: MainActivity, val datapath: File, val fid: ByteArray) val fct = { buf: ByteArray, fid: ByteArray?, _: String? -> context.tinyNode.incoming_pkt(buf,fid!!) } context.tinyDemux.arm_dmx(new_dmx, fct, fid) + val (_, sz) = Bipf.varint_decode(pkt, DMX_LEN + 1, DMX_LEN + 4) + val content = pkt.sliceArray(8 + sz until 36) + + context.wai.sendIncompleteEntryToFrontend(fid, seq, (nam + pkt).sha256().sliceArray(0 until HASH_LEN), content) + if(sendToFront) context.wai.sendTinyEventToFrontend(fid, seq, (nam + pkt).sha256().sliceArray(0 until HASH_LEN), read_content(seq)!!) return true diff --git a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Repo.kt b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Repo.kt index 6d7ec91..aa78cae 100644 --- a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Repo.kt +++ b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Repo.kt @@ -303,6 +303,7 @@ class Repo(val context: MainActivity) { frontier.delete() frontier.createNewFile() frontier.appendBytes(new_state.toWire()) + context.wai.frontend_frontier.edit().putInt(f.name, new_state.max_pos + 1).apply() Files.move(new_log.toPath(), File(feed_dir, "log.bin").toPath(), StandardCopyOption.ATOMIC_MOVE) File(feed_dir, "log").delete() From fa68496182de578e1224e6cbda762a7d3fd1a79f Mon Sep 17 00:00:00 2001 From: Jannick Heisch Date: Fri, 8 Dec 2023 13:27:59 +0100 Subject: [PATCH 3/6] Add documentation --- .../app/src/main/assets/web/tremola.js | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/android/tinySSB/app/src/main/assets/web/tremola.js b/android/tinySSB/app/src/main/assets/web/tremola.js index bc30613..b02952c 100644 --- a/android/tinySSB/app/src/main/assets/web/tremola.js +++ b/android/tinySSB/app/src/main/assets/web/tremola.js @@ -981,22 +981,46 @@ function b2f_local_peer(type, identifier, displayname, status) { /** * This function is called, when the backend received a new log entry and successfully completed the corresponding sidechain. - * The backend assures, that the log entries are sent to the frontend in the same sequential order as in the append-only log. + * The backend assures, that the log entries are sent to the frontend in the exact same sequential order as in the append-only log. * * @param {Object} e Object containing all information of the log_entry. - * @param {Object} e.hdr Contains basic information about the log entry.{tst: 1700234500672, ref: 417869, fid: '@AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=.ed25519'} + * @param {Object} e.hdr Contains basic information about the log entry. * @param {number} e.hdr.tst Timestamp at which the message was created. (Number of milliseconds elapsed since midnight at the beginning of January 1970 00:00 UTC) * @param {string} e.hdr.ref The message ID of this log entry. * @param {string} e.hdr.fid The public key of the author encoded in base64. * @param {[]} e.public The payload of the message * */ + function b2f_new_in_order_event(e) { } +/** + * This function is invoked whenever the backend receives a new log entry, regardless of whether the associated sidechain is fully loaded or not. + * + * @param {Object} e Object containing all information of the log_entry. + * @param {Object} e.hdr Contains basic information about the log entry. + * @param {number} e.hdr.tst Timestamp at which the message was created. (Number of milliseconds elapsed since midnight at the beginning of January 1970 00:00 UTC) + * @param {string} e.hdr.ref The message ID of this log entry. + * @param {string} e.hdr.fid The public key of the author encoded in base64. + * @param {[]} e.public The payload of the logentry, without the content of the sidechain + * + */ function b2f_new_incomplete_event(e) { } +/** + * This function is called, when the backend received a new log entry and successfully completed the corresponding sidechain. + * This callback does not ensure any specific order; the log entries are forwarded in the order they are received. + * + * @param {Object} e Object containing all information of the log_entry. + * @param {Object} e.hdr Contains basic information about the log entry. + * @param {number} e.hdr.tst Timestamp at which the message was created. (Number of milliseconds elapsed since midnight at the beginning of January 1970 00:00 UTC) + * @param {string} e.hdr.ref The message ID of this log entry. + * @param {string} e.hdr.fid The public key of the author encoded in base64. + * @param {[]} e.public The payload of the message + * + */ function b2f_new_event(e) { // incoming SSB log event: we get map with three entries // console.log('hdr', JSON.stringify(e.header)) console.log('pub', JSON.stringify(e.public)) From ec0765a2d8420ed32e035995a4c8b6d3e4168b92 Mon Sep 17 00:00:00 2001 From: Jannick Heisch Date: Fri, 8 Dec 2023 14:10:12 +0100 Subject: [PATCH 4/6] Move the kanban board event handling logic to the in-order delivery callback --- .../app/src/main/assets/web/tremola.js | 49 +++++++++++++++++-- .../tremolavossbol/WebAppInterface.kt | 2 +- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/android/tinySSB/app/src/main/assets/web/tremola.js b/android/tinySSB/app/src/main/assets/web/tremola.js index b02952c..a196e1b 100644 --- a/android/tinySSB/app/src/main/assets/web/tremola.js +++ b/android/tinySSB/app/src/main/assets/web/tremola.js @@ -988,11 +988,33 @@ function b2f_local_peer(type, identifier, displayname, status) { * @param {number} e.hdr.tst Timestamp at which the message was created. (Number of milliseconds elapsed since midnight at the beginning of January 1970 00:00 UTC) * @param {string} e.hdr.ref The message ID of this log entry. * @param {string} e.hdr.fid The public key of the author encoded in base64. - * @param {[]} e.public The payload of the message + * @param {[]} e.public The payload of the message. The first entry is a String that represents the application to which the message belongs. All additional entries are application-specific parameters. * */ - function b2f_new_in_order_event(e) { + + console.log("b2f inorder event") + + if (!(e.header.fid in tremola.contacts)) { + var a = id2b32(e.header.fid); + tremola.contacts[e.header.fid] = { + "alias": a, "initial": a.substring(0, 1).toUpperCase(), + "color": colors[Math.floor(colors.length * Math.random())], + "iam": "", "forgotten": false + } + load_contact_list() + } + + switch (e.public[0]) { + case "KAN": + console.log("New kanban event") + kanban_new_event(e) + break + default: + return + } + persist(); + must_redraw = true; } /** @@ -1007,6 +1029,25 @@ function b2f_new_in_order_event(e) { * */ function b2f_new_incomplete_event(e) { + + if (!(e.header.fid in tremola.contacts)) { + var a = id2b32(e.header.fid); + tremola.contacts[e.header.fid] = { + "alias": a, "initial": a.substring(0, 1).toUpperCase(), + "color": colors[Math.floor(colors.length * Math.random())], + "iam": "", "forgotten": false + } + load_contact_list() + } + + switch (e.public[0]) { + default: + return + } + persist(); + must_redraw = true; + + } /** @@ -1018,7 +1059,7 @@ function b2f_new_incomplete_event(e) { * @param {number} e.hdr.tst Timestamp at which the message was created. (Number of milliseconds elapsed since midnight at the beginning of January 1970 00:00 UTC) * @param {string} e.hdr.ref The message ID of this log entry. * @param {string} e.hdr.fid The public key of the author encoded in base64. - * @param {[]} e.public The payload of the message + * @param {[]} e.public The payload of the message. The first entry is a String that represents the application to which the message belongs. All additional entries are application-specific parameters. * */ function b2f_new_event(e) { // incoming SSB log event: we get map with three entries @@ -1083,8 +1124,6 @@ function b2f_new_event(e) { // incoming SSB log event: we get map with three ent // if (curr_scenario == "chats") // the updated conversation could bubble up load_chat_list(); } else if (e.public[0] == "KAN") { // Kanban board event - console.log("New kanban event") - kanban_new_event(e) } else if (e.public[0] == "IAM") { var contact = tremola.contacts[e.header.fid] var old_iam = contact.iam diff --git a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt index 9868bea..30bf9d1 100644 --- a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt +++ b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt @@ -368,7 +368,7 @@ class WebAppInterface(val act: MainActivity, val webView: WebView) { // in-order api val replica = act.tinyRepo.fid2replica(fid) - if (frontend_frontier.getInt(fid.toHex(), 0) == seq && replica != null) { + if (frontend_frontier.getInt(fid.toHex(), 1) == seq && replica != null) { for (i in seq .. replica.state.max_seq ) { val content = replica.read(i) val message_id= replica.get_mid(seq) From 036a4b38a1607ea21cb71dd18aa3bcd5835aa340 Mon Sep 17 00:00:00 2001 From: Jannick Heisch Date: Sat, 9 Dec 2023 16:31:05 +0100 Subject: [PATCH 5/6] Fix: in-order delivery callback also receives log entries with incomplete sidechains --- android/tinySSB/app/src/main/assets/web/tremola.js | 2 +- .../java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt | 2 +- .../main/java/nz/scuttlebutt/tremolavossbol/tssb/Replica.kt | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/android/tinySSB/app/src/main/assets/web/tremola.js b/android/tinySSB/app/src/main/assets/web/tremola.js index a196e1b..2d2e8a6 100644 --- a/android/tinySSB/app/src/main/assets/web/tremola.js +++ b/android/tinySSB/app/src/main/assets/web/tremola.js @@ -993,7 +993,7 @@ function b2f_local_peer(type, identifier, displayname, status) { */ function b2f_new_in_order_event(e) { - console.log("b2f inorder event") + console.log("b2f inorder event:", JSON.stringify(e.public)) if (!(e.header.fid in tremola.contacts)) { var a = id2b32(e.header.fid); diff --git a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt index 30bf9d1..d6d55c5 100644 --- a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt +++ b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt @@ -372,7 +372,7 @@ class WebAppInterface(val act: MainActivity, val webView: WebView) { for (i in seq .. replica.state.max_seq ) { val content = replica.read(i) val message_id= replica.get_mid(seq) - if(content == null || message_id == null) + if(content == null || message_id == null || !replica.isSidechainComplete(i)) break e = toFrontendObject(fid, i, message_id, content) if (e != null) diff --git a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Replica.kt b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Replica.kt index 2fa0800..833ae8f 100644 --- a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Replica.kt +++ b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/tssb/Replica.kt @@ -451,4 +451,9 @@ class Replica(val context: MainActivity, val datapath: File, val fid: ByteArray) Log.d("replica", "write success, len: ${log_entry.size}") return seq } + + fun isSidechainComplete(seq: Int): Boolean { + return !state.pend_sc.containsKey(seq) + } + } \ No newline at end of file From acd110b8d3faa943e3d9d6e6a356270aa6ad08ea Mon Sep 17 00:00:00 2001 From: Jannick Heisch Date: Tue, 12 Dec 2023 13:38:17 +0100 Subject: [PATCH 6/6] fix: called non existing function --- .../main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt index d6d55c5..3a0f8bd 100644 --- a/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt +++ b/android/tinySSB/app/src/main/java/nz/scuttlebutt/tremolavossbol/WebAppInterface.kt @@ -370,7 +370,7 @@ class WebAppInterface(val act: MainActivity, val webView: WebView) { if (frontend_frontier.getInt(fid.toHex(), 1) == seq && replica != null) { for (i in seq .. replica.state.max_seq ) { - val content = replica.read(i) + val content = replica.read_content(i) val message_id= replica.get_mid(seq) if(content == null || message_id == null || !replica.isSidechainComplete(i)) break