TransportGUN.js

/* eslint-disable max-len, camelcase, n */
/* global Gun */
/*
This Transport layers uses GUN.

See https://github.com/internetarchive/dweb-mirror/issues/43 for meta issue
*/
const Url = require('url');

process.env.GUN_ENV = 'false';

/* This should be done in the caller (see dweb-archive/archive.html for example)
const Gun = require('gun/gun.js');  // gun/gun is the minimized version
// Raw Gun has almost nothing in it, it needs at least the following to work properly.
require('gun/lib/path.js');         // So that .path works

//WORKAROUND-GUN-STORAGE
// The next step is to stop it failing as soon as its cached 5Mb in localstorage
// see https://github.com/amark/gun/blob/master/test/tmp/indexedDB.html and https://github.com/amark/gun/issues/590
// but the instructions on how to do this are obviously broken so waiting on @amark to get this working.

// See https://github.com/internetarchive/dweb-archive/issues/106 unable to get this working (Gun doesnt work well with webpack)
//require('gun/nts');
//require('gun/lib/wire'); // NodeJS websocket
//require('gun/lib/multicast'); // Local area broadcasting needs passing `multicast: true` in options (which is safe in node + browser)
require('gun/lib/radix.js'); // loaded by store but required for webpack
require('gun/lib/radisk.js'); // loaded by store but required for webpack
require('gun/lib/store.js');
require('gun/lib/rindexed.js');
*/
const debug = require('debug')('dweb-transports:gun');
const canonicaljson = require('@stratumn/canonicaljson');

// Other Dweb modules
const { TransportError, CodingError, ToBeImplementedError } = require('./Errors'); // Standard Dweb Errors
const Transport = require('./Transport.js'); // Base class for TransportXyz
const Transports = require('./Transports'); // Manage all Transports that are loaded
const utils = require('./utils'); // Utility functions

// Utility packages (ours) And one-liners
// unused currently: function delay(ms, val) { return new Promise(resolve => {setTimeout(() => { resolve(val); },ms)})}

const defaultoptions = {
  // peers: ['https://dweb.me:4246/gun'],
  peers: ['https://www-dweb-gun.dev.archive.org/gun'],
  localStorage: false                         // Need to be false to turn localStorage off on browser (do this if include radix/radisk)
};
// To run a superpeer - cd wherever; node install gun; cd node_modules/gun; npm start - starts server by default on port 8080, or set an "env" - see http.js
// setenv GUN_ENV false; node examples/http.js 4246
// Make sure to open of the port (typically in /etc/ferm)
// TODO-GUN - copy example from systemctl here

/*
    WORKING AROUND GUN WEIRDNESS/SUBOPTIMAL (of course, whats weird/sub-optimal to me, might be ideal to someone else) - search the code to see where worked around

    WORKAROUND-GUN-UNDERSCORE .once() and possibly .on() send an extra GUN internal field "_" which needs filtering. Reported and hopefully will get fixed
    .once behaves differently on node or the browser - this is a bug https://github.com/amark/gun/issues/586 and for now this code doesnt work on Node
    WORKAROUND-GUN-CURRENT: .once() and .on() deliver existing values as well as changes, reported & hopefully will get way to find just new ones.
    WORKAROUND-GUN-DELETE:  There is no way to delete an item, setting it to null is recorded and is by convention a deletion. BUT the field will still show up in .once and .on,
    WORKAROUND-GUN-PROMISES: GUN is not promisified, there is only one place we care, and that is .once (since .on is called multiple times).
    WORKAROUND-GUN-ERRORS: GUN does an unhelpful job with errors, for example returning undefined when it cant find something (e.g. if connection to superpeer is down),
        for now just throw an error on undefined
    WORKAROUND-GUN-STORAGE: GUN defaults to local storage, which then fails on 5Mb or more of data, need to use radix, which has to be included and has bizarre config requirement I can't figure out
    TODO-GUN, handle error callbacks which are available in put etc
    Errors and Promises: Note that GUN's use of promises is seriously unexpected (aka weird), see https://gun.eco/docs/SEA#errors
        instead of using .reject or throwing an error at async it puts the error in SEA.err, so how that works in async parallel context is anyone's guess
 */

class TransportGUN extends Transport {
  /*
  GUN specific transport - over IPFS

  Fields:
  gun: object returned when starting GUN
   */

  constructor(options) {
    super(options);
    this.options = options;         // Dictionary of options
    this.gun = undefined;
    this.name = 'GUN';          // For console log etc
    this.supportURLs = ['gun'];
    this.supportFunctions = ['fetch', // 'store'
      'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor',
      'add', 'list', 'listmonitor', 'newlisturls'];
    this.supportFeatures = []; // Doesnt support noCache and is mutable
    this.setStatus(Transport.STATUS_LOADED);
  }

  connection(url) {
    /*
    TODO-GUN need to determine what a "rooted" Url is in gun, is it specific to a superpeer for example
    Utility function to get Gun object for this URL (note this isn't async)
    url:        URL string or structure, to find list of of form [gun|dweb]:/gun/<database>/<table>[/<key]  but could be arbitrary gun path
    resolves:   Gun a connection to use for get's etc, undefined if fails
    */
    // eslint-disable-next-line  no-param-reassign
    url = Url.parse(url); // Accept string or Url structure
    const patharray = url.pathname.split('/');   // [ 'gun', database, table ] but could be arbitrary length path
    patharray.shift();  // Loose leading ""
    patharray.shift();    // Loose "gun"
    debug('path=', patharray);
    return this.gun.path(patharray);           // Not sure how this could become undefined as it will return g before the path is walked, but if do a lookup on this "g" then should get undefined
  }

  // TODO-SPLIT define load()

  static setup0(options) {
    /*
        First part of setup, create obj, add to Transports but dont attempt to connect, typically called instead of p_setup if want to parallelize connections.
        options: { gun: { }, }   Set of options - "gun" is used for those to pass direct to Gun
    */
    const combinedoptions = Transport.mergeoptions(defaultoptions, options.gun);
    debug('options %o', combinedoptions);
    const t = new TransportGUN(combinedoptions);     // Note doesnt start IPFS or OrbitDB
    t.gun = new Gun(t.options);                         // This doesnt connect, just creates db structure
    Transports.addtransport(t);
    return t;
  }

  async p_setup1() {
    /*
    This sets up for GUN.
    Throws: TODO-GUN-DOC document possible error behavior
    */
    try {
      this.setStatus(Transport.STATUS_STARTING);   // Should display, but probably not refreshed in most case
      // TODO-GUN-TEST - try connect and retrieve info then look at ._.opt.peers
      await this.p_status();
    } catch (err) {
      debug('%s failed to start %o', this.name, err);
      this.setStatus(Transport.STATUS_FAILED);
    }
    return this;
  }

  async p_status() {
    /*
    Return an integer for the status of a transport see Transport
     */
    // TODO-GUN-TEST - try connect and retrieve info then look at ._.opt.peers
    this.setStatus(Transport.STATUS_CONNECTED);  // TODO-GUN how do I know if/when I'm connected (see comment on p_setup1 as well)
    return this.status;
  }
  // ===== DATA ======

  async p_rawfetch(url) {
    // eslint-disable-next-line no-param-reassign
    url = Url.parse(url);   // Accept url as string or object
    const g = this.connection(url); // Goes all the way to the key
    const val = await this._p_once(g);
    // g.on((data)=>debug("Got late result of: %o", data)); // Checking for bug in GUN issue#586 - ignoring result
    if (!val) throw new TransportError('GUN unable to retrieve: ' + url.href);  // WORKAROUND-GUN-ERRORS - gun doesnt throw errors when it cant find something
    const o = typeof val === 'string' ? JSON.parse(val) : val;  // This looks like it is sync (see same code on p_get and p_rawfetch)
    // TODO-GUN this is a hack because the metadata such as metadata/audio is getting cached in GUN and in this case is wrong.
    if (o.metadata && o.metadata.thumbnaillinks && o.metadata.thumbnaillinks.find(t => t.includes('ipfs/zb2'))) {
      throw new TransportError('GUN retrieving legacy data at: ' + url.href);
    }
    return o;
  }


  // ===== LISTS ========

  // noinspection JSCheckFunctionSignatures
  async p_rawlist(url) {
    /*
    Fetch all the objects in a list, these are identified by the url of the public key used for signing.
    (Note this is the 'signedby' parameter of the p_rawadd call, not the 'url' parameter
    Returns a promise that resolves to the list.
    Each item of the list is a dict: {"url": url, "date": date, "signature": signature, "signedby": signedby}
    List items may have other data (e.g. reference ids of underlying transport)

    :param string url: String with the url that identifies the list.
    :resolve array: An array of objects as stored on the list.
     */
    try {
      const g = this.connection(url);
      const data = await this._p_once(g);
      const res = data ? Object.keys(data).filter(k => k !== '_').sort().map(k => data[k]) : []; // See WORKAROUND-GUN-UNDERSCORE
      // .filter((obj) => (obj.signedby.includes(url))); // upper layers verify, which filters
      debug('p_rawlist found', ...utils.consolearr(res));
      return res;
    } catch (err) {
      // Will be logged by Transports
      throw (err);
    }
  }

  listmonitor(url, callback, { current = false } = {}) {
    /*
     Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_rawlist to get any more items not returned by p_rawlist.

     url:       string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd
     callback:  function(obj)  Callback for each new item added to the list
                    obj is same format as p_rawlist or p_rawreverse
     current    true if should send list of existing elements
      */
    const g = this.connection(url);
    if (!current) { // See WORKAROUND-GUN-CURRENT have to keep an extra copy to compare for which calls are new.
      g.once(data => {
        this.monitored = data ? Object.keys(data) : []; //  Keep a copy - could actually just keep high water mark unless getting partial knowledge of state of array.
        g.map().on((v, k) => {
          if (!(this.monitored.includes(k)) && (k !== '_')) { // See WORKAROUND-GUN-UNDERSCORE
            this.monitored.push(k);
            callback(JSON.parse(v));
          }
        });
      });
    } else {
      g.map().on((v, k) => callback('set', k, JSON.parse(v)));
    }
  }

  // noinspection JSCheckFunctionSignatures
  async p_rawadd(url, sig) {
    /*
    Store a new list item, it should be stored so that it can be retrieved either by "signedby" (using p_rawlist) or
    by "url" (with p_rawreverse). The underlying transport does not need to guarantee the signature,
    an invalid item on a list should be rejected on higher layers.

    :param string url: String identifying list to post to
    :param Signature sig: Signature object containing at least:
        date - date of signing in ISO format,
        urls - array of urls for the object being signed
        signature - verifiable signature of date+urls
        signedby - urls of public key used for the signature
    :resolve undefined:
    */
    // noinspection JSUnresolvedVariable
    // Logged by Transports
    if (!(url && sig.urls.length && sig.signature && sig.signedby.length)) {
      throw new Error('TransportGUN.p_rawadd args passed bad arguments:');
    }
    this.connection(url)
      .set(canonicaljson.stringify(sig.preflight(Object.assign({}, sig))));
  }

  // noinspection JSCheckFunctionSignatures
  async p_newlisturls(cl) {
    const u = await this._p_newgun(cl);
    return [u, u];
  }

  //= ======KEY VALUE TABLES ========

  // noinspection JSMethodCanBeStatic
  async _p_newgun(pubkey) {
    if (pubkey.hasOwnProperty('keypair')) pubkey = pubkey.keypair.signingexport();
    // By this point pubkey should be an export of a public key of form xyz:abc where xyz
    // specifies the type of public key (NACL VERIFY being the only kind we expect currently)
    return `gun:/gun/${encodeURIComponent(pubkey)}`;
  }

  async p_newdatabase(pubkey) {
    /*
    Request a new database
    For GUN it doesnt actually create anything, just generates the URLs
    TODO-GUN simple version first - userid based on my keypair first, then switch to Gun's userid and its keypair
    Include gun/sea.js; user.create(<alias>,<passphrase>); user.auth(<alias>,<passphrase>); # See gun.eco/docs/Auth

    returns: {publicurl: "gun:/gun/<publickey>", privateurl:  "gun:/gun/<publickey>">
    */
    const u = await this._p_newgun(pubkey);
    return { publicurl: u, privateurl: u };
  }

  async p_newtable(pubkey, table) {
    /*
    Request a new table
    For GUN it doesnt actually create anything, just generates the URLs

    returns: {publicurl: "gun:/gun/<publickey>/<table>", privateurl:  "gun:/gun/<publickey>/<table>">
    */
    if (!pubkey) throw new CodingError('p_newtable currently requires a pubkey');
    const database = await this.p_newdatabase(pubkey);
    // If have use cases without a database, then call p_newdatabase first
    return { privateurl: `${database.privateurl}/${table}`,  publicurl: `${database.publicurl}/${table}` };  // No action required to create it
  }

  async p_set(url, keyvalues, value) {  // url = yjs:/yjs/database/table
    /*
    Set key values
    keyvalues:  string (key) in which case value should be set there OR
            object in which case value is ignored
     */
    const table = this.connection(url);
    if (typeof keyvalues === 'string') {
      table.path(keyvalues).put(canonicaljson.stringify(value));
    } else {
      // Store all key-value pairs without destroying any other key/value pairs previously set
      if (Array.isArray(keyvalues)) {
        throw new Error('TransportGUN - should not be passing an array as the keyvalues');
      }
      table.put(
        Object.keys(keyvalues).reduce(
          (previous, key) => { previous[key] = canonicaljson.stringify(keyvalues[key]); return previous; },
          {}
        )
      );
    }
  }

  async p_get(url, keys) {
    const table = this.connection(url);
    if (Array.isArray(keys)) {
      throw new ToBeImplementedError("p_get(url, [keys]) isn't supported - because of ambiguity better to explicitly loop on set of keys or use getall and filter");
      /*
      return keys.reduce(function(previous, key) {
          let val = table.get(key);
          previous[key] = typeof val === "string" ? JSON.parse(val) : val;    // Handle undefined
          return previous;
      }, {});
      */
    } else {
      const val = await this._p_once(table.get(keys));    // Resolves to value
      return typeof val === 'string' ? JSON.parse(val) : val;  // This looks like it is sync (see same code on p_get and p_rawfetch)
    }
  }

  async p_delete(url, keys) {
    const table = this.connection(url);
    if (typeof keys === 'string') {
      table.path(keys).put(null);
    } else {
      keys.map((key) => table.path(key).put(null));  // This looks like it is sync
    }
  }

  // WORKAROUND-GUN-PROMISE suggest p_once as a good single addition
  // TODO-GUN expand this to workaround Gun weirdness with errors.
  _p_once(gun) {  // Note in some cases (e.g. p_getall) this will resolve to a object, others a string/number (p_get)
    // TODO-GUN Temporarily added a 2000ms delay to workaround https://github.com/internetarchive/dweb-archive/issues/106 / https://github.com/amark/gun/issues/762
    return new Promise((resolve) => gun.once(resolve, { wait: 2000 }));
  }

  async p_keys(url) {
    const res = await this._p_once(this.connection(url));
    return Object.keys(res)
      .filter(k => (k !== '_') && (res[k] !== null)); // See WORKAROUND-GUN-UNDERSCORE and WORKAROUND-GUN-DELETE
  }

  async p_getall(url) {
    const res = await this._p_once(this.connection(url));
    return Object.keys(res)
      .filter(k => (k !== '_') && res[k] !== null) // See WORKAROUND-GUN-UNDERSCORE and WORKAROUND-GUN-DELETE
      .reduce((previous, key) => { previous[key] = JSON.parse(res[key]); return previous; }, {});
  }

  async monitor(url, callback, { current = false } = {}) {
    /*
     Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_getall to get any more items not returned by p_getall.
     Stack: KVT()|KVT.p_new => KVT.monitor => (a: Transports.monitor => GUN.monitor)(b: dispatchEvent)

     url:         string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd
     callback:    function({type, key, value})  Callback for each new item added to the list (type = "set"|"delete")
     current            Send existing items to the callback as well
      */
    const g = this.connection(url);
    if (!current) { // See WORKAROUND-GUN-CURRENT have to keep an extra copy to compare for which calls are new.
      g.once(data => {
        this.monitored = Object.assign({}, data); //  Make a copy of data (this.monitored = data won't work as just points at same structure)
        g.map().on((v, k) => {
          if ((v !== this.monitored[k]) && (k !== '_')) { // See WORKAROUND-GUN-UNDERSCORE
            this.monitored[k] = v;
            callback('set', k, JSON.parse(v));
          }
        });
      });
    } else {
      g.map().on((v, k) => callback('set', k, JSON.parse(v)));
    }
  }

  static async p_test() {
    debug('p_test');
    try {
      const t = this.setup0({});   // TODO-GUN when works with peers commented out, try passing peers: []
      await t.p_setup1();
      await t.p_setup2(); // - this one does nothing on GUN
      // noinspection JSIgnoredPromiseFromCall
      await t.p_test_kvt('gun:/gun/NACL');
      // t.p_test_list("gun:/gun/NACL"); //TODO test_list needs fixing to not create a dependency on Signature
    } catch (err) {
      debug('WARNING Exception thrown in TransportGUN.test: %s', err.message);
      throw err;
    }
  }

  // noinspection JSUnusedGlobalSymbols
  static async demo_bugs() {
    const gun = new Gun();
    gun.get('foo').get('bar').put('baz');
    debug('TESTING: Expect {bar: "baz"} but get {_:..., bar: "baz"}');
    gun.get('foo').once(data => debug(data));
    gun.get('zip').get('bar').set('alice');
    debug("TESTING: Expect {12345: 'alice'} but get {_:..., 12345: 'alice'}");
    gun.get('foo').once(data => debug(data));
    // Returns extra "_" field
  }
}
Transports._transportclasses.GUN = TransportGUN;
// Defines global.Gun
TransportGUN.requires =  TransportGUN.scripts = ['gun/gun.js', 'gun/lib/path.js', 'gun/lib/radix.js',
  'gun/lib/radisk.js', 'gun/lib/store.js', 'gun/lib/rindexed.js'];
exports = module.exports = TransportGUN;