Merge branch 'main' into pinner/new-refactor-3

.github/workflows/docker-image.yml

@@ -0,0 +1,18 @@
+name: Docker Image CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build the Docker image
+      run: docker build . --file Dockerfile --tag orbit-db-pinner:$(date +%s)

.github/workflows/node.js.yml

@@ -0,0 +1,29 @@
+# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [12.x, 13.x, 14.x]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - run: npm ci
+    - run: npm run build --if-present
+    - run: npm test

Makefile

@@ -0,0 +1,8 @@
+clean:
+	docker image prune --filter label=stage=orbit-db-pinner-builder
+
+docker-build:
+	docker build . -t orbit-db-pinner:latest
+
+docker-run-dev:
+	docker run orbit-db-pinner:latest

config/default.js

@@ -0,0 +1,23 @@
+const IpfsApi = require('ipfs-http-client')
+const ipfs = IpfsApi.create()
+
+// const IPFS = require('ipfs')
+// const ipfs = IPFS.create({
+//   repo: './orbitdb/pinner',
+//   start: true,
+//   EXPERIMENTAL: {
+//     pubsub: true
+//   },
+//   config: {}
+// }) // Inject my config
+
+module.exports = () => {
+  return {
+    http: {
+      port: 3000,
+      enabled: true
+    },
+    // 'ipfsHttpModule': ipfsHttp,
+    ipfsModule: ipfs
+  }
+}

config/index.js

@@ -0,0 +1,5 @@
+module.exports = require(
+  process.env.NODE_ENV !== 'production'
+    ? './default'
+    : './production'
+)

index.js

@@ -0,0 +1,3 @@
+module.exports.pinner = require('./lib/OrbitPinner')
+module.exports.http = require('./lib/httpServer')
+module.exports.follow = require('./lib/pinningList')

lib/OrbitPinner.js

@@ -0,0 +1,55 @@
+'use strict'
+const OrbitDB = require('orbit-db')
+
+let orbitdb
+
+class Pinner {
+  constructor (db) {
+    this.db = db
+    this.address = db.id
+  }
+
+  static async create (address) {
+    const ipfs = await require('./ipfsInstance')
+    if (!orbitdb) orbitdb = await OrbitDB.createInstance(ipfs)
+    const db = await Pinner.openDatabase(orbitdb, address)
+    return Promise.resolve(new Pinner(db))
+  }
+
+  drop () {
+    // console.log(this.orbitdb)
+    // this.orbitdb.disconnect()
+  }
+
+  get estimatedSize () {
+    let size = 0
+
+    if (this.db) {
+      // This is very crude
+      size = JSON.stringify(this.db._oplog.values).length
+    }
+
+    return size
+  }
+
+  static async openDatabase (orbitdb, address) {
+    try {
+      if (!OrbitDB.isValidAddress(address)) {
+        console.log(`Failed to add ${address}. This is not a valid address`)
+        return
+      }
+
+      console.log(`opening database from ${address}`)
+      const db = await orbitdb.open(address, { sync: true })
+
+      console.log('Listening for updates to the database...')
+      await db.load()
+
+      return db
+    } catch (e) {
+      console.error(e)
+    }
+  }
+}
+
+module.exports = Pinner

lib/httpServer.js

@@ -0,0 +1,57 @@
+const config = require('config')
+
+const express = require('express')
+const app = express()
+
+const pinningList = require('./pinningList')
+
+class server {
+  constructor (httpPort) {
+    const port = httpPort || config.get('http.port')
+
+    app.get('/stats', async (req, res) => {
+      const numDatabases = (await pinningList.getContents()).length
+      const pinners = await pinningList.getPinners()
+
+      const pinnerStats = Object.values(pinners).map((pinner) => {
+        return ({
+          size: pinner.estimatedSize
+        })
+      })
+
+      res.json({
+        pinners: pinnerStats,
+        num_databases: numDatabases,
+        total_size: pinnerStats.reduce((a, b) => a + b.size, 0)
+      })
+    })
+
+    app.get('/pin', (req, res) => {
+      const address = req.query.address
+
+      if (req.query.address) {
+        pinningList.add(address)
+
+        res.send(`adding... ${address}`)
+      } else {
+        res.send('missing \'address\' query parameter')
+      }
+    })
+
+    app.get('/unpin', (req, res) => {
+      const address = req.query.address
+
+      if (req.query.address) {
+        pinningList.remove(address)
+
+        res.send(`removing... ${address}`)
+      } else {
+        res.send('missing \'address\' query parameter')
+      }
+    })
+
+    app.listen(port, () => console.log(`Orbit-pinner listening on port ${port}`))
+  }
+}
+
+module.exports = server

lib/ipfsInstance.js

@@ -0,0 +1,10 @@
+const path = require('path')
+const config = require(path.join(process.cwd(), 'config/index.js'))()
+
+// Good idea pass default lib?
+
+if ('ipfsHttpModule' in config) {
+  module.exports = config.ipfsHttpModule
+} else if ('ipfsModule' in config) {
+  module.exports = config.ipfsModule
+}

lib/pinningList/index.js

@@ -0,0 +1,124 @@
+const OrbitDB = require('orbit-db')
+
+const OrbitPinner = require('../OrbitPinner')
+const orbitInstance = require('./orbitInstance')
+
+const pinners = {}
+
+const getContents =
+  async () => {
+    const db = await orbitInstance()
+
+    return db.iterator({ limit: -1 })
+      .collect()
+      .map(
+        e => {
+          return e.payload.value
+        }
+      )
+  }
+
+const getPinners = () => pinners
+
+const add =
+  async (address) => {
+    const db = await orbitInstance()
+
+    if (!OrbitDB.isValidAddress(address)) {
+      console.log(`Failed to add ${address}. This is not a valid address`)
+      return
+    }
+
+    const addresses = await getContents()
+
+    if (!addresses.includes(address)) {
+      await db.add(address)
+      createPinnerInstance(address)
+
+      console.log(`${address} added.`)
+    } else {
+      console.warn(`Attempted to add ${address}, but already present in db.`)
+    }
+  }
+
+const createPinnerInstance =
+        async (address) => {
+          if (!OrbitDB.isValidAddress(address)) {
+            console.log(`Failed to pin ${address}. This is not a valid address`)
+            return
+          }
+
+          console.log(`Pinning orbitdb @ ${address}`)
+          const pinner = await OrbitPinner.create(address)
+          pinners[address] = pinner
+
+          return pinners[address]
+        }
+
+const startPinning =
+        async () => {
+          const addresses = await getContents()
+
+          if (addresses.length === 0) {
+            console.log('Pinning list is empty')
+          }
+
+          addresses
+            .map(createPinnerInstance)
+        }
+
+const remove =
+  async (address) => {
+    if (!OrbitDB.isValidAddress(address)) {
+      console.log(`Failed to unpin ${address}. This is not a valid address`)
+      return
+    }
+
+    if (!pinners[address]) {
+      console.log(`Failed to unpin ${address}. Address not found in pinning list.`)
+      return
+    }
+
+    const db = await orbitInstance()
+    const dbAddresses = await getContents()
+
+    // stop pinning
+    pinners[address].drop()
+    delete pinners[address]
+
+    // Unfortunately, since we can't remove a item from the database without it's hash
+    // So we have to rebuild the data every time we remove an item.
+    await db.drop()
+
+    dbAddresses
+      .filter(addr => (addr !== address))
+      .forEach(
+        address => db.add(address)
+      )
+
+    console.log(`${address} removed.`)
+  }
+
+const follow =
+  async (address) => {
+    if (!OrbitDB.isValidAddress(address)) {
+      console.log(`Failed to follow ${address}. This is not a valid address`)
+
+      return
+    }
+
+    // await db.drop()
+    await orbitInstance(address)
+    startPinning()
+  }
+
+console.log('Pinning previously added orbitdbs: ')
+startPinning()
+
+module.exports = {
+  add,
+  getContents,
+  getPinners,
+  remove,
+  follow
+}

lib/pinningList/orbitInstance.js

@@ -0,0 +1,31 @@
+const OrbitDB = require('orbit-db')
+
+const ipfsInstanceP = require('../ipfsInstance')
+
+const orbitInstance = new Promise(resolve => {
+  ipfsInstanceP.then(ipfsInstance => {
+    resolve(OrbitDB.createInstance(ipfsInstance, {
+      directory: './orbitdb/pinner/Manifest'
+    }))
+  })
+})
+
+const createDbInstance = async addr => {
+  const address = addr || 'dbList'
+  const dbInstance = await orbitInstance
+
+  const pinningList = {
+    create: true,
+    overwrite: true,
+    localOnly: false,
+    type: 'feed'
+  }
+
+  const db = await dbInstance.open(address, pinningList)
+
+  await db.load()
+
+  return db
+}
+
+module.exports = createDbInstance