Files add interface

package.json

@@ -5,13 +5,15 @@
   "main": "lib/index.js",
   "jsnext:main": "src/index.js",
   "dependencies": {
-    "bl": "^1.1.2",
+    "async": "^2.0.0-rc.5",
     "babel-runtime": "^6.6.1",
+    "bl": "^1.1.2",
     "bs58": "^3.0.0",
     "detect-node": "^2.0.3",
     "flatmap": "0.0.3",
     "glob": "^7.0.3",
     "ipfs-merkle-dag": "^0.6.0",
+    "isstream": "^0.1.2",
     "multiaddr": "^2.0.0",
     "multipart-stream": "^2.0.1",
     "ndjson": "^1.4.3",

src/add-to-dagnode-transform.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const async = require('async')
+const getDagNode = require('./get-dagnode')
+
+// transform { Hash: '...' } objects into { path: 'string', node: DAGNode }
+module.exports = function (err, res, send, done) {
+  if (err) {
+    return done(err)
+  }
+  async.map(res, function map (entry, next) {
+    getDagNode(send, entry.Hash, function (err, node) {
+      if (err) {
+        return next(err)
+      }
+      var obj = {
+        path: entry.Name,
+        node: node
+      }
+      next(null, obj)
+    })
+  }, function (err, res) {
+    done(err, res)
+  })
+}

src/api/add-files.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const addToDagNodesTransform = require('../add-to-dagnode-transform')
+
+module.exports = (send) => {
+  return function add (path, opts, cb) {
+    if (typeof (opts) === 'function' && cb === undefined) {
+      cb = opts
+      opts = {}
+    }
+
+    if (typeof (path) !== 'string') {
+      return cb(new Error('"path" must be a string'))
+    }
+
+    var sendWithTransform = send.withTransform(addToDagNodesTransform)
+
+    return sendWithTransform('add', null, opts, path, cb)
+  }
+}

src/api/add-url.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const Wreck = require('wreck')
+const addToDagNodesTransform = require('../add-to-dagnode-transform')
+
+module.exports = (send) => {
+  return function add (url, opts, cb) {
+    if (typeof (opts) === 'function' && cb === undefined) {
+      cb = opts
+      opts = {}
+    }
+
+    if (typeof url !== 'string' || !url.startsWith('http')) {
+      return cb(new Error('"url" param must be an http(s) url'))
+    }
+
+    var sendWithTransform = send.withTransform(addToDagNodesTransform)
+
+    Wreck.request('GET', url, null, (err, res) => {
+      if (err) return cb(err)
+
+      sendWithTransform('add', null, opts, res, cb)
+    })
+  }
+}

src/api/add.js

@@ -1,6 +1,7 @@
 'use strict'
 
-const Wreck = require('wreck')
+const isStream = require('isstream')
+const addToDagNodesTransform = require('../add-to-dagnode-transform')
 
 module.exports = (send) => {
   return function add (files, opts, cb) {
@@ -9,14 +10,16 @@ module.exports = (send) => {
       opts = {}
     }
 
-    if (typeof files === 'string' && files.startsWith('http')) {
-      return Wreck.request('GET', files, null, (err, res) => {
-        if (err) return cb(err)
+    const good = Buffer.isBuffer(files) ||
+               isStream.isReadable(files) ||
+               Array.isArray(files)
 
-        send('add', null, opts, res, cb)
-      })
+    if (!good) {
+      return cb(new Error('"files" must be a buffer, readable stream, or array of objects'))
     }
 
-    return send('add', null, opts, files, cb)
+    var sendWithTransform = send.withTransform(addToDagNodesTransform)
+
+    return sendWithTransform('add', null, opts, files, cb)
   }
 }

src/get-dagnode.js

@@ -0,0 +1,38 @@
+'use strict'
+
+const DAGNode = require('ipfs-merkle-dag').DAGNode
+const bl = require('bl')
+const async = require('async')
+
+module.exports = function (send, hash, cb) {
+  // Retrieve the object and its data in parallel, then produce a DAGNode
+  // instance using this information.
+  async.parallel([
+    function get (done) {
+      send('object/get', hash, null, null, done)
+    },
+
+    function data (done) {
+      // WORKAROUND: request the object's data separately, since raw bits in JSON
+      // are interpreted as UTF-8 and corrupt the data.
+      // See https://github.com/ipfs/go-ipfs/issues/1582 for more details.
+      send('object/data', hash, null, null, done)
+    }],
+
+    function done (err, res) {
+      if (err) {
+        return cb(err)
+      }
+
+      var object = res[0]
+      var stream = res[1]
+
+      stream.pipe(bl(function (err, data) {
+        if (err) {
+          return cb(err)
+        }
+
+        cb(err, new DAGNode(data, object.Links))
+      }))
+    })
+}

src/load-commands.js

@@ -1,8 +1,7 @@
 'use strict'
 
 function requireCommands () {
-  return {
-    add: require('./api/add'),
+  var cmds = {
     bitswap: require('./api/bitswap'),
     block: require('./api/block'),
     cat: require('./api/cat'),
@@ -11,7 +10,6 @@ function requireCommands () {
     dht: require('./api/dht'),
     diag: require('./api/diag'),
     id: require('./api/id'),
-    files: require('./api/files'),
     log: require('./api/log'),
     ls: require('./api/ls'),
     mount: require('./api/mount'),
@@ -24,6 +22,24 @@ function requireCommands () {
     update: require('./api/update'),
     version: require('./api/version')
   }
+
+  // TODO: crowding the 'files' namespace temporarily for interface-ipfs-core
+  // compatibility, until 'files vs mfs' naming decision is resolved.
+  cmds.files = function (send) {
+    const files = require('./api/files')(send)
+    files.add = require('./api/add')(send)
+    return files
+  }
+
+  cmds.util = function (send) {
+    const util = {
+      addFiles: require('./api/add-files')(send),
+      addUrl: require('./api/add-url')(send)
+    }
+    return util
+  }
+
+  return cmds
 }
 
 function loadCommands (send) {

src/request-api.js

@@ -110,7 +110,7 @@ function requestAPI (config, path, args, qs, files, buffer, cb) {
 // -- Interface
 
 exports = module.exports = function getRequestAPI (config) {
-  return function (path, args, qs, files, buffer, cb) {
+  var send = function (path, args, qs, files, buffer, cb) {
     if (typeof buffer === 'function') {
       cb = buffer
       buffer = false
@@ -127,4 +127,41 @@ exports = module.exports = function getRequestAPI (config) {
 
     return requestAPI(config, path, args, qs, files, buffer, cb)
   }
+
+  // Wraps the 'send' function such that an asynchronous transform may be
+  // applied to its result before passing it on to either its callback or
+  // promise.
+  send.withTransform = function (transform) {
+    return function (path, args, qs, files, buffer, cb) {
+      if (typeof buffer === 'function') {
+        cb = buffer
+        buffer = false
+      }
+
+      var p = send(path, args, qs, files, buffer, wrap(cb))
+
+      if (p instanceof Promise) {
+        return p.then((res) => {
+          return new Promise(function (resolve, reject) {
+            transform(null, res, send, function (err, res) {
+              if (err) reject(err)
+              else resolve(res)
+            })
+          })
+        })
+      } else {
+        return p
+      }
+
+      function wrap (done) {
+        if (done) {
+          return function (err, res) {
+            transform(err, res, send, done)
+          }
+        }
+      }
+    }
+  }
+
+  return send
 }