Outsourcing resource location resolution to resolver-engine

package.json

@@ -38,13 +38,14 @@
     "npm-link-local": "^1.1.0",
     "npm-run-all": "^4.0.2",
     "onchange": "^3.2.1",
-    "remix-debug": "0.3.1",
     "remix-analyzer": "0.3.1",
+    "remix-debug": "0.3.1",
     "remix-lib": "0.4.1",
     "remix-solidity": "0.3.1",
     "remix-tests": "0.1.1",
     "remixd": "0.1.8-alpha.6",
     "request": "^2.83.0",
+    "@resolver-engine/imports": "^0.3.0",
     "rimraf": "^2.6.1",
     "selenium-standalone": "^6.0.1",
     "solc": "^0.5.0",

src/app.js

@@ -20,6 +20,7 @@ var Storage = remixLib.Storage
 var Browserfiles = require('./app/files/browser-files')
 var BrowserfilesTree = require('./app/files/browser-files-tree')
 var SharedFolder = require('./app/files/shared-folder')
+var ReadOnlyFileProvider = require('./app/files/readOnlyFileProvider')
 var Config = require('./config')
 var Renderer = require('./app/ui/renderer')
 var executionContext = require('./execution-context')
@@ -32,7 +33,6 @@ var TxLogger = require('./app/execution/txLogger')
 var Txlistener = remixLib.execution.txListener
 var EventsDecoder = remixLib.execution.EventsDecoder
 var FileManager = require('./app/files/fileManager')
-var BasicReadOnlyExplorer = require('./app/files/basicReadOnlyExplorer')
 var NotPersistedExplorer = require('./app/files/NotPersistedExplorer')
 var toolTip = require('./app/ui/tooltip')
 var TransactionReceiptResolver = require('./transactionReceiptResolver')
@@ -147,18 +147,21 @@ class App {
     })
 
     self._components.filesProviders['localhost'] = new SharedFolder(remixd)
-    self._components.filesProviders['swarm'] = new BasicReadOnlyExplorer('swarm')
-    self._components.filesProviders['github'] = new BasicReadOnlyExplorer('github')
     self._components.filesProviders['gist'] = new NotPersistedExplorer('gist')
-    self._components.filesProviders['ipfs'] = new BasicReadOnlyExplorer('ipfs')
-    self._components.filesProviders['https'] = new BasicReadOnlyExplorer('https')
-    self._components.filesProviders['http'] = new BasicReadOnlyExplorer('http')
+    self._components.readOnly = new ReadOnlyFileProvider()
+    self._components.filesProviders['swarm'] = self._components.readOnly // proof of concept, awaiting acceptance before digging deeper
+    self._components.filesProviders['github'] = self._components.readOnly
+    self._components.filesProviders['ipfs'] = self._components.readOnly
+    self._components.filesProviders['https'] = self._components.readOnly
+    self._components.filesProviders['http'] = self._components.readOnly
+    self._components.filesProviders['external'] = self._components.readOnly
     registry.put({api: self._components.filesProviders['localhost'], name: 'fileproviders/localhost'})
     registry.put({api: self._components.filesProviders['swarm'], name: 'fileproviders/swarm'})
     registry.put({api: self._components.filesProviders['github'], name: 'fileproviders/github'})
     registry.put({api: self._components.filesProviders['gist'], name: 'fileproviders/gist'})
     registry.put({api: self._components.filesProviders['ipfs'], name: 'fileproviders/ipfs'})
     registry.put({api: self._components.filesProviders, name: 'fileproviders'})
+    registry.put({api: self._components.readOnly, name: 'fileproviders/readonly'})
 
     self._view = {}
 

src/app/compiler/compiler-imports.js

@@ -1,7 +1,7 @@
 'use strict'
 var base64 = require('js-base64').Base64
-var swarmgw = require('swarmgw')()
 var request = require('request')
+var resolver = require('@resolver-engine/imports').ImportsEngine()
 
 module.exports = class CompilerImports {
   constructor (githubAccessToken) {
@@ -30,95 +30,37 @@ module.exports = class CompilerImports {
       })
   }
 
-  handleSwarmImport (url, cleanUrl, cb) {
-    swarmgw.get(url, function (err, content) {
-      cb(err, content, cleanUrl)
-    })
-  }
-
-  handleIPFS (url, cb) {
-    // replace ipfs:// with /ipfs/
-    url = url.replace(/^ipfs:\/\/?/, 'ipfs/')
-
-    return request.get(
-      {
-        url: 'https://gateway.ipfs.io/' + url
-      },
-      (err, r, data) => {
-        if (err) {
-          return cb(err || 'Unknown transport error')
-        }
-        cb(null, data, url)
-      })
-  }
-
-  handleHttpCall (url, cleanUrl, cb) {
-    return request.get(
-      {
-        url
-      },
-    (err, r, data) => {
-      if (err) {
-        return cb(err || 'Unknown transport error')
-      }
-      cb(null, data, cleanUrl)
-    })
-  }
-
-  handlers () {
-    return [
-      { type: 'github', match: /^(https?:\/\/)?(www.)?github.com\/([^/]*\/[^/]*)\/(.*)/, handler: (match, cb) => { this.handleGithubCall(match[3], match[4], cb) } },
-      { type: 'http', match: /^(http?:\/\/?(.*))$/, handler: (match, cb) => { this.handleHttpCall(match[1], match[2], cb) } },
-      { type: 'https', match: /^(https?:\/\/?(.*))$/, handler: (match, cb) => { this.handleHttpCall(match[1], match[2], cb) } },
-      { type: 'swarm', match: /^(bzz-raw?:\/\/?(.*))$/, handler: (match, cb) => { this.handleSwarmImport(match[1], match[2], cb) } },
-      { type: 'ipfs', match: /^(ipfs:\/\/?.+)/, handler: (match, cb) => { this.handleIPFS(match[1], cb) } }
-    ]
-  }
-
   isRelativeImport (url) {
     return /^([^/]+)/.exec(url)
   }
 
-  import (url, loadingCb, cb) {
-    var self = this
-    var imported = this.previouslyHandled[url]
+  import (uri, loadingCb, cb) {
+    var imported = this.previouslyHandled[uri]
     if (imported) {
-      return cb(null, imported.content, imported.cleanUrl, imported.type, url)
+      return cb(null, imported.content, imported.cleanUrl, imported.type, uri)
     }
-    var handlers = this.handlers()
-
-    var found = false
-    handlers.forEach(function (handler) {
-      if (found) {
-        return
-      }
-
-      var match = handler.match.exec(url)
-      if (match) {
-        found = true
 
-        loadingCb('Loading ' + url + ' ...')
-        handler.handler(match, function (err, content, cleanUrl) {
-          if (err) {
-            cb('Unable to import "' + cleanUrl + '": ' + err)
-            return
-          }
-          self.previouslyHandled[url] = {
-            content: content,
-            cleanUrl: cleanUrl,
-            type: handler.type
-          }
-          cb(null, content, cleanUrl, handler.type, url)
-        })
-      }
-    })
-
-    if (found) {
-      return
-    } else if (/^[^:]*:\/\//.exec(url)) {
-      cb('Unable to import "' + url + '": Unsupported URL schema')
-    } else {
-      cb('Unable to import "' + url + '": File not found')
-    }
+    var self = this
+    resolver
+     .resolve(uri)
+     .then(result => {
+       loadingCb('Loading ' + uri + ' ...')
+       return resolver.require(uri)
+     })
+     .then(result => {
+       var cleanUrl = result.url
+       var content = result.source
+       var type = result.provider
+       self.previouslyHandled[uri] = {
+         content,
+         cleanUrl,
+         type
+       }
+       cb(null, content, cleanUrl, type, uri)
+     })
+     .catch(err => {
+       console.log('import', err) // TODO remove this and all my other console logs
+       cb('Unable to import "' + uri + '": File not found')
+     })
   }
 }

src/app/files/NotPersistedExplorer.js

@@ -1,13 +1,93 @@
 'use strict'
-var ReadOnlyExplorer = require('./basicReadOnlyExplorer')
 var toolTip = require('../ui/tooltip')
+var EventManager = require('../../lib/events')
 
-class NotPersistedExplorer extends ReadOnlyExplorer {
+class NotPersistedExplorer {
   constructor (type) {
-    super(type)
+    this.event = new EventManager()
+    this.files = {}
+    this.paths = {}
+    this.normalizedNames = {} // contains the raw url associated with the displayed path
+    this.paths[type] = {}
+    this.type = type
     this.readonly = false
   }
 
+  close (cb) {
+    this.files = {}
+    cb()
+  }
+
+  init (cb) {
+    this.files = {}
+  }
+
+  exists (path, cb) {
+    if (!this.files) return cb(null, false)
+    var unprefixedPath = this.removePrefix(path)
+    cb(null, this.files[unprefixedPath] !== undefined)
+  }
+
+  get (path, cb) {
+    if (this.normalizedNames[path]) path = this.normalizedNames[path] // ensure we actually use the normalized path from here
+    var unprefixedPath = this.removePrefix(path)
+    var content = this.files[unprefixedPath]
+    if (!content) {
+      content = this.files[this.type + '/' + this.normalizedNames[path]]
+    }
+    if (cb) {
+      cb(null, content)
+    }
+    return content
+  }
+
+  set (path, content, cb) {
+    var unprefixedPath = this.removePrefix(path)
+    this.addReadOnly(unprefixedPath, content)
+    if (cb) cb()
+    return true
+  }
+
+  addReadOnly (path, content, rawPath) {
+    try { // lazy try to format JSON
+      content = JSON.stringify(JSON.parse(content), null, '\t')
+    } catch (e) {}
+    if (!rawPath) rawPath = path
+    // splitting off the path in a tree structure, the json tree is used in `resolveDirectory`
+    var split = path
+    var folder = false
+    while (split.lastIndexOf('/') !== -1) {
+      var subitem = split.substring(split.lastIndexOf('/'))
+      split = split.substring(0, split.lastIndexOf('/'))
+      if (!this.paths[this.type + '/' + split]) {
+        this.paths[this.type + '/' + split] = {}
+      }
+      this.paths[this.type + '/' + split][split + subitem] = { isDirectory: folder }
+      folder = true
+    }
+    this.paths[this.type][split] = { isDirectory: folder }
+    this.files[path] = content
+    this.normalizedNames[rawPath] = path
+    this.event.trigger('fileAdded', [path, true])
+    return true
+  }
+
+  list () {
+    return this.files
+  }
+
+  resolveDirectory (path, callback) {
+    var self = this
+    if (path[0] === '/') path = path.substring(1)
+    if (!path) return callback(null, { [self.type]: { } })
+    // we just return the json tree populated by `addReadOnly`
+    callback(null, this.paths[path])
+  }
+
+  removePrefix (path) {
+    return path.indexOf(this.type + '/') === 0 ? path.replace(this.type + '/', '') : path
+  }
+
   remove (path) {
     var unprefixedPath = this.removePrefix(path)
     var folderPath = path.substring(0, path.lastIndexOf('/'))
@@ -31,8 +111,8 @@ class NotPersistedExplorer extends ReadOnlyExplorer {
     })
   }
 
-  isReadOnly (path) {
-    return false
+  isReadOnly () {
+    return this.readonly
   }
 }
 

src/app/files/browser-files.js

@@ -8,6 +8,8 @@ function Files (storage) {
   var readonly = {}
   this.type = 'browser'
 
+  this.storage = storage
+
   this.exists = function (path, cb) {
     cb(null, this._exists(path))
   }

src/app/files/file-explorer.js

@@ -84,6 +84,22 @@ function fileExplorer (localRegistry, files) {
           }
         })
       }
+
+      // after adding the file, immediately show it in the tree view
+      try {
+        var allPaths = Object.keys(self.files.paths)
+        allPaths = allPaths.sort((a, b) => a.length - b.length)
+        allPaths.forEach(path => {
+          self.files.resolveDirectory(path, (error, fileTree) => {
+            if (error) console.error(error)
+            if (!fileTree) return
+            var newTree = normalize(path, fileTree)
+            self.treeView.updateNodeFromJSON(path, newTree, true)
+          })
+        })
+      } catch (err) {
+        console.log('Jumping to ', filepath)
+      }
     })
   }
 

src/app/files/fileManager.js

@@ -29,7 +29,8 @@ class FileManager {
       localhostExplorer: self._components.registry.get('fileproviders/localhost').api,
       configExplorer: self._components.registry.get('fileproviders/config').api,
       gistExplorer: self._components.registry.get('fileproviders/gist').api,
-      filesProviders: self._components.registry.get('fileproviders').api
+      filesProviders: self._components.registry.get('fileproviders').api,
+      readOnly: self._components.registry.get('fileproviders/readonly').api
     }
 
     self._deps.browserExplorer.event.register('fileRenamed', (oldName, newName, isFolder) => { this.fileRenamedEvent(oldName, newName, isFolder) })
@@ -42,6 +43,9 @@ class FileManager {
     self._deps.gistExplorer.event.register('fileRemoved', (path) => { this.fileRemovedEvent(path) })
     self._deps.localhostExplorer.event.register('errored', (event) => { this.removeTabsOf(self._deps.localhostExplorer) })
     self._deps.localhostExplorer.event.register('closed', (event) => { this.removeTabsOf(self._deps.localhostExplorer) })
+
+    self._deps.readOnly.event.register('fileRenamed', (oldName, newName, isFolder) => { this.fileRenamedEvent(oldName, newName, isFolder) })
+    self._deps.readOnly.event.register('fileRemoved', (path) => { this.fileRemovedEvent(path) })
   }
 
   fileRenamedEvent (oldName, newName, isFolder) {
@@ -151,10 +155,16 @@ class FileManager {
       })
     }
     function _switchFile (file) {
+      var provider = self.fileProviderOf(file)
+      if (!provider) {
+        console.error(`Provider for ${file} does not exist`)
+        return
+      }
       self.saveCurrentFile()
       self._deps.config.set('currentFile', file)
       self.refreshTabs(file)
-      self.fileProviderOf(file).get(file, (error, content) => {
+
+      provider.get(file, (error, content) => {
         if (error) {
           console.log(error)
         } else {
@@ -183,13 +193,8 @@ class FileManager {
     if (provider !== null && this._deps.filesProviders[provider[0]]) {
       return this._deps.filesProviders[provider[0]]
     } else {
-      for (var handler of this._components.compilerImport.handlers()) {
-        if (handler.match.exec(file)) {
-          return this._deps.filesProviders[handler.type]
-        }
-      }
+      return this._deps.readOnly
     }
-    return null
   }
 
   saveCurrentFile () {