Merge pull request #79 from threefoldtech/flist-local-sync

readonly: support readonly mode

Author: maxux

src/config.py.sample

@@ -85,9 +85,17 @@ config = {
     ##
     # 'userdata-root-path': '/opt/0-hub/public',
     # 'workdir-root-path': '/opt/0-hub/workdir',
- 
+
+    ## In order to host a hub which is a mirror of another
+    ## hub, since hub right now don't support multiple master,
+    ## you should enable this read-only mode to avoid modifying
+    ## your local database and be able to stay synced with your
+    ## main hub instance. For a normal or main hub instance,
+    ## keep this setting to False
+    'readonly': False,
+
     ## By default, the hub is made to be used publicly
-    ## and needs to be protected (with itsyou.online)
+    ## and needs to be protected (with threefold connect)
     ##
     ## If you are running a local test hub on your local
     ## network and you don't have any security issue, you

src/flist-uploader.py

@@ -272,6 +272,13 @@ def logout():
 def login_method():
     return internalRedirect("logins.html")
 
+@app.route('/readonly')
+def read_only_mode():
+    if not config['readonly']:
+        return redirect("/")
+
+    return globalTemplate("readonly.html", {})
+
 @app.route('/login-iyo')
 @hub.itsyouonline.force_login()
 def login_iyo():

src/hub/security.py

@@ -8,6 +8,9 @@ def apicall():
     def decorator(handler):
         @wraps(handler)
         def _wrapper(*args, **kwargs):
+            if config['readonly']:
+                return jsonify({"message": "readonly mode", "status": "error"}), 400
+
             if not config['authentication']:
                 session['accounts'] = ['Administrator']
                 session['username'] = 'Administrator'
@@ -48,6 +51,9 @@ def protected():
     def decorator(handler):
         @wraps(handler)
         def _wrapper(*args, **kwargs):
+            if config['readonly']:
+                return redirect("/readonly")
+
             if not config['authentication']:
                 session['accounts'] = ['Administrator']
                 session['username'] = 'Administrator'

src/templates/readonly.html

@@ -0,0 +1,13 @@
+{% extends "layout.html" %}
+{% block title %}Zero-OS Hub{% endblock %}
+
+{% block content %}
+<div class="jumbotron">
+    <div class="container">
+        <h1>Instance modification disabled</h1>
+        <p>This Hub instance doesn't allow modification, this is a <code>read-only hub instance</code>.</p>
+        <p>This is probably a mirror of a main instance, you should point to that one, or
+        this instance is a special serve-only instance.<p>
+    </div>
+</div>
+{% endblock %}

tools/flist-copy-local.py

@@ -0,0 +1,111 @@
+import redis
+import tempfile
+import requests
+import subprocess
+import json
+import os
+import sys
+
+class FListRemoteClone:
+    def __init__(self, host, port, basehub="https://hub.grid.tf"):
+        self.host = host
+        self.port = port
+        self.basehub = basehub
+
+        self.local = redis.Redis(host, port)
+        self.remote = redis.Redis("hub.grid.tf", 9900)
+
+        self.zflist = "/home/maxux/git/0-flist/zflist/zflist"
+        self.backend = ""
+        self.workdir = tempfile.mkdtemp(prefix="zflist-cloning")
+        self.tempdir = tempfile.mkdtemp(prefix="zflist-source")
+
+        print(self.workdir)
+        print(self.tempdir)
+
+        self.environ = dict(
+            os.environ,
+            ZFLIST_JSON="1",
+            ZFLIST_MNT=self.workdir
+        )
+
+    def authenticate(self):
+        pass
+
+    def download(self, target):
+        url = f"{self.basehub}/{target}"
+        destination = f"{self.tempdir}/download.flist"
+
+        print(f"[+] fetching: {url}")
+
+        r = requests.get(url)
+        with open(destination, "wb") as f:
+            f.write(r.content)
+
+        length = len(r.content) / 1024
+        print(f"[+] fetched {length:.2f} KB into {destination}")
+
+        return destination
+
+    def execute(self, args):
+        command = [self.zflist] + args
+
+        print(command)
+        p = subprocess.Popen(command, env=self.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        (output, err) = p.communicate()
+
+        return json.loads(output)
+
+    def chunks(self, flist):
+        self.execute(["open", flist])
+        reply = self.execute(["chunks"])
+
+        chunks = reply["response"]["content"]
+        bchunks = []
+
+        for chunk in chunks:
+            bchunk = bytes.fromhex(chunk)
+            bchunks.append(bchunk)
+
+        return bchunks
+
+    def metadata(self):
+        pass
+
+    def commit(self, destination):
+        self.execute(["commit", destination])
+
+    def sync(self, chunks):
+        proceed = 0
+        total = len(chunks)
+
+        print("[+] syncing database...")
+
+        for chunk in chunks:
+            data = self.remote.get(chunk)
+            self.local.execute_command("SETX", chunk, data)
+
+            proceed += 1
+            percent = (proceed / total) * 100
+
+            sys.stdout.write(f"\r[+] syncing database: {proceed} / {total} [{percent:.2f} %%]")
+            sys.stdout.flush()
+
+        print("[+] database synchronized")
+
+    def clone(self, target):
+        flist = self.download(target)
+        chunks = self.chunks(flist)
+        self.sync(chunks)
+        self.metadata()
+        self.commit("/tmp/destination.flist")
+
+
+if len(sys.argv) < 2:
+    print("[-] missing target flist to clone")
+    sys.exit(1)
+
+target = sys.argv[1]
+
+x = FListRemoteClone("127.0.0.1", 9900)
+x.clone(target)