Merge pull request #71 from mitre/resolve-gcp-loc

Resolve location on GCP
mitre · Apr 10, 2018 · bb2e3f0 · bb2e3f0
2 parents e01dcbb + 5c285b8
commit bb2e3f0
Show file tree

Hide file tree

Showing 140 changed files with 46,596 additions and 133 deletions.
diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/awsutil/aws.go b/awsutil/aws.go
@@ -15,11 +15,13 @@
 package awsutil
 
 import (
+	"encoding/json"
 	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
+	"path/filepath"
 	"strings"
 	"syscall"
 	"time"
@@ -142,6 +144,98 @@ func ReadNgcFile(path string) ([]byte, error) {
 	return bytes, err
 }
 
+func ResolveRegion() (string, error) {
+	// Attempt to resolve the location on aws or gs.
+	loc, err := resolveAwsRegion()
+	if err != nil {
+		// could be on google
+		// retain aws error message
+		msg := err.Error()
+		loc, err = resolveGcpZone()
+		if err != nil {
+			// return both aws and google error messages
+			return "", errors.Wrap(err, msg)
+		}
+	}
+	return loc, nil
+}
+
+func resolveAwsRegion() (string, error) {
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   1 * time.Second,
+				KeepAlive: 1 * time.Second,
+				DualStack: true,
+			}).DialContext,
+			MaxIdleConns:          1000,
+			MaxIdleConnsPerHost:   1000,
+			IdleConnTimeout:       500 * time.Millisecond,
+			TLSHandshakeTimeout:   500 * time.Millisecond,
+			ExpectContinueTimeout: 500 * time.Millisecond,
+		},
+	}
+	// maybe we are on an AWS instance and can resolve what region we are in.
+	// let's try it out and if we timeout we'll return an error.
+	// use this url: http://169.254.169.254/latest/dynamic/instance-identity/document
+	resp, err := client.Get("http://169.254.169.254/latest/dynamic/instance-identity/document")
+	if err != nil {
+		return "", errors.Wrapf(err, "location was not provided, fusera attempted to resolve region but encountered an error, this feature only works when fusera is on an amazon or google instance")
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", errors.Errorf("issue trying to resolve region, got: %d: %s", resp.StatusCode, resp.Status)
+	}
+	var payload struct {
+		Region string `json:"region"`
+	}
+	err = json.NewDecoder(resp.Body).Decode(&payload)
+	if err != nil {
+		return "", errors.New("issue trying to resolve region, couldn't decode response from amazon")
+	}
+	if payload.Region == "" {
+		return "", errors.New("issue trying to resolve region, amazon returned empty region")
+	}
+	return "s3." + payload.Region, nil
+}
+
+func resolveGcpZone() (string, error) {
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   1 * time.Second,
+				KeepAlive: 1 * time.Second,
+				DualStack: true,
+			}).DialContext,
+			MaxIdleConns:          1000,
+			MaxIdleConnsPerHost:   1000,
+			IdleConnTimeout:       500 * time.Millisecond,
+			TLSHandshakeTimeout:   500 * time.Millisecond,
+			ExpectContinueTimeout: 500 * time.Millisecond,
+		},
+	}
+	req, err := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/instance/zone?alt=json", nil)
+	req.Header.Add("Metadata-Flavor", "Google")
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", errors.Wrapf(err, "location was not provided, fusera attempted to resolve region but encountered an error, this feature only works when fusera is on an amazon or google instance")
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", errors.Errorf("issue trying to resolve region, got: %d: %s", resp.StatusCode, resp.Status)
+	}
+	var payload string
+	err = json.NewDecoder(resp.Body).Decode(&payload)
+	if err != nil {
+		return "", errors.New("issue trying to resolve region, couldn't decode response from google")
+	}
+	path := filepath.Base(payload)
+	if path == "" || len(path) == 1 {
+		return "", errors.New("issue trying to resolve region, google returned empty region")
+	}
+	return "gs." + path, nil
+}
+
 func parseHTTPError(code int) error {
 	switch code {
 	case 400:
@@ -166,6 +260,85 @@ func parseHTTPError(code int) error {
 	}
 }
 
+var Directory = CloudDirectory{
+	"gs": map[string]bool{
+		"US": true,
+
+		"us-east1-b": true,
+		"us-east1-c": true,
+		"us-east1-d": true,
+
+		"us-east4-a": true,
+		"us-east4-b": true,
+		"us-east4-c": true,
+
+		"us-central1-a": true,
+		"us-central1-b": true,
+		"us-central1-c": true,
+		"us-central1-f": true,
+
+		"us-west1-a": true,
+		"us-west1-b": true,
+		"us-west1-c": true,
+	},
+	"s3": map[string]bool{
+		"us-east-1": true,
+	},
+}
+
+type CloudDirectory map[string]map[string]bool
+
+var IncorrectLocationMessage = `
+================
+gs.[region]
+================
+
+regions for gs:
+----------------
+
+US
+
+us-east1-b us-east1-c us-east1-d
+
+us-east4-a us-east4-b us-east4-c
+
+us-central1-a us-central1-b us-central1-c us-central1-f
+
+us-west1-a us-west1-b us-west1-c
+
+================
+s3.[region]
+================
+
+regions for s3:
+----------------
+
+us-east-1
+
+================
+For accessing files on ncbi, use the location ftp-ncbi
+================
+
+`
+
+func IsLocation(loc string) bool {
+	ll := strings.Split(loc, ".")
+	if len(ll) != 2 {
+		if loc == "ftp-ncbi" {
+			return true
+		}
+		return false
+	}
+	regions, ok := Directory[ll[0]]
+	if !ok {
+		return false
+	}
+	if _, ok := regions[ll[1]]; !ok {
+		return false
+	}
+	return true
+}
+
 func String(s string) *string {
 	return &s
 }

diff --git a/cmd/fusera/flags.go b/cmd/fusera/flags.go
@@ -17,11 +17,9 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"os/user"
 	"strconv"
@@ -114,9 +112,6 @@ func NewApp() (app *cli.App, cmd *Commands) {
 				Aliases: []string{"m"},
 				Usage:   "to mount a folder",
 				Action: func(c *cli.Context) error {
-					// if c.IsSet("help") {
-					// 	cli.ShowAppHelpAndExit(c, 0)
-					// }
 					cmd.IsMount = true
 					twig.SetDebug(c.IsSet("debug"))
 					// Populate and parse flags.
@@ -310,15 +305,14 @@ func PopulateMountFlags(c *cli.Context) (ret *Flags, err error) {
 	}
 	loc := c.String("loc")
 	if !c.IsSet("loc") {
-		// Attempt to resolve the location on aws or gs.
-		loc, err = resolveLocation()
+		loc, err = awsutil.ResolveRegion()
 		if err != nil {
 			return nil, err
 		}
 	}
-	loc, err = parseLocation(loc)
-	if err != nil {
-		return nil, err
+	ok := awsutil.IsLocation(loc)
+	if !ok {
+		return nil, errors.Errorf("gave location of %s, location must match one of these possibilities:\n%s", loc, awsutil.IncorrectLocationMessage)
 	}
 	f.Loc = loc
 
@@ -334,54 +328,6 @@ func PopulateMountFlags(c *cli.Context) (ret *Flags, err error) {
 	return f, nil
 }
 
-func resolveLocation() (string, error) {
-	// maybe we are on an AWS instance and can resolve what region we are in.
-	// let's try it out and if we timeout we'll return an error.
-	// use this url: http://169.254.169.254/latest/dynamic/instance-identity/document
-	resp, err := http.Get("http://169.254.169.254/latest/dynamic/instance-identity/document")
-	if err != nil {
-		return "", errors.Wrapf(err, "location was not provided, fusera attempted to resolve region but encountered an error, this feature only works when fusera is on an amazon instance")
-	}
-	if resp.StatusCode != http.StatusOK {
-		return "", errors.Errorf("issue trying to resolve region, got: %d: %s", resp.StatusCode, resp.Status)
-	}
-	var payload struct {
-		Region string `json:"region"`
-	}
-	err = json.NewDecoder(resp.Body).Decode(&payload)
-	if err != nil {
-		return "", errors.New("issue trying to resolve region, couldn't decode response from amazon")
-	}
-	if payload.Region == "" {
-		return "", errors.New("issue trying to resolve region, amazon returned empty region")
-	}
-	return "s3." + payload.Region, nil
-}
-
-func parseLocation(loc string) (string, error) {
-	ll := strings.Split(loc, ".")
-	if len(ll) != 2 {
-		if loc == "ftp-ncbi" {
-			return loc, nil
-		}
-		return "", errors.New("location must be either: gs.US, s3.us-east-1, or ftp-ncbi")
-	}
-	if ll[0] != "gs" && ll[0] != "s3" {
-		return "", errors.Errorf("the service %s is not supported, please use gs or s3", ll[0])
-	}
-	if ll[0] == "gs" {
-		if ll[1] != "US" {
-			return "", errors.Errorf("the region %s isn't supported on gs, only US", ll[1])
-		}
-	}
-	if ll[0] == "s3" {
-		if ll[1] != "us-east-1" {
-			return "", errors.Errorf("the region %s isn't supported on s3, only us-east-1", ll[1])
-		}
-	}
-	return loc, nil
-}
-
 func MassageMountFlags(args []string) (ret []string) {
 	if len(args) == 5 && args[3] == "-o" {
 		// looks like it's coming from fstab!

diff --git a/cmd/fusera/main.go b/cmd/fusera/main.go
@@ -127,7 +127,7 @@ func main() {
 	app, cmd := NewApp()
 	err := app.Run(MassageMountFlags(os.Args))
 	if err != nil {
-		fmt.Printf("parsing arguments failed, please review the help with -h:\n %s\n", err.Error())
+		fmt.Println("starting fusera with given arguments failed, please review the help with -h")
 		twig.Debugf("%+v\n", err)
 		os.Exit(1)
 	}
@@ -140,15 +140,11 @@ func main() {
 			fmt.Println("Fusera failed to mount the file system")
 			if strings.Contains(err.Error(), "no such file or directory") {
 				fmt.Println("It seems like the directory you want to mount to does not exist or you do not have correct permissions to access it. Please create the directory or correct the permissions on it before trying again.")
-				twig.Debugf("%+v\n", err)
-				os.Exit(1)
 			}
 			if strings.Contains(err.Error(), "EOF") {
 				fmt.Println("It seems like the directory you want to mount to is already mounted by fusera or another device. Choose another directory or try using the unmount command before trying again. Be considerate of the unmount command, if anything is using the device mounted while attempting to unmount, it will fail.")
-				twig.Debugf("%+v\n", err)
-				os.Exit(1)
 			}
-			fmt.Printf("%s\n", err.Error())
+			fmt.Println("Details: " + err.Error())
 			twig.Debugf("%+v\n", err)
 			os.Exit(1)
 		}