Merge pull request #2373 from threefoldtech/add-failover-graphql

Add multiple graphql endpoints

Author: Eslam-Nawara

pkg/environment/environment.go

@@ -49,7 +49,7 @@ type Environment struct {
 	//   PLEASE MAKE SURE THAT ANY ENV HAS NO MORE THAN FOUR RELAYS CONFIGURED
 	RelayURL      []string
 	ActivationURL string
-	GraphQL       string
+	GraphQL       []string
 
 	// private vlan to join
 	// if set, zos will use this as its priv vlan
@@ -116,7 +116,10 @@ var (
 		ActivationURL: "https://activation.dev.grid.tf/activation/activate",
 		FlistURL:      "redis://hub.grid.tf:9900",
 		BinRepo:       "tf-zos-v3-bins.dev",
-		GraphQL:       "https://graphql.dev.grid.tf/graphql",
+		GraphQL: []string{
+			"https://graphql.dev.grid.tf/graphql",
+			"https://graphql.02.dev.grid.tf/graphql",
+		},
 	}
 
 	envTest = Environment{
@@ -131,7 +134,10 @@ var (
 		ActivationURL: "https://activation.test.grid.tf/activation/activate",
 		FlistURL:      "redis://hub.grid.tf:9900",
 		BinRepo:       "tf-zos-v3-bins.test",
-		GraphQL:       "https://graphql.test.grid.tf/graphql",
+		GraphQL: []string{
+			"https://graphql.test.grid.tf/graphql",
+			"https://graphql.02.test.grid.tf/graphql",
+		},
 	}
 
 	envQA = Environment{
@@ -146,7 +152,10 @@ var (
 		ActivationURL: "https://activation.qa.grid.tf/activation/activate",
 		FlistURL:      "redis://hub.grid.tf:9900",
 		BinRepo:       "tf-zos-v3-bins.qanet",
-		GraphQL:       "https://graphql.qa.grid.tf/graphql",
+		GraphQL: []string{
+			"https://graphql.qa.grid.tf/graphql",
+			"https://graphql.02.qa.grid.tf/graphql",
+		},
 	}
 
 	envProd = Environment{
@@ -164,7 +173,10 @@ var (
 		ActivationURL: "https://activation.grid.tf/activation/activate",
 		FlistURL:      "redis://hub.grid.tf:9900",
 		BinRepo:       "tf-zos-v3-bins",
-		GraphQL:       "https://graphql.grid.tf/graphql",
+		GraphQL: []string{
+			"https://graphql.grid.tf/graphql",
+			"https://graphql.02.grid.tf/graphql",
+		},
 	}
 )
 

pkg/perf/graphql/graphql_nodes.go

@@ -2,6 +2,7 @@ package graphql
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math/rand"
 	"time"
@@ -16,13 +17,15 @@ const (
 
 // GraphQl for tf graphql client
 type GraphQl struct {
-	client *graphql.Client
+	urls []string
 }
 
 // NewGraphQl creates a new tf graphql client
-func NewGraphQl(url string) GraphQl {
-	client := graphql.NewClient(url, nil)
-	return GraphQl{client: client}
+func NewGraphQl(urls ...string) (GraphQl, error) {
+	if len(urls) == 0 {
+		return GraphQl{}, errors.New("urls can't be empty")
+	}
+	return GraphQl{urls: urls}, nil
 }
 
 // Node from graphql
@@ -85,7 +88,7 @@ func (g *GraphQl) GetUpNodes(ctx context.Context, nodesNum int, farmID, excludeF
 		Nodes []Node
 	}{}
 
-	if err := g.client.Exec(ctx, query, &res, nil); err != nil {
+	if err := g.exec(ctx, query, &res, nil); err != nil {
 		return []Node{}, err
 	}
 
@@ -101,9 +104,20 @@ func (g *GraphQl) getItemTotalCount(ctx context.Context, itemName string, option
 		}
 	}{}
 
-	if err := g.client.Exec(ctx, query, &res, nil); err != nil {
+	if err := g.exec(ctx, query, &res, nil); err != nil {
 		return 0, err
 	}
 
 	return res.Items.Count, nil
 }
+
+// exec is a wrapper around graphql.Client.Exec to retry another endpoints in case some are down.
+func (g *GraphQl) exec(ctx context.Context, query string, result interface{}, variables map[string]interface{}, options ...graphql.Option) (err error) {
+	for _, url := range g.urls {
+		client := graphql.NewClient(url, nil)
+		if err = client.Exec(ctx, query, result, variables, options...); err == nil {
+			return
+		}
+	}
+	return
+}

pkg/perf/healthcheck/network.go

@@ -18,9 +18,10 @@ const defaultRequestTimeout = 5 * time.Second
 func networkCheck(ctx context.Context) []error {
 	env := environment.MustGet()
 	servicesUrl := []string{
-		env.ActivationURL, env.GraphQL, env.FlistURL,
+		env.ActivationURL, env.FlistURL,
 	}
 	servicesUrl = append(append(servicesUrl, env.SubstrateURL...), env.RelayURL...)
+	servicesUrl = append(servicesUrl, env.GraphQL...)
 
 	var errors []error
 

pkg/perf/iperf/iperf_task.go

@@ -76,7 +76,10 @@ func (t *IperfTest) Jitter() uint32 {
 // Run runs the tcp test and returns the result
 func (t *IperfTest) Run(ctx context.Context) (interface{}, error) {
 	env := environment.MustGet()
-	g := graphql.NewGraphQl(env.GraphQL)
+	g, err := graphql.NewGraphQl(env.GraphQL...)
+	if err != nil {
+		return nil, err
+	}
 
 	// get public up nodes
 	freeFarmNodes, err := g.GetUpNodes(ctx, 0, 1, 0, true, true)

pkg/perf/publicip/publicip_task.go

@@ -34,11 +34,15 @@ const (
 	FetchRealIPFailed   = "failed to get real public IP to the node"
 )
 
-var errPublicIPLookup = errors.New("failed to reach public ip service")
-var errSkippedValidating = errors.New("skipped, there is a node with less ID available")
+var (
+	errPublicIPLookup    = errors.New("failed to reach public ip service")
+	errSkippedValidating = errors.New("skipped, there is a node with less ID available")
+)
 
-const testMacvlan = "pub"
-const testNamespace = "pubtestns"
+const (
+	testMacvlan   = "pub"
+	testNamespace = "pubtestns"
+)
 
 type publicIPValidationTask struct{}
 
@@ -96,7 +100,6 @@ func (p *publicIPValidationTask) Run(ctx context.Context) (interface{}, error) {
 		report, err = p.validateIPs(farm.PublicIPs)
 		return err
 	})
-
 	if err != nil {
 		return nil, fmt.Errorf("failed to run public IP validation: %w", err)
 	}
@@ -179,7 +182,10 @@ func (p *publicIPValidationTask) validateIPs(publicIPs []substrate.PublicIP) (ma
 
 func isLeastValidNode(ctx context.Context, farmID uint32, substrateGateway *stubs.SubstrateGatewayStub) (bool, error) {
 	env := environment.MustGet()
-	gql := graphql.NewGraphQl(env.GraphQL)
+	gql, err := graphql.NewGraphQl(env.GraphQL...)
+	if err != nil {
+		return false, err
+	}
 
 	nodes, err := gql.GetUpNodes(ctx, 0, farmID, 0, false, false)
 	if err != nil {
@@ -196,7 +202,6 @@ func isLeastValidNode(ctx context.Context, farmID uint32, substrateGateway *stub
 		}
 		return nil
 	}, backoff.NewConstantBackOff(10*time.Second))
-
 	if err != nil {
 		return false, fmt.Errorf("failed to get node id: %w", err)
 	}