Support for CouchDB and PouchDB backends

README.md

@@ -26,7 +26,7 @@ Its goal is to be a part of the developer's toolbox where [Linked Data](http://l
   * (simplified) [MQL](./docs/MQL.md), for [Freebase](https://en.wikipedia.org/wiki/Freebase) fans
 * Plays well with multiple backend stores:
   * KVs: [Bolt](https://github.com/boltdb/bolt), [LevelDB](https://github.com/google/leveldb)
-  * NoSQL: [MongoDB](https://www.mongodb.org), [ElasticSearch](https://www.elastic.co/products/elasticsearch)
+  * NoSQL: [MongoDB](https://www.mongodb.org), [ElasticSearch](https://www.elastic.co/products/elasticsearch), [CouchDB](http://couchdb.apache.org/)/[PouchDB](https://pouchdb.com/)
   * SQL: [PostgreSQL](http://www.postgresql.org), [CockroachDB](https://www.cockroachlabs.com), [MySQL](https://www.mysql.com)
   * In-memory, ephemeral
 * Modular design; easy to extend with new languages and backends

docs/Configuration.md

@@ -37,7 +37,9 @@ All command line flags take precedence over the configuration file.
 
   * `mongo`: Stores the graph data and indices in a [MongoDB](https://www.mongodb.com/) instance.
   * `elastic`: Stores the graph data and indices in a [ElasticSearch](https://www.elastic.co/products/elasticsearch) instance.
-
+  * `couch`: Stores the graph data and indices in a [CouchDB](http://couchdb.apache.org/) instance.
+  * `pouch`: Stores the graph data and indices in a [PouchDB](https://pouchdb.com/). Requires building with [GopherJS](https://github.com/gopherjs/gopherjs).
+
   **SQL backends**
 
   * `postgres`: Stores the graph data and indices in a [PostgreSQL](https://www.postgresql.org) instance.
@@ -56,7 +58,8 @@ All command line flags take precedence over the configuration file.
   * `leveldb`: Directory to hold the LevelDB database files.
   * `bolt`: Path to the persistent single Bolt database file.
   * `mongo`: "hostname:port" of the desired MongoDB server. More options can be provided in [mgo](https://godoc.org/gopkg.in/mgo.v2#Dial) address format.
-  * `elastic`: "http://host:port" of the desired ElasticSearch server.
+  * `elastic`: `http://host:port` of the desired ElasticSearch server.
+  * `couch`: `http://user:pass@host:port/dbname` of the desired CouchDB server.
   * `postgres`,`cockroach`: `postgres://[username:password@]host[:port]/database-name?sslmode=disable` of the PostgreSQL database and credentials. Sslmode is optional. More option available on [pq](https://godoc.org/github.com/lib/pq) page.
   * `mysql`: `[username:password@]tcp(host[:3306])/database-name` of the MqSQL database and credentials. More option available on [driver](https://github.com/go-sql-driver/mysql#dsn-data-source-name) page.
 

glide.yaml

@@ -48,3 +48,5 @@ import:
 - package: github.com/dennwc/graphql
 - package: github.com/tylertreat/BoomFilters
 - package: gopkg.in/olivere/elastic.v5
+- package: github.com/go-kivik/kivik
+- package: github.com/go-kivik/couchdb

graph/all/all.go

@@ -8,6 +8,7 @@ import (
 	_ "github.com/cayleygraph/cayley/graph/memstore"
 	_ "github.com/cayleygraph/cayley/graph/nosql/elastic"
 	_ "github.com/cayleygraph/cayley/graph/nosql/mongo"
+	_ "github.com/cayleygraph/cayley/graph/nosql/ouch"
 	_ "github.com/cayleygraph/cayley/graph/sql/cockroach"
 	_ "github.com/cayleygraph/cayley/graph/sql/mysql"
 	_ "github.com/cayleygraph/cayley/graph/sql/postgres"

graph/graphtest/graphtest.go

@@ -2,6 +2,7 @@ package graphtest
 
 import (
 	"context"
+	"math"
 	"sort"
 	"testing"
 	"time"
@@ -24,6 +25,7 @@ type Config struct {
 	TimeInMs     bool
 	TimeInMcs    bool
 	TimeRound    bool
+	PageSize     int // result page size for pagination (large iterator) tests
 
 	OptimizesComparison bool
 
@@ -64,6 +66,9 @@ func TestAll(t *testing.T, gen testutil.DatabaseFunc, conf *Config) {
 	t.Run("writers", func(t *testing.T) {
 		TestWriters(t, gen, conf)
 	})
+	t.Run("1k", func(t *testing.T) {
+		Test1K(t, gen, conf)
+	})
 	t.Run("paths", func(t *testing.T) {
 		pathtest.RunTestMorphisms(t, gen)
 	})
@@ -115,6 +120,9 @@ func IteratedQuads(t testing.TB, qs graph.QuadStore, it graph.Iterator) []quad.Q
 	}
 	require.Nil(t, it.Err())
 	sort.Sort(res)
+	if res == nil {
+		return []quad.Quad(nil) // GopherJS seems to have a bug with this type conversion for a nil value
+	}
 	return res
 }
 
@@ -124,6 +132,9 @@ func ExpectIteratedQuads(t testing.TB, qs graph.QuadStore, it graph.Iterator, ex
 		sort.Sort(quad.ByQuadString(exp))
 		sort.Sort(quad.ByQuadString(got))
 	}
+	if len(exp) == 0 {
+		exp = nil // GopherJS seems to have a bug with nil value
+	}
 	require.Equal(t, exp, got)
 }
 
@@ -134,16 +145,21 @@ func ExpectIteratedRawStrings(t testing.TB, qs graph.QuadStore, it graph.Iterato
 	require.Equal(t, exp, got)
 }
 
-func ExpectIteratedValues(t testing.TB, qs graph.QuadStore, it graph.Iterator, exp []quad.Value) {
+func ExpectIteratedValues(t testing.TB, qs graph.QuadStore, it graph.Iterator, exp []quad.Value, sortVals bool) {
 	//sort.Strings(exp)
 	got := IteratedValues(t, qs, it)
 	//sort.Strings(got)
+	if sortVals {
+		exp = append([]quad.Value{}, exp...)
+		sort.Sort(quad.ByValueString(exp))
+	}
+
 	require.Equal(t, len(exp), len(got), "%v\nvs\n%v", exp, got)
 	for i := range exp {
 		if eq, ok := exp[i].(quad.Equaler); ok {
 			require.True(t, eq.Equal(got[i]))
 		} else {
-			require.True(t, exp[i] == got[i])
+			require.True(t, exp[i] == got[i], "%v\nvs\n%v\n\n%v\nvs\n%v", exp[i], got[i], exp, got)
 		}
 	}
 }
@@ -275,6 +291,32 @@ func TestWriters(t *testing.T, gen testutil.DatabaseFunc, c *Config) {
 	}
 }
 
+func Test1K(t *testing.T, gen testutil.DatabaseFunc, c *Config) {
+	qs, _, closer := gen(t)
+	defer closer()
+
+	pg := c.PageSize
+	if pg == 0 {
+		pg = 100
+	}
+	n := pg*3 + 1
+
+	w, err := writer.NewSingle(qs, graph.IgnoreOpts{})
+	require.NoError(t, err)
+
+	qw := graph.NewWriter(w)
+	exp := make([]quad.Quad, 0, n)
+	for i := 0; i < n; i++ {
+		q := quad.Make(i, i, i, nil)
+		exp = append(exp, q)
+		qw.WriteQuad(q)
+	}
+	err = qw.Flush()
+	require.NoError(t, err)
+
+	ExpectIteratedQuads(t, qs, qs.QuadsAllIterator(), exp, true)
+}
+
 type ValueSizer interface {
 	SizeOf(graph.Value) int64
 }
@@ -429,7 +471,7 @@ func TestHasA(t testing.TB, gen testutil.DatabaseFunc, conf *Config) {
 	for i := 0; i < 3; i++ {
 		exp = append(exp, quad.Raw("status"))
 	}
-	ExpectIteratedValues(t, qs, it, exp)
+	ExpectIteratedValues(t, qs, it, exp, false)
 }
 
 func TestSetIterator(t testing.TB, gen testutil.DatabaseFunc, _ *Config) {
@@ -823,21 +865,36 @@ var casesCompare = []struct {
 		quad.IRI("alice"), quad.IRI("bob"),
 	}},
 	{gte, quad.Int(111), []quad.Value{
-		quad.Int(112),
+		quad.Int(112), quad.Int(math.MaxInt64 - 1), quad.Int(math.MaxInt64),
 	}},
 	{gte, quad.Int(110), []quad.Value{
-		quad.Int(110), quad.Int(112),
+		quad.Int(110), quad.Int(112), quad.Int(math.MaxInt64 - 1), quad.Int(math.MaxInt64),
+	}},
+	{lt, quad.Int(20), []quad.Value{
+		quad.Int(math.MinInt64 + 1), quad.Int(math.MinInt64),
 	}},
-	{lt, quad.Int(20), nil},
 	{lte, quad.Int(20), []quad.Value{
-		quad.Int(20),
+		quad.Int(math.MinInt64 + 1), quad.Int(math.MinInt64), quad.Int(20),
 	}},
 	{lte, quad.Time(tzero.Add(time.Hour)), []quad.Value{
 		quad.Time(tzero), quad.Time(tzero.Add(time.Hour)),
 	}},
 	{gt, quad.Time(tzero.Add(time.Hour)), []quad.Value{
 		quad.Time(tzero.Add(time.Hour * 49)), quad.Time(tzero.Add(time.Hour * 24 * 365)),
 	}},
+	// precision tests
+	{gt, quad.Int(math.MaxInt64 - 1), []quad.Value{
+		quad.Int(math.MaxInt64),
+	}},
+	{gte, quad.Int(math.MaxInt64 - 1), []quad.Value{
+		quad.Int(math.MaxInt64 - 1), quad.Int(math.MaxInt64),
+	}},
+	{lt, quad.Int(math.MinInt64 + 1), []quad.Value{
+		quad.Int(math.MinInt64),
+	}},
+	{lte, quad.Int(math.MinInt64 + 1), []quad.Value{
+		quad.Int(math.MinInt64 + 1), quad.Int(math.MinInt64),
+	}},
 }
 
 func TestCompareTypedValues(t testing.TB, gen testutil.DatabaseFunc, conf *Config) {
@@ -854,18 +911,33 @@ func TestCompareTypedValues(t testing.TB, gen testutil.DatabaseFunc, conf *Confi
 	t3 := t2.Add(time.Hour * 48)
 	t4 := t1.Add(time.Hour * 24 * 365)
 
-	err := w.AddQuadSet([]quad.Quad{
+	quads := []quad.Quad{
 		{quad.BNode("alice"), quad.BNode("bob"), quad.BNode("charlie"), quad.BNode("dani")},
 		{quad.IRI("alice"), quad.IRI("bob"), quad.IRI("charlie"), quad.IRI("dani")},
 		{quad.String("alice"), quad.String("bob"), quad.String("charlie"), quad.String("dani")},
 		{quad.Int(100), quad.Int(112), quad.Int(110), quad.Int(20)},
 		{quad.Time(t1), quad.Time(t2), quad.Time(t3), quad.Time(t4)},
-	})
+		// test precision as well
+		{quad.Int(math.MaxInt64), quad.Int(math.MaxInt64 - 1), quad.Int(math.MinInt64 + 1), quad.Int(math.MinInt64)},
+	}
+
+	err := w.AddQuadSet(quads)
 	require.NoError(t, err)
 
+	var vals []quad.Value
+	for _, q := range quads {
+		for _, d := range quad.Directions {
+			if v := q.Get(d); v != nil {
+				vals = append(vals, v)
+			}
+		}
+	}
+	ExpectIteratedValues(t, qs, qs.NodesAllIterator(), vals, true)
+
 	for _, c := range casesCompare {
+		//t.Log(c.op, c.val)
 		it := iterator.NewComparison(qs.NodesAllIterator(), c.op, c.val, qs)
-		ExpectIteratedValues(t, qs, it, c.expect)
+		ExpectIteratedValues(t, qs, it, c.expect, true)
 	}
 
 	for _, c := range casesCompare {
@@ -878,7 +950,7 @@ func TestCompareTypedValues(t testing.TB, gen testutil.DatabaseFunc, conf *Confi
 			require.Equal(t, s, ns)
 		}
 		nit := shape.BuildIterator(qs, ns)
-		ExpectIteratedValues(t, qs, nit, c.expect)
+		ExpectIteratedValues(t, qs, nit, c.expect, true)
 	}
 }
 
@@ -916,7 +988,7 @@ func TestNodeDelete(t testing.TB, gen testutil.DatabaseFunc, conf *Config) {
 		quad.Raw("follows"),
 		quad.Raw("status"),
 		quad.Raw("status_graph"),
-	})
+	}, true)
 }
 
 func TestSchema(t testing.TB, gen testutil.DatabaseFunc, conf *Config) {

graph/iterator/value_comparison.go

@@ -37,6 +37,21 @@ import (
 
 type Operator int
 
+func (op Operator) String() string {
+	switch op {
+	case CompareLT:
+		return "<"
+	case CompareLTE:
+		return "<="
+	case CompareGT:
+		return ">"
+	case CompareGTE:
+		return ">="
+	default:
+		return fmt.Sprintf("op(%d)", int(op))
+	}
+}
+
 const (
 	CompareLT Operator = iota
 	CompareLTE

graph/nosql/elastic/elastic_test.go

@@ -11,22 +11,22 @@ import (
 	"github.com/cayleygraph/cayley/internal/dock"
 )
 
-func makeElastic(t testing.TB) (nosql.Database, graph.Options, func()) {
+func makeElastic(t testing.TB) (nosql.Database, *nosql.Options, graph.Options, func()) {
 	var conf dock.Config
 
 	conf.Image = "elasticsearch"
 	conf.OpenStdin = true
 	conf.Tty = true
 
-	addr, closer := dock.RunAndWait(t, conf, dock.WaitPort("9200"))
-	addr = "http://" + addr + ":9200"
+	addr, closer := dock.RunAndWait(t, conf, "9200", nil)
+	addr = "http://" + addr
 
 	db, err := dialDB(addr, nil)
 	if err != nil {
 		closer()
 		t.Fatal(err)
 	}
-	return db, nil, func() {
+	return db, nil, nil, func() {
 		db.Close()
 		closer()
 	}

graph/nosql/iterator.go

@@ -189,7 +189,7 @@ func (it *Iterator) Contains(ctx context.Context, v graph.Value) bool {
 	if qv == nil {
 		return false
 	}
-	d := toDocumentValue(qv)
+	d := it.qs.opt.toDocumentValue(qv)
 	for _, f := range it.constraint {
 		if !f.Matches(d) {
 			return false
@@ -210,6 +210,9 @@ func (it *Iterator) Size() (int64, bool) {
 	if it.limit > 0 && it.size > it.limit {
 		it.size = it.limit
 	}
+	if it.size < 0 {
+		return it.qs.Size(), false
+	}
 	return it.size, true
 }
 

graph/nosql/mongo/mongo_test.go

@@ -11,22 +11,21 @@ import (
 	"github.com/cayleygraph/cayley/internal/dock"
 )
 
-func makeMongo(t testing.TB) (nosql.Database, graph.Options, func()) {
+func makeMongo(t testing.TB) (nosql.Database, *nosql.Options, graph.Options, func()) {
 	var conf dock.Config
 
 	conf.Image = "mongo:3"
 	conf.OpenStdin = true
 	conf.Tty = true
 
-	addr, closer := dock.Run(t, conf)
+	addr, closer := dock.RunAndWait(t, conf, "27017", nil)
 
-	addr = addr + ":27017"
 	qs, err := dialDB(addr, nil)
 	if err != nil {
 		closer()
 		t.Fatal(err)
 	}
-	return qs, nil, func() {
+	return qs, nil, nil, func() {
 		qs.Close()
 		closer()
 	}

graph/nosql/nosqltest/nosqltest_all.go

@@ -16,12 +16,14 @@ import (
 	"github.com/cayleygraph/cayley/quad"
 )
 
-type DatabaseFunc func(t testing.TB) (nosql.Database, graph.Options, func())
+type DatabaseFunc func(t testing.TB) (nosql.Database, *nosql.Options, graph.Options, func())
 
 type Config struct {
 	FloatToInt bool // database silently converts all float values to ints, if possible
+	IntToFloat bool // database always converts all int values to floats
 	TimeInMs   bool
 	Recreate   bool // tests should re-create database instance from scratch on each run
+	PageSize   int  // result page size for pagination (large iterator) tests
 }
 
 func (c Config) quadStore() *graphtest.Config {
@@ -35,14 +37,14 @@ func (c Config) quadStore() *graphtest.Config {
 }
 
 func NewQuadStore(t testing.TB, gen DatabaseFunc) (graph.QuadStore, graph.Options, func()) {
-	db, opt, closer := gen(t)
+	db, nopt, opt, closer := gen(t)
 	err := nosql.Init(db, opt)
 	if err != nil {
 		db.Close()
 		closer()
 		require.Fail(t, "init failed", "%v", err)
 	}
-	kdb, err := nosql.New(db, opt)
+	kdb, err := nosql.NewQuadStore(db, nopt, opt)
 	if err != nil {
 		db.Close()
 		closer()