added engine v2

ripping old engine out

replaced old engine

Taskfile.yml

@@ -230,14 +230,12 @@ tasks:
     desc: Run unit tests
     cmds:
       - go test ./core/... -tags=ext_test -count=1
-      - CGO_ENABLED=1 go test ./parse/... -tags=ext_test -count=1
       - CGO_ENABLED=1 go test ./... -tags=ext_test,pglive -count=1 -p=1 # no parallel for now because several try to use one pg database
 
   test:unit:race:
     desc: Run unit tests with the race detector
     cmds:
       - go test ./core/... -tags=ext_test -count=1 -race
-      - CGO_ENABLED=1 go test ./parse/... -tags=ext_test -count=1 -race
       - CGO_ENABLED=1 go test ./... -tags=ext_test,pglive -count=1 -race
 
   # test:it:

_previous/core/types/schema.go

@@ -1141,7 +1141,7 @@ func (c *DataType) PGString() (string, error) {
 		scalar = "UINT256"
 	case DecimalStr:
 		if c.Metadata == ZeroMetadata {
-			return "", fmt.Errorf("decimal type must have metadata")
+			return "NUMERIC", nil
 		}
 
 		scalar = fmt.Sprintf("NUMERIC(%d,%d)", c.Metadata[0], c.Metadata[1])
@@ -1171,7 +1171,7 @@ func (c *DataType) Clean() error {
 		return nil
 	case DecimalStr:
 		if c.Metadata == ZeroMetadata {
-			return fmt.Errorf("decimal type must have metadata")
+			return nil
 		}
 
 		err := decimal.CheckPrecisionAndScale(c.Metadata[0], c.Metadata[1])

app/node/build.go

@@ -26,7 +26,7 @@ import (
 	"github.com/kwilteam/kwil-db/node/accounts"
 	blockprocessor "github.com/kwilteam/kwil-db/node/block_processor"
 	"github.com/kwilteam/kwil-db/node/consensus"
-	"github.com/kwilteam/kwil-db/node/engine/execution"
+	"github.com/kwilteam/kwil-db/node/engine/interpreter"
 	"github.com/kwilteam/kwil-db/node/listeners"
 	"github.com/kwilteam/kwil-db/node/mempool"
 	"github.com/kwilteam/kwil-db/node/meta"
@@ -346,7 +346,7 @@ func (c *coreDependencies) service(loggerName string) *common.Service {
 }
 
 func buildTxApp(ctx context.Context, d *coreDependencies, db *pg.DB, accounts *accounts.Accounts,
-	votestore *voting.VoteStore, engine *execution.GlobalContext) *txapp.TxApp {
+	votestore *voting.VoteStore, engine common.Engine) *txapp.TxApp {
 	signer := auth.GetNodeSigner(d.privKey)
 
 	txapp, err := txapp.NewTxApp(ctx, db, engine, signer, nil, d.service("TxAPP"), accounts, votestore)
@@ -472,7 +472,7 @@ func failBuild(err error, msg string) {
 	})
 }
 
-func buildEngine(d *coreDependencies, db *pg.DB) *execution.GlobalContext {
+func buildEngine(d *coreDependencies, db *pg.DB) *interpreter.ThreadSafeInterpreter {
 	extensions := precompiles.RegisteredPrecompiles()
 	for name := range extensions {
 		d.logger.Info("registered extension", "name", name)
@@ -484,23 +484,17 @@ func buildEngine(d *coreDependencies, db *pg.DB) *execution.GlobalContext {
 	}
 	defer tx.Rollback(d.ctx)
 
-	err = execution.InitializeEngine(d.ctx, tx)
+	interp, err := interpreter.NewInterpreter(d.ctx, tx, d.service("engine"))
 	if err != nil {
 		failBuild(err, "failed to initialize engine")
 	}
 
-	eng, err := execution.NewGlobalContext(d.ctx, tx,
-		extensions, d.newService("engine"))
-	if err != nil {
-		failBuild(err, "failed to build engine")
-	}
-
 	err = tx.Commit(d.ctx)
 	if err != nil {
 		failBuild(err, "failed to commit engine init db txn")
 	}
 
-	return eng
+	return interp
 }
 
 func buildSnapshotStore(d *coreDependencies) *snapshotter.SnapshotStore {

app/shared/display/message_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/kwilteam/kwil-db/core/crypto/auth"
 	"github.com/kwilteam/kwil-db/core/types"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

cmd/kwil-cli/cmds/database/call.go

@@ -84,14 +84,17 @@ func callCmd() *cobra.Command {
 				if len(tuples) == 0 {
 					tuples = append(tuples, []any{})
 				}
+				if len(tuples) > 1 {
+					return display.PrintErr(cmd, errors.New("only one set of inputs can be provided to call"))
+				}
 
 				data, err := clnt.Call(ctx, dbid, action, tuples[0])
 				if err != nil {
 					return display.PrintErr(cmd, fmt.Errorf("error calling action/procedure: %w", err))
 				}
 
 				if data == nil {
-					data = &clientType.CallResult{}
+					data = &types.CallResult{}
 				}
 
 				return display.PrintCmd(cmd, &respCall{
@@ -109,13 +112,13 @@ func callCmd() *cobra.Command {
 }
 
 type respCall struct {
-	Data      *clientType.CallResult
+	Data      *types.CallResult
 	PrintLogs bool
 }
 
 func (r *respCall) MarshalJSON() ([]byte, error) {
 	if !r.PrintLogs {
-		return json.Marshal(r.Data.Records.ToStrings()) // this is for backwards compatibility
+		return json.Marshal(r.Data.QueryResult) // this is for backwards compatibility
 	}
 
 	bts, err := json.Marshal(r.Data)
@@ -128,10 +131,10 @@ func (r *respCall) MarshalJSON() ([]byte, error) {
 
 func (r *respCall) MarshalText() (text []byte, err error) {
 	if !r.PrintLogs {
-		return recordsToTable(r.Data.Records), nil
+		return recordsToTable(r.Data.QueryResult.ExportToStringMap()), nil
 	}
 
-	bts := recordsToTable(r.Data.Records)
+	bts := recordsToTable(r.Data.QueryResult.ExportToStringMap())
 
 	if len(r.Data.Logs) > 0 {
 		bts = append(bts, []byte("\n\nLogs:")...)
@@ -145,25 +148,90 @@ func (r *respCall) MarshalText() (text []byte, err error) {
 
 // buildProcedureInputs will build the inputs for either
 // an action or procedure executon/call.
-func buildExecutionInputs(ctx context.Context, client clientType.Client, dbid string, proc string, inputs []map[string]string) ([][]any, error) {
-	schema, err := client.GetSchema(ctx, dbid)
+func buildExecutionInputs(ctx context.Context, client clientType.Client, namespace string, action string, inputs []map[string]string) ([][]any, error) {
+	params, err := getParamList(ctx, client, namespace, action)
 	if err != nil {
-		return nil, fmt.Errorf("error getting schema: %w", err)
+		return nil, err
 	}
 
-	for _, a := range schema.Actions {
-		if strings.EqualFold(a.Name, proc) {
-			return buildActionInputs(a, inputs)
+	var results [][]any
+	for _, in := range inputs {
+		var tuple []any
+		for _, p := range params {
+			val, ok := in[p.Name]
+			if !ok {
+				tuple = append(tuple, nil)
+				continue
+			}
+
+			encoded, err := encodeBasedOnType(p.Type, val)
+			if err != nil {
+				return nil, err
+			}
+
+			tuple = append(tuple, encoded)
 		}
+
+		results = append(results, tuple)
+	}
+
+	return results, nil
+}
+
+func getParamList(ctx context.Context, client clientType.Client, namespace, action string) ([]paramList, error) {
+	res, err := client.Query(ctx, "{info}SELECT parameters FROM actions WHERE namespace = $namespace AND name = $action", map[string]any{
+		"namespace": namespace,
+		"action":    action,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(res.Values) == 0 {
+		return nil, errors.New(`action "%s" not found in namespace "%s"`)
+	}
+	if len(res.Values) > 1 {
+		return nil, errors.New(`action "%s" is ambiguous in namespace "%s"`)
+	}
+
+	var strVal string
+	switch res.Values[0][0].(type) {
+	case nil:
+		return nil, nil // no inputs
+	case string:
+		strVal = res.Values[0][0].(string)
+	default:
+		return nil, errors.New("unexpected type for action parameters. this is a bug")
+	}
+
+	p := []struct {
+		Name     string `json:"name"`
+		DataType string `json:"data_type"`
+	}{}
+
+	if err := json.Unmarshal([]byte(strVal), &p); err != nil {
+		return nil, err
 	}
 
-	for _, p := range schema.Procedures {
-		if strings.EqualFold(p.Name, proc) {
-			return buildProcedureInputs(p, inputs)
+	params := make([]paramList, len(p))
+	for i, param := range p {
+		dt, err := types.ParseDataType(param.DataType)
+		if err != nil {
+			return nil, err
+		}
+
+		params[i] = paramList{
+			Name: param.Name,
+			Type: dt,
 		}
 	}
 
-	return nil, errors.New("procedure/action not found")
+	return params, nil
+}
+
+type paramList struct {
+	Name string
+	Type *types.DataType
 }
 
 // decodeMany attempts to parse command-line inputs as base64 encoded values.
@@ -190,97 +258,33 @@ func decodeMany(inputs []string) ([][]byte, bool) {
 	return b64Arr, b64Ok
 }
 
-func buildActionInputs(a *types.Action, inputs []map[string]string) ([][]any, error) {
-	tuples := [][]any{}
-	for _, input := range inputs {
-		newTuple := []any{}
-		for _, inputField := range a.Parameters {
-			// unlike procedures, actions do not have typed parameters,
-			// so we should try to always parse arrays.
-
-			val, ok := input[inputField]
-			if !ok {
-				fmt.Println(len(newTuple))
-				// if not found, we should just add nil
-				newTuple = append(newTuple, nil)
-				continue
-			}
-
-			split, err := splitIgnoringQuotedCommas(val)
-			if err != nil {
-				return nil, err
-			}
-
-			// attempt to decode base64 encoded values
-			b64Arr, b64Ok := decodeMany(split)
-			if b64Ok {
-				// additional check here in case user is sending a single base64 value, we don't
-				// want to encode it as an array.
-				if len(b64Arr) == 1 {
-					newTuple = append(newTuple, b64Arr[0])
-					continue
-				}
-
-				newTuple = append(newTuple, b64Arr)
-			} else {
-				// if nothing was split, then keep the original value, not the []string{}
-				if len(split) == 1 {
-					newTuple = append(newTuple, split[0])
-					continue
-				}
-
-				newTuple = append(newTuple, split)
-			}
+// encodeBasedOnType will encode the input value based on the type of the input.
+// If it is an array, it will properly split the input value by commas.
+// If the input value is base64 encoded, it will decode it.
+func encodeBasedOnType(t *types.DataType, v string) (any, error) {
+	if t.IsArray {
+		split, err := splitIgnoringQuotedCommas(v)
+		if err != nil {
+			return nil, err
 		}
-		tuples = append(tuples, newTuple)
-	}
-
-	return tuples, nil
-}
-
-func buildProcedureInputs(p *types.Procedure, inputs []map[string]string) ([][]any, error) {
-	tuples := [][]any{}
-	for _, input := range inputs {
-		newTuple := []any{}
-		for _, inputField := range p.Parameters {
-			v, ok := input[inputField.Name]
-			if !ok {
-				// if not found, we should just add nil
-				newTuple = append(newTuple, nil)
-				continue
-			}
-
-			// if the input is an array, split it by commas
-			if inputField.Type.IsArray {
-				split, err := splitIgnoringQuotedCommas(v)
-				if err != nil {
-					return nil, err
-				}
 
-				// attempt to decode base64 encoded values
-				b64Arr, b64Ok := decodeMany(split)
-				if b64Ok {
-					newTuple = append(newTuple, b64Arr)
-				} else {
-					newTuple = append(newTuple, split)
-				}
-				continue
-			}
-
-			// attempt to decode base64 encoded values
-
-			bts, ok := decodeMany([]string{v})
-			if ok {
-				newTuple = append(newTuple, bts[0])
-			} else {
-				newTuple = append(newTuple, input[inputField.Name])
-			}
+		// attempt to decode base64 encoded values
+		b64Arr, b64Ok := decodeMany(split)
+		if b64Ok {
+			return b64Arr, nil
 		}
 
-		tuples = append(tuples, newTuple)
+		return split, nil
+	}
+
+	// attempt to decode base64 encoded values
+	bts, ok := decodeMany([]string{v})
+	if ok {
+		return bts[0], nil
 	}
 
-	return tuples, nil
+	// otherwise, just keep it as string and let the server handle it
+	return v, nil
 }
 
 // splitIgnoringQuotedCommas splits a string by commas, but ignores commas that are inside single or double quotes.

cmd/kwil-cli/cmds/database/cmd.go

@@ -19,17 +19,13 @@ var (
 func NewCmdDatabase() *cobra.Command {
 	// readOnlyCmds do not create a transaction.
 	readOnlyCmds := []*cobra.Command{
-		listCmd(),
-		readSchemaCmd(),
 		queryCmd(),
 		callCmd(), // no tx, but may required key for signature, for now
 	}
 	dbCmd.AddCommand(readOnlyCmds...)
 
 	// writeCmds create a transactions, requiring a private key for signing/
 	writeCmds := []*cobra.Command{
-		deployCmd(),
-		dropCmd(),
 		executeCmd(),
 		batchCmd(),
 	}