trying to fix encoding issues

cmd/common/display/message_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 
 	"github.com/kwilteam/kwil-db/core/crypto/auth"
+	"github.com/kwilteam/kwil-db/core/types"
 	"github.com/kwilteam/kwil-db/core/types/transactions"
 
 	"github.com/stretchr/testify/assert"
@@ -90,8 +91,15 @@ func getExampleTxQueryResponse() *transactions.TcTxQueryResponse {
 	rawPayload := transactions.ActionExecution{
 		DBID:   "xf617af1ca774ebbd6d23e8fe12c56d41d25a22d81e88f67c6c6ee0d4",
 		Action: "create_user",
-		Arguments: [][]string{
-			{"foo", "32"},
+		Arguments: [][]*transactions.EncodedValue{
+			{
+				{
+					Type: transactions.DataType{
+						Name: types.TextType.Name,
+					},
+					Data: [][]byte{[]byte("foo")},
+				},
+			},
 		},
 	}
 
@@ -128,15 +136,14 @@ func getExampleTxQueryResponse() *transactions.TcTxQueryResponse {
 
 func Example_respTxQuery_text() {
 	Print(&RespTxQuery{Msg: getExampleTxQueryResponse(), WithRaw: true}, nil, "text")
-	// Output:
 	// Transaction ID: 31303234
 	// Status: success
 	// Height: 10
 	// Log: This is log
-	// Raw: 0001f8e4f850b841cb3fed7f6ff36e59054c04a831b215e514052753ee353e6fe31d4b4ef736acd6155127db555d3006ba14fcb4c79bbad56c8e63b81a9896319bb053a9e2534758008c736563703235366b315f6570f888a25468697320697320612074657374207472616e73616374696f6e20666f7220636c69b8540001f850b8397866363137616631636137373465626264366432336538666531326335366434316432356132326438316538386636376336633665653064348b6372656174655f75736572c8c783666f6f8233328765786563757465640a846173646686636f6e63617480
+	// Raw: 0001f8eaf850b841cb3fed7f6ff36e59054c04a831b215e514052753ee353e6fe31d4b4ef736acd6155127db555d3006ba14fcb4c79bbad56c8e63b81a9896319bb053a9e2534758008c736563703235366b315f6570f88ea25468697320697320612074657374207472616e73616374696f6e20666f7220636c69b85a0001f856b8397866363137616631636137373465626264366432336538666531326335366434316432356132326438316538386636376336633665653064348b6372656174655f75736572cecdccc6847465787480c483666f6f8765786563757465640a846173646686636f6e63617480
 	// WARNING! HASH MISMATCH:
 	// 	Requested 31303234
-	// 	Received  fe77ea3e9c86de9b6afcf8a27c38ac9ddf102086f8ac51263659525d1c39fbf0
+	// 	Received  f866b4251d21552de1bc5b819a4b563a540146954e956e8150163574ce5325ac
 }
 
 func Example_respTxQuery_json() {
@@ -153,7 +160,7 @@ func Example_respTxQuery_json() {
 	//       },
 	//       "body": {
 	//         "desc": "This is a test transaction for cli",
-	//         "payload": "AAH4ULg5eGY2MTdhZjFjYTc3NGViYmQ2ZDIzZThmZTEyYzU2ZDQxZDI1YTIyZDgxZTg4ZjY3YzZjNmVlMGQ0i2NyZWF0ZV91c2VyyMeDZm9vgjMy",
+	//         "payload": "AAH4Vrg5eGY2MTdhZjFjYTc3NGViYmQ2ZDIzZThmZTEyYzU2ZDQxZDI1YTIyZDgxZTg4ZjY3YzZjNmVlMGQ0i2NyZWF0ZV91c2Vyzs3MxoR0ZXh0gMSDZm9v",
 	//         "type": "execute",
 	//         "fee": "100",
 	//         "nonce": 10,
@@ -187,7 +194,7 @@ func Example_respTxQuery_WithRaw_json() {
 	//       },
 	//       "body": {
 	//         "desc": "This is a test transaction for cli",
-	//         "payload": "AAH4ULg5eGY2MTdhZjFjYTc3NGViYmQ2ZDIzZThmZTEyYzU2ZDQxZDI1YTIyZDgxZTg4ZjY3YzZjNmVlMGQ0i2NyZWF0ZV91c2VyyMeDZm9vgjMy",
+	//         "payload": "AAH4Vrg5eGY2MTdhZjFjYTc3NGViYmQ2ZDIzZThmZTEyYzU2ZDQxZDI1YTIyZDgxZTg4ZjY3YzZjNmVlMGQ0i2NyZWF0ZV91c2Vyzs3MxoR0ZXh0gMSDZm9v",
 	//         "type": "execute",
 	//         "fee": "100",
 	//         "nonce": 10,
@@ -202,8 +209,8 @@ func Example_respTxQuery_WithRaw_json() {
 	//       "gas_used": 10,
 	//       "gas_wanted": 10
 	//     },
-	//     "raw": "0001f8e4f850b841cb3fed7f6ff36e59054c04a831b215e514052753ee353e6fe31d4b4ef736acd6155127db555d3006ba14fcb4c79bbad56c8e63b81a9896319bb053a9e2534758008c736563703235366b315f6570f888a25468697320697320612074657374207472616e73616374696f6e20666f7220636c69b8540001f850b8397866363137616631636137373465626264366432336538666531326335366434316432356132326438316538386636376336633665653064348b6372656174655f75736572c8c783666f6f8233328765786563757465640a846173646686636f6e63617480",
-	//     "warning": "HASH MISMATCH: requested 31303234; received fe77ea3e9c86de9b6afcf8a27c38ac9ddf102086f8ac51263659525d1c39fbf0"
+	//     "raw": "0001f8eaf850b841cb3fed7f6ff36e59054c04a831b215e514052753ee353e6fe31d4b4ef736acd6155127db555d3006ba14fcb4c79bbad56c8e63b81a9896319bb053a9e2534758008c736563703235366b315f6570f88ea25468697320697320612074657374207472616e73616374696f6e20666f7220636c69b85a0001f856b8397866363137616631636137373465626264366432336538666531326335366434316432356132326438316538386636376336633665653064348b6372656174655f75736572cecdccc6847465787480c483666f6f8765786563757465640a846173646686636f6e63617480",
+	//     "warning": "HASH MISMATCH: requested 31303234; received f866b4251d21552de1bc5b819a4b563a540146954e956e8150163574ce5325ac"
 	//   },
 	//   "error": ""
 	// }
@@ -217,13 +224,7 @@ func Test_TxHashAndExecResponse(t *testing.T) {
 		Hash:      hash,
 		QueryResp: &RespTxQuery{Msg: qr},
 	}
-	expectJson := `{"tx_hash":"0102030405","exec_result":{"hash":"0102030405","height":10,"tx":` +
-		`{` +
-		`"signature":{"sig":"yz/tf2/zblkFTASoMbIV5RQFJ1PuNT5v4x1LTvc2rNYVUSfbVV0wBroU/LTHm7rVbI5juBqYljGbsFOp4lNHWAA=","type":"secp256k1_ep"},` +
-		`"body":{"desc":"This is a test transaction for cli","payload":"AAH4ULg5eGY2MTdhZjFjYTc3NGViYmQ2ZDIzZThmZTEyYzU2ZDQxZDI1YTIyZDgxZTg4ZjY3YzZjNmVlMGQ0i2NyZWF0ZV91c2VyyMeDZm9vgjMy","type":"execute","fee":"100","nonce":10,"chain_id":"asdf"},` +
-		`"serialization":"concat","sender":""},` +
-		`"tx_result":{"code":0,"log":"This is log","gas_used":10,"gas_wanted":10}}` +
-		`}`
+	expectJson := `{"tx_hash":"0102030405","exec_result":{"hash":"0102030405","height":10,"tx":{"signature":{"sig":"yz/tf2/zblkFTASoMbIV5RQFJ1PuNT5v4x1LTvc2rNYVUSfbVV0wBroU/LTHm7rVbI5juBqYljGbsFOp4lNHWAA=","type":"secp256k1_ep"},"body":{"desc":"This is a test transaction for cli","payload":"AAH4Vrg5eGY2MTdhZjFjYTc3NGViYmQ2ZDIzZThmZTEyYzU2ZDQxZDI1YTIyZDgxZTg4ZjY3YzZjNmVlMGQ0i2NyZWF0ZV91c2Vyzs3MxoR0ZXh0gMSDZm9v","type":"execute","fee":"100","nonce":10,"chain_id":"asdf"},"serialization":"concat","sender":""},"tx_result":{"code":0,"log":"This is log","gas_used":10,"gas_wanted":10}}}`
 	expectText := "TxHash: 0102030405\nStatus: success\nHeight: 10\nLog: This is log"
 
 	outText, err := resp.MarshalText()

cmd/kwil-cli/cmds/database/call.go

@@ -118,7 +118,7 @@ func buildExecutionInputs(ctx context.Context, client clientType.Client, dbid st
 
 	for _, p := range schema.Procedures {
 		if strings.EqualFold(p.Name, proc) {
-			return buildProcedureInputs(p, inputs), nil
+			return buildProcedureInputs(p, inputs)
 		}
 	}
 
@@ -139,16 +139,70 @@ func buildActionInputs(a *types.Action, inputs []map[string]any) [][]any {
 	return tuples
 }
 
-func buildProcedureInputs(p *types.Procedure, inputs []map[string]any) [][]any {
+func buildProcedureInputs(p *types.Procedure, inputs []map[string]any) ([][]any, error) {
 	tuples := [][]any{}
 	for _, input := range inputs {
 		newTuple := []any{}
 		for _, inputField := range p.Parameters {
+			// if the input is an array, split it by commas
+			if inputField.Type.IsArray {
+				split, err := splitIgnoringQuotedCommas(input[inputField.Name].(string))
+				if err != nil {
+					return nil, err
+				}
+
+				newTuple = append(newTuple, split)
+				continue
+			}
 			newTuple = append(newTuple, input[inputField.Name])
 		}
 
 		tuples = append(tuples, newTuple)
 	}
 
-	return tuples
+	return tuples, nil
+}
+
+// splitIgnoringQuotedCommas splits a string by commas, but ignores commas that are inside single or double quotes.
+// It will return an error if there are unclosed quotes.
+func splitIgnoringQuotedCommas(input string) ([]string, error) {
+	var result []string
+	var currentToken []rune
+	inSingleQuote := false
+	inDoubleQuote := false
+
+	for _, char := range input {
+		switch char {
+		case '\'':
+			if !inDoubleQuote { // Toggle single quote flag if not inside double quotes
+				inSingleQuote = !inSingleQuote
+				continue // Skip appending this quote character to token
+			}
+			currentToken = append(currentToken, char)
+		case '"':
+			if !inSingleQuote { // Toggle double quote flag if not inside single quotes
+				inDoubleQuote = !inDoubleQuote
+				continue // Skip appending this quote character to token
+			}
+			currentToken = append(currentToken, char)
+		case ',':
+			if inSingleQuote || inDoubleQuote { // If inside quotes, treat comma as a normal character
+				currentToken = append(currentToken, char)
+			} else { // Otherwise, it's a delimiter
+				result = append(result, string(currentToken))
+				currentToken = []rune{}
+			}
+		default:
+			currentToken = append(currentToken, char)
+		}
+	}
+
+	// Append the last token
+	result = append(result, string(currentToken))
+
+	if inSingleQuote || inDoubleQuote {
+		return nil, fmt.Errorf("unclosed quote in array inputs")
+	}
+
+	return result, nil
 }

cmd/kwil-cli/cmds/database/call_test.go

@@ -0,0 +1,49 @@
+package database
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_SplitArray(t *testing.T) {
+	type testCase struct {
+		input    string
+		expected []string
+		wantErr  bool
+	}
+
+	tests := []testCase{
+		{
+			input: `name,"id",'age'`,
+			expected: []string{
+				"name",
+				"id",
+				"age",
+			},
+		},
+		{
+			input:   `name,"id",'age`,
+			wantErr: true,
+		},
+		{
+			input: `"val1,val1",val2,val3`,
+			expected: []string{
+				"val1,val1",
+				"val2",
+				"val3",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			got, err := splitIgnoringQuotedCommas(tt.input)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("splitIgnoringQuotedCommas() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			require.Equal(t, tt.expected, got)
+		})
+	}
+}

core/client/client.go

@@ -12,7 +12,6 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/cstockton/go-conv"
 	"github.com/kwilteam/kwil-db/core/crypto/auth"
 	"github.com/kwilteam/kwil-db/core/log"
 	rpcclient "github.com/kwilteam/kwil-db/core/rpc/client"
@@ -228,16 +227,19 @@ func (c *Client) ExecuteAction(ctx context.Context, dbid string, action string,
 // It can take any number of inputs, and if multiple tuples of inputs are passed,
 // it will execute them in the same transaction.
 func (c *Client) Execute(ctx context.Context, dbid string, procedure string, tuples [][]any, opts ...clientType.TxOpt) (transactions.TxHash, error) {
-	stringTuples, isNil, err := convertTuples(tuples)
-	if err != nil {
-		return nil, err
+	encodedTuples := make([][]*transactions.EncodedValue, len(tuples))
+	for i, tuple := range tuples {
+		encoded, err := encodeTuple(tuple)
+		if err != nil {
+			return nil, err
+		}
+		encodedTuples[i] = encoded
 	}
 
 	executionBody := &transactions.ActionExecution{
 		Action:    procedure,
 		DBID:      dbid,
-		Arguments: stringTuples,
-		NilArg:    isNil,
+		Arguments: encodedTuples,
 	}
 
 	txOpts := clientType.GetTxOpts(opts)
@@ -262,16 +264,15 @@ func (c *Client) CallAction(ctx context.Context, dbid string, action string, inp
 
 // Call calls a procedure or action. It returns the result records.
 func (c *Client) Call(ctx context.Context, dbid string, procedure string, inputs []any) (*clientType.Records, error) {
-	stringInputs, isNil, err := convertTuple(inputs)
+	encoded, err := encodeTuple(inputs)
 	if err != nil {
 		return nil, err
 	}
 
 	payload := &transactions.ActionCall{
 		DBID:      dbid,
 		Action:    procedure,
-		Arguments: stringInputs,
-		NilArg:    isNil,
+		Arguments: encoded,
 	}
 
 	msg, err := transactions.CreateCallMessage(payload)
@@ -319,45 +320,18 @@ func (c *Client) GetAccount(ctx context.Context, acctID []byte, status types.Acc
 	return c.txClient.GetAccount(ctx, acctID, status)
 }
 
-// convertTuples converts user passed tuples to strings.
-// this is necessary for RLP encoding
-func convertTuples(tuples [][]any) ([][]string, [][]bool, error) {
-	ins := make([][]string, 0, len(tuples))
-	nils := make([][]bool, 0, len(tuples))
-	for _, tuple := range tuples {
-		stringTuple, isNil, err := convertTuple(tuple)
+// encodeTuple encodes a tuple for usage in a transaction.
+func encodeTuple(tup []any) ([]*transactions.EncodedValue, error) {
+	encoded := make([]*transactions.EncodedValue, 0, len(tup))
+	for _, val := range tup {
+		ev, err := transactions.EncodeValue(val)
 		if err != nil {
-			return nil, nil, err
+			return nil, err
 		}
-		ins = append(ins, stringTuple)
-		nils = append(nils, isNil)
-	}
-
-	return ins, nils, nil
-}
-
-// convertTuple converts user passed tuple to strings.
-func convertTuple(tuple []any) ([]string, []bool, error) {
-	stringTuple := make([]string, 0, len(tuple))
-	isNil := make([]bool, 0, len(tuple))
-	for _, val := range tuple {
-		if val == nil {
-			stringTuple = append(stringTuple, "")
-			isNil = append(isNil, true)
-			continue
-		}
-
-		// conv.String would make it "<null>", which could very well be an intended string
-		stringVal, err := conv.String(val)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		stringTuple = append(stringTuple, stringVal)
-		isNil = append(isNil, false)
+		encoded = append(encoded, ev)
 	}
 
-	return stringTuple, isNil, nil
+	return encoded, nil
 }
 
 // TxQuery get transaction by hash.