Merge pull request #59 from everFinance/feature/getItems

add   GetBundleItems(bundleInId string, itemsIds []string)

Author: zyjblockchain

.gitignore

@@ -1,4 +1,6 @@
 main.go
 go.sum
 test-keyfile.json
-.DS_Store
+.DS_Store
+.idea
+vendor

client.go

@@ -10,6 +10,7 @@ import (
 	"github.com/tidwall/gjson"
 	"gopkg.in/h2non/gentleman.v2"
 	"io/ioutil"
+	"math"
 	"math/big"
 	"net/http"
 	"net/url"
@@ -1052,3 +1053,92 @@ func (c *Client) SubmitToWarp(tx *types.Transaction) ([]byte, error) {
 	defer resp.Body.Close()
 	return ioutil.ReadAll(resp.Body)
 }
+
+/**
+
+ bundleBinary Data Format
++------------------+-----------------------------+--------------------------------------------+
+| Number of Items  |      Headers (64 bytes)     |          Items' Binary Data                |
+|      32 bytes    | 32 bytes (length) + 32 bytes |                                            |
+|                  |       (ID) for each item    |                                            |
++------------------+-----------------------------+--------------------------------------------+
+*/
+
+func (c *Client) GetBundleItems(bundleInId string, itemsIds []string) (items []*types.BundleItem, err error) {
+	offset, err := c.getTransactionOffset(bundleInId)
+	if err != nil {
+		return nil, err
+	}
+
+	size, err := strconv.ParseInt(offset.Size, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+	endOffset, err := strconv.ParseInt(offset.Offset, 10, 64)
+	startOffset := endOffset - size + 1
+
+	firstChunk, err := c.getChunkData(startOffset)
+	if err != nil {
+		return nil, err
+	}
+
+	itemsNum := utils.ByteArrayToLong(firstChunk[:32])
+
+	// get Headers endOffset
+	bundleItemStart := 32 + itemsNum*64
+	var containHeadersChunks []byte
+	if len(firstChunk) < bundleItemStart {
+		// To calculate headers, you need to pull several chunks and fetch an integer upwards
+		chunkNum := int(math.Ceil(float64(bundleItemStart) / float64(types.MAX_CHUNK_SIZE)))
+
+		for i := 0; i < chunkNum; i++ {
+			chunk, err := c.getChunkData(startOffset + int64(i*types.MAX_CHUNK_SIZE))
+			if err != nil {
+				return nil, err
+			}
+			containHeadersChunks = append(containHeadersChunks, chunk...)
+		}
+	} else {
+		containHeadersChunks = firstChunk
+	}
+
+	for i := 0; i < itemsNum; i++ {
+		headerBegin := 32 + i*64
+		end := headerBegin + 64
+
+		headerByte := containHeadersChunks[headerBegin:end]
+		itemBinaryLength := utils.ByteArrayToLong(headerByte[:32])
+		id := utils.Base64Encode(headerByte[32:64])
+
+		// if item is in itemsIds
+		if utils.ContainsInSlice(itemsIds, id) {
+
+			startChunkNum := bundleItemStart / types.MAX_CHUNK_SIZE
+			startChunkOffset := bundleItemStart % types.MAX_CHUNK_SIZE
+			data := make([]byte, 0, itemBinaryLength)
+
+			for offset := startOffset + int64(startChunkNum*types.MAX_CHUNK_SIZE); offset <= startOffset+int64(bundleItemStart+itemBinaryLength); {
+				chunk, err := c.getChunkData(offset)
+
+				if err != nil {
+					return nil, err
+				}
+				data = append(data, chunk...)
+				offset += int64(len(chunk))
+			}
+
+			itemData := data[startChunkOffset : startChunkOffset+itemBinaryLength]
+			item, err := utils.DecodeBundleItem(itemData)
+			if err != nil {
+				return nil, err
+			}
+
+			items = append(items, item)
+		}
+		// next itemBy start offset
+		bundleItemStart += itemBinaryLength
+	}
+
+	return items, nil
+
+}

client_test.go

@@ -348,3 +348,49 @@ func TestNewTempConn2(t *testing.T) {
 	err = utils.VerifyBundleItem(*item)
 	assert.NoError(t, err)
 }
+
+// https://arweave.net/tx/x-q8ibbTfXIcdDXqQ3xaPD3PuShj832G_xzNT5QrVjY/offset
+// {"size":"753","offset":"146739359163367"}
+func Test_getChunkData(t *testing.T) {
+	c := NewClient("https://arweave.net")
+	data, err := c.getChunkData(146739359163367)
+	assert.NoError(t, err)
+
+	t.Log(string(data))
+
+}
+
+func TestClient_GetBundleItems(t *testing.T) {
+	c := NewClient("https://arweave.net")
+	itemsIds := []string{"UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGw"}
+	items, err := c.GetBundleItems("47KozLIAfVMKdxq1q3D1xFZmRpkahOOBQ8boOjSydnQ", itemsIds)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(items))
+	assert.Equal(t, "UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGw", items[0].Id)
+}
+
+func TestClient_GetBundleItems2(t *testing.T) {
+	c := NewClient("https://arweave.net")
+	itemsIds := []string{"UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGw", "FCUfgEEPmZB3YQMTfbwYl6VA-JT54zLr5PrcJw2EFeM", "zlU0o99c81n0CP64F31ANpyJeOtlz5DKvsKohmbMxqU"}
+	items, err := c.GetBundleItems("47KozLIAfVMKdxq1q3D1xFZmRpkahOOBQ8boOjSydnQ", itemsIds)
+
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(items))
+	assert.Equal(t, "UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGw", items[2].Id)
+	assert.Equal(t, "FCUfgEEPmZB3YQMTfbwYl6VA-JT54zLr5PrcJw2EFeM", items[1].Id)
+	assert.Equal(t, "zlU0o99c81n0CP64F31ANpyJeOtlz5DKvsKohmbMxqU", items[0].Id)
+}
+
+// https://viewblock.io/zh-CN/arweave/tx/PRBVxEX00aVMN59EY8gznt83FTlGXZvESwv1WTP7ReQ  5000 items
+func TestClient_GetBundleItems3(t *testing.T) {
+	c := NewClient("https://arweave.net")
+	itemsIds := []string{"QD0ryQTy4CBr7kluWRLT1strRcXWJOgUUoIYat4lk1s", "BzsIVzo6rPfGQg0PP-5Y_HErPey51_it0d6aGIUfQnY", "fy3aOYoRf7OzCEd9_WrD-RfqbzNZ1LsJ4PKIIGUALik"}
+	items, err := c.GetBundleItems("PRBVxEX00aVMN59EY8gznt83FTlGXZvESwv1WTP7ReQ", itemsIds)
+
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(items))
+	assert.Equal(t, "QD0ryQTy4CBr7kluWRLT1strRcXWJOgUUoIYat4lk1s", items[0].Id)
+	assert.Equal(t, "fy3aOYoRf7OzCEd9_WrD-RfqbzNZ1LsJ4PKIIGUALik", items[1].Id)
+	assert.Equal(t, "BzsIVzo6rPfGQg0PP-5Y_HErPey51_it0d6aGIUfQnY", items[2].Id)
+
+}

utils/slice.go

@@ -0,0 +1,11 @@
+package utils
+
+// ContainsInSlice checks if a string is in a slice of strings
+func ContainsInSlice(items []string, item string) bool {
+	for _, eachItem := range items {
+		if eachItem == item {
+			return true
+		}
+	}
+	return false
+}

utils/slice_test.go

@@ -0,0 +1,40 @@
+package utils
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestContainsInSlice(t *testing.T) {
+	type args struct {
+		items []string
+		item  string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "string in slice",
+			args: args{
+				items: []string{"a", "b", "c"},
+				item:  "a",
+			},
+			want: true,
+		},
+		{
+			name: "string not in slice",
+			args: args{
+				items: []string{"a", "b", "c"},
+				item:  "x",
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, ContainsInSlice(tt.args.items, tt.args.item), "ContainsInSlice(%v, %v)", tt.args.items, tt.args.item)
+		})
+	}
+}