Improve sharedSecret performance backed by Rust (#75)

* add api client with grpc kotlin

* add bench marking for performance debugging

* is it a macOS issue

* add a integration test instead

* identified the performance spot

* back by rust

* remove unused imports

* remove the test

* fix up the linter

* fix api level

* move tests around to work with jna

* remove unused imports

* add back stream tests

* add back test

* fix linter issue

* update attachmentTest

* add the so files

* add the correct SO files

* remove perf test

---------

Co-authored-by: Elise Alix <elise@xmtp.com>

Author: nplasterer

library/build.gradle

@@ -78,12 +78,13 @@ dependencies {
     implementation 'io.grpc:grpc-protobuf-lite:1.51.0'
     implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4'
     implementation 'org.web3j:crypto:5.0.0'
+    implementation "net.java.dev.jna:jna:5.13.0@aar"
     api 'com.google.protobuf:protobuf-kotlin-lite:3.22.3'
     api 'org.xmtp:proto-kotlin:3.23.2'
 
-    testImplementation 'app.cash.turbine:turbine:0.12.1'
     testImplementation 'junit:junit:4.13.2'
-    testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.4'
+    androidTestImplementation 'app.cash.turbine:turbine:0.12.1'
+    androidTestImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.4'
     androidTestImplementation 'androidx.test.ext:junit:1.1.5'
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
 }

library/proguard-rules.pro

@@ -18,4 +18,8 @@
 
 # If you keep the line number information, uncomment this to
 # hide the original source file name.
-#-renamesourcefileattribute SourceFile
+#-renamesourcefileattribute SourceFile
+
+-dontwarn java.awt.*
+-keep class com.sun.jna.* { *; }
+-keepclassmembers class * extends com.sun.jna.* { public *; }

library/src/test/java/org/xmtp/android/library/AttachmentTest.kt library/src/androidTest/java/org/xmtp/android/library/AttachmentTest.kt

@@ -1,13 +1,16 @@
 package org.xmtp.android.library
 
+import androidx.test.ext.junit.runners.AndroidJUnit4
 import com.google.protobuf.kotlin.toByteStringUtf8
 import org.junit.Assert.assertEquals
 import org.junit.Test
+import org.junit.runner.RunWith
 import org.xmtp.android.library.codecs.Attachment
 import org.xmtp.android.library.codecs.AttachmentCodec
 import org.xmtp.android.library.codecs.ContentTypeAttachment
 import org.xmtp.android.library.messages.walletAddress
 
+@RunWith(AndroidJUnit4::class)
 class AttachmentTest {
     @Test
     fun testCanUseAttachmentCodec() {

library/src/test/java/org/xmtp/android/library/CodecTest.kt library/src/androidTest/java/org/xmtp/android/library/CodecTest.kt

@@ -1,8 +1,10 @@
 package org.xmtp.android.library
 
+import androidx.test.ext.junit.runners.AndroidJUnit4
 import com.google.protobuf.kotlin.toByteStringUtf8
 import org.junit.Assert.assertEquals
 import org.junit.Test
+import org.junit.runner.RunWith
 import org.xmtp.android.library.codecs.CompositeCodec
 import org.xmtp.android.library.codecs.ContentCodec
 import org.xmtp.android.library.codecs.ContentTypeId
@@ -35,7 +37,7 @@ data class NumberCodec(
     override fun decode(content: EncodedContent): Double =
         content.content.toStringUtf8().filter { it.isDigit() || it == '.' }.toDouble()
 }
-
+@RunWith(AndroidJUnit4::class)
 class CodecTest {
 
     @Test

library/src/test/java/org/xmtp/android/library/ConversationTest.kt library/src/androidTest/java/org/xmtp/android/library/ConversationTest.kt

@@ -1,12 +1,15 @@
 package org.xmtp.android.library
 
+import androidx.test.ext.junit.runners.AndroidJUnit4
 import app.cash.turbine.test
+import com.google.protobuf.kotlin.toByteString
 import kotlinx.coroutines.ExperimentalCoroutinesApi
-import kotlinx.coroutines.test.runTest
+import org.junit.Assert
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertThrows
 import org.junit.Before
 import org.junit.Test
+import org.junit.runner.RunWith
 import org.web3j.crypto.Hash
 import org.xmtp.android.library.codecs.TextCodec
 import org.xmtp.android.library.messages.EnvelopeBuilder
@@ -37,6 +40,7 @@ import java.nio.charset.StandardCharsets
 import java.util.Date
 
 @OptIn(ExperimentalCoroutinesApi::class)
+@RunWith(AndroidJUnit4::class)
 class ConversationTest {
     lateinit var fakeApiClient: FakeApiClient
     lateinit var aliceWallet: PrivateKeyBuilder
@@ -439,7 +443,7 @@ class ConversationTest {
     }
 
     @Test
-    fun testCanStreamConversationsV2() = runTest {
+    fun testCanStreamConversationsV2() = kotlinx.coroutines.test.runTest {
         bobClient.conversations.stream().test {
             val conversation = bobClient.conversations.newConversation(alice.walletAddress)
             conversation.send(content = "hi")
@@ -449,7 +453,7 @@ class ConversationTest {
     }
 
     @Test
-    fun testStreamingMessagesFromV1Conversation() = runTest {
+    fun testStreamingMessagesFromV1Conversation() = kotlinx.coroutines.test.runTest {
         // Overwrite contact as legacy
         fixtures.publishLegacyContact(client = bobClient)
         fixtures.publishLegacyContact(client = aliceClient)
@@ -478,7 +482,7 @@ class ConversationTest {
     }
 
     @Test
-    fun testStreamingMessagesFromV2Conversations() = runTest {
+    fun testStreamingMessagesFromV2Conversations() = kotlinx.coroutines.test.runTest {
         val conversation = aliceClient.conversations.newConversation(bob.walletAddress)
         conversation.streamMessages().test {
             val encoder = TextCodec()
@@ -504,7 +508,7 @@ class ConversationTest {
     }
 
     @Test
-    fun testStreamAllMessagesGetsMessageFromKnownConversation() = runTest {
+    fun testStreamAllMessagesGetsMessageFromKnownConversation() = kotlinx.coroutines.test.runTest {
         val fixtures = fixtures()
         val client = fixtures.aliceClient
         val bobConversation = fixtures.bobClient.conversations.newConversation(client.address)
@@ -568,4 +572,40 @@ class ConversationTest {
         assertEquals("hi", message.body)
         assertEquals(message.id, messageID)
     }
+
+    @Test
+    fun testFetchConversation() {
+        // Generated from JS script
+        val ints = arrayOf(
+            31, 116, 198, 193, 189, 122, 19, 254, 191, 189, 211, 215, 255, 131,
+            171, 239, 243, 33, 4, 62, 143, 86, 18, 195, 251, 61, 128, 90, 34, 126, 219, 236
+        )
+        val bytes =
+            ints.foldIndexed(ByteArray(ints.size)) { i, a, v -> a.apply { set(i, v.toByte()) } }
+
+        val key = PrivateKey.newBuilder().also {
+            it.secp256K1 = it.secp256K1.toBuilder().also { builder ->
+                builder.bytes = bytes.toByteString()
+            }.build()
+            it.publicKey = it.publicKey.toBuilder().also { builder ->
+                builder.secp256K1Uncompressed =
+                    builder.secp256K1Uncompressed.toBuilder().also { keyBuilder ->
+                        keyBuilder.bytes =
+                            KeyUtil.addUncompressedByte(KeyUtil.getPublicKey(bytes)).toByteString()
+                    }.build()
+            }.build()
+        }.build()
+
+        val client = Client().create(account = PrivateKeyBuilder(key))
+        Assert.assertEquals(client.apiClient.environment, XMTPEnvironment.DEV)
+        val conversations = client.conversations.list()
+        Assert.assertEquals(1, conversations.size)
+        val topic = conversations[0].topic
+        val conversation = client.fetchConversation(topic)
+        Assert.assertEquals(conversations[0].topic, conversation?.topic)
+        Assert.assertEquals(conversations[0].peerAddress, conversation?.peerAddress)
+
+        val noConversation = client.fetchConversation("invalid_topic")
+        Assert.assertEquals(null, noConversation)
+    }
 }

library/src/test/java/org/xmtp/android/library/ConversationsTest.kt library/src/androidTest/java/org/xmtp/android/library/ConversationsTest.kt

@@ -1,7 +1,9 @@
 package org.xmtp.android.library
 
+import androidx.test.ext.junit.runners.AndroidJUnit4
 import org.junit.Assert.assertEquals
 import org.junit.Test
+import org.junit.runner.RunWith
 import org.xmtp.android.library.codecs.TextCodec
 import org.xmtp.android.library.messages.EnvelopeBuilder
 import org.xmtp.android.library.messages.InvitationV1
@@ -16,6 +18,7 @@ import org.xmtp.android.library.messages.toPublicKeyBundle
 import org.xmtp.android.library.messages.walletAddress
 import java.util.Date
 
+@RunWith(AndroidJUnit4::class)
 class ConversationsTest {
 
     @Test

library/src/test/java/org/xmtp/android/library/InvitationTest.kt library/src/androidTest/java/org/xmtp/android/library/InvitationTest.kt

@@ -1,8 +1,14 @@
 package org.xmtp.android.library
 
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.google.protobuf.kotlin.toByteString
+import org.junit.Assert
 import org.junit.Assert.assertEquals
 import org.junit.Test
+import org.junit.runner.RunWith
 import org.xmtp.android.library.messages.InvitationV1
+import org.xmtp.android.library.messages.PrivateKey
+import org.xmtp.android.library.messages.PrivateKeyBuilder
 import org.xmtp.android.library.messages.PrivateKeyBundleV1
 import org.xmtp.android.library.messages.SealedInvitation
 import org.xmtp.android.library.messages.SealedInvitationBuilder
@@ -14,8 +20,37 @@ import org.xmtp.android.library.messages.header
 import org.xmtp.android.library.messages.toV2
 import java.util.Date
 
+@RunWith(AndroidJUnit4::class)
 class InvitationTest {
+    @Test
+    fun testExistingWallet() {
+        // Generated from JS script
+        val ints = arrayOf(
+            31, 116, 198, 193, 189, 122, 19, 254, 191, 189, 211, 215, 255, 131,
+            171, 239, 243, 33, 4, 62, 143, 86, 18, 195, 251, 61, 128, 90, 34, 126, 219, 236
+        )
+        val bytes =
+            ints.foldIndexed(ByteArray(ints.size)) { i, a, v -> a.apply { set(i, v.toByte()) } }
+        val key = PrivateKey.newBuilder().also {
+            it.secp256K1 =
+                it.secp256K1.toBuilder().also { builder -> builder.bytes = bytes.toByteString() }
+                    .build()
+            it.publicKey = it.publicKey.toBuilder().also { builder ->
+                builder.secp256K1Uncompressed =
+                    builder.secp256K1Uncompressed.toBuilder().also { keyBuilder ->
+                        keyBuilder.bytes =
+                            KeyUtil.addUncompressedByte(KeyUtil.getPublicKey(bytes)).toByteString()
+                    }.build()
+            }.build()
+        }.build()
 
+        val client = Client().create(account = PrivateKeyBuilder(key))
+        Assert.assertEquals(client.apiClient.environment, XMTPEnvironment.DEV)
+        val conversations = client.conversations.list()
+        Assert.assertEquals(1, conversations.size)
+        val message = conversations[0].messages().firstOrNull()
+        Assert.assertEquals(message?.body, "hello")
+    }
     @Test
     fun testGenerateSealedInvitation() {
         val aliceWallet = FakeWallet.generate()

library/src/androidTest/java/org/xmtp/android/library/InstrumentedTest.kt library/src/androidTest/java/org/xmtp/android/library/LocalInstrumentedTest.kt

@@ -25,7 +25,7 @@ import org.xmtp.proto.message.contents.PrivateKeyOuterClass
 import java.util.Date
 
 @RunWith(AndroidJUnit4::class)
-class InstrumentedTest {
+class LocalInstrumentedTest {
     @Test
     fun testPublishingAndFetchingContactBundlesWithWhileGeneratingKeys() {
         val aliceWallet = PrivateKeyBuilder()