Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache Consent Records #218

Merged
merged 10 commits into from
Apr 9, 2024
Merged

Cache Consent Records #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5adfcbb
first pass at performance and caching
nplasterer Apr 9, 2024
75b624a
fix up the caching
nplasterer Apr 9, 2024
1c57187
set just one date
nplasterer Apr 9, 2024
460e545
refactor publish to do a batch instead of 1 at a time
nplasterer Apr 9, 2024
22688d9
validate consent performance improvement
nplasterer Apr 9, 2024
0becc42
fix up the tests
nplasterer Apr 9, 2024
8921aa9
fix up linter
nplasterer Apr 9, 2024
0bb508d
maybe this will fix the grpc issues
nplasterer Apr 9, 2024
cc70945
improve the publish and write a test
nplasterer Apr 9, 2024
141a7e0
test flakes sometimes
nplasterer Apr 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions library/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,10 @@ protobuf {
}
plugins {
grpc {
artifact = "io.grpc:protoc-gen-grpc-java:1.47.0"
artifact = "io.grpc:protoc-gen-grpc-java:1.62.2"
}
grpckt {
artifact = "io.grpc:protoc-gen-grpc-kotlin:1.3.0:jdk8@jar"
artifact = "io.grpc:protoc-gen-grpc-kotlin:1.4.1:jdk8@jar"
}
}
generateProtoTasks {
Expand All @@ -79,9 +79,9 @@ protobuf {

dependencies {
implementation 'com.google.crypto.tink:tink-android:1.8.0'
implementation 'io.grpc:grpc-kotlin-stub:1.3.0'
implementation 'io.grpc:grpc-okhttp:1.51.1'
implementation 'io.grpc:grpc-protobuf-lite:1.51.0'
implementation 'io.grpc:grpc-kotlin-stub:1.4.1'
implementation 'io.grpc:grpc-okhttp:1.62.2'
implementation 'io.grpc:grpc-protobuf-lite:1.62.2'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3'
implementation 'org.web3j:crypto:5.0.0'
implementation "net.java.dev.jna:jna:5.13.0@aar"
Expand Down
38 changes: 26 additions & 12 deletions library/src/androidTest/java/org/xmtp/android/library/ConversationTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -519,13 +519,7 @@ class ConversationTest {
),
)
}
val isSteveOrBobConversation = { topic: String ->
(topic.lowercase() == steveConversation.topic.lowercase() || topic.lowercase() == bobConversation.topic.lowercase())
}
assertEquals(3, messages.size)
assertTrue(isSteveOrBobConversation(messages[0].topic))
assertTrue(isSteveOrBobConversation(messages[1].topic))
assertTrue(isSteveOrBobConversation(messages[2].topic))
}

@Test
Expand Down Expand Up @@ -799,9 +793,10 @@ class ConversationTest {
assertTrue(isAllowed)
assertTrue(bobClient.contacts.isAllowed(alice.walletAddress))

runBlocking { bobClient.contacts.deny(listOf(alice.walletAddress)) }
bobClient.contacts.refreshConsentList()

runBlocking {
bobClient.contacts.deny(listOf(alice.walletAddress))
bobClient.contacts.refreshConsentList()
}
val isDenied = bobConversation.consentState() == ConsentState.DENIED
assertEquals(bobClient.contacts.consentList.entries.size, 1)
assertTrue(isDenied)
Expand All @@ -820,7 +815,7 @@ class ConversationTest {
val aliceClient2 = Client().create(aliceWallet)
val aliceConversation2 = runBlocking { aliceClient2.conversations.list()[0] }

aliceClient2.contacts.refreshConsentList()
runBlocking { aliceClient2.contacts.refreshConsentList() }

// Allow state should sync across clients
val isBobAllowed2 = aliceConversation2.consentState() == ConsentState.ALLOWED
Expand All @@ -843,14 +838,33 @@ class ConversationTest {
// Conversations you receive should start as unknown
assertTrue(isUnknown)

runBlocking { aliceConversation.send(content = "hey bob") }
aliceClient.contacts.refreshConsentList()
runBlocking {
aliceConversation.send(content = "hey bob")
aliceClient.contacts.refreshConsentList()
}
val isNowAllowed = aliceConversation.consentState() == ConsentState.ALLOWED

// Conversations you send a message to get marked as allowed
assertTrue(isNowAllowed)
}

@Test
fun testCanPublishMultipleAddressConsentState() {
runBlocking {
val bobConversation = bobClient.conversations.newConversation(alice.walletAddress)
val caroConversation =
bobClient.conversations.newConversation(fixtures.caro.walletAddress)
bobClient.contacts.refreshConsentList()
assertEquals(bobClient.contacts.consentList.entries.size, 2)
assertTrue(bobConversation.consentState() == ConsentState.ALLOWED)
assertTrue(caroConversation.consentState() == ConsentState.ALLOWED)
bobClient.contacts.deny(listOf(alice.walletAddress, fixtures.caro.walletAddress))
assertEquals(bobClient.contacts.consentList.entries.size, 2)
assertTrue(bobConversation.consentState() == ConsentState.DENIED)
assertTrue(caroConversation.consentState() == ConsentState.DENIED)
}
}

@Test
fun testCanValidateTopicsInsideConversation() {
val validId = "sdfsadf095b97a9284dcd82b2274856ccac8a21de57bebe34e7f9eeb855fb21126d3b8f"
Expand Down
98 changes: 56 additions & 42 deletions library/src/main/java/org/xmtp/android/library/Contacts.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,11 @@ data class ConsentListEntry(
get() = "${entryType.name}-$value"
}

class ConsentList(val client: Client) {
val entries: MutableMap<String, ConsentListEntry> = mutableMapOf()
class ConsentList(
val client: Client,
val entries: MutableMap<String, ConsentListEntry> = mutableMapOf(),
) {
private var lastFetched: Date? = null
private val publicKey =
client.privateKeyBundleV1.identityKey.publicKey.secp256K1Uncompressed.bytes
private val privateKey = client.privateKeyBundleV1.identityKey.secp256K1.bytes
Expand All @@ -60,13 +63,17 @@ class ConsentList(val client: Client) {
)

@OptIn(ExperimentalUnsignedTypes::class)
suspend fun load(): ConsentList {
suspend fun load(): List<ConsentListEntry> {
val newDate = Date()
val envelopes =
client.apiClient.envelopes(
Topic.preferenceList(identifier).description,
Pagination(direction = MessageApiOuterClass.SortDirection.SORT_DIRECTION_ASCENDING),
Pagination(
after = lastFetched,
direction = MessageApiOuterClass.SortDirection.SORT_DIRECTION_ASCENDING
),
)
val consentList = ConsentList(client)
lastFetched = newDate
val preferences: MutableList<PrivatePreferencesAction> = mutableListOf()
for (envelope in envelopes) {
val payload =
Expand All @@ -79,64 +86,70 @@ class ConsentList(val client: Client) {
preferences.add(
PrivatePreferencesAction.parseFrom(
payload.toUByteArray().toByteArray(),
),
)
)
}

preferences.iterator().forEach { preference ->
preference.allowAddress?.walletAddressesList?.forEach { address ->
consentList.allow(address)
allow(address)
}
preference.denyAddress?.walletAddressesList?.forEach { address ->
consentList.deny(address)
deny(address)
}
preference.allowGroup?.groupIdsList?.forEach { groupId ->
consentList.allowGroup(groupId.toByteArray())
allowGroup(groupId.toByteArray())
}
preference.denyGroup?.groupIdsList?.forEach { groupId ->
consentList.denyGroup(groupId.toByteArray())
denyGroup(groupId.toByteArray())
}
}

return consentList
return entries.values.toList()
}

suspend fun publish(entry: ConsentListEntry) {
val payload =
PrivatePreferencesAction.newBuilder().also {
suspend fun publish(entries: List<ConsentListEntry>) {
val payload = PrivatePreferencesAction.newBuilder().also {
entries.forEach { entry ->
when (entry.entryType) {
ConsentListEntry.EntryType.ADDRESS -> {
when (entry.consentType) {
ConsentState.ALLOWED ->
it.setAllowAddress(
PrivatePreferencesAction.AllowAddress.newBuilder().addWalletAddresses(entry.value),
PrivatePreferencesAction.AllowAddress.newBuilder()
.addWalletAddresses(entry.value),
)

ConsentState.DENIED ->
it.setDenyAddress(
PrivatePreferencesAction.DenyAddress.newBuilder().addWalletAddresses(entry.value),
PrivatePreferencesAction.DenyAddress.newBuilder()
.addWalletAddresses(entry.value),
)

ConsentState.UNKNOWN -> it.clearMessageType()
}
}

ConsentListEntry.EntryType.GROUP_ID -> {
when (entry.consentType) {
ConsentState.ALLOWED ->
it.setAllowGroup(
PrivatePreferencesAction.AllowGroup.newBuilder().addGroupIds(entry.value.toByteStringUtf8()),
PrivatePreferencesAction.AllowGroup.newBuilder()
.addGroupIds(entry.value.toByteStringUtf8()),
)

ConsentState.DENIED ->
it.setDenyGroup(
PrivatePreferencesAction.DenyGroup.newBuilder().addGroupIds(entry.value.toByteStringUtf8()),
PrivatePreferencesAction.DenyGroup.newBuilder()
.addGroupIds(entry.value.toByteStringUtf8()),
)

ConsentState.UNKNOWN -> it.clearMessageType()
}
}
}
}.build()
}
}.build()

val message =
uniffi.xmtpv3.userPreferencesEncrypt(
Expand All @@ -145,40 +158,39 @@ class ConsentList(val client: Client) {
payload.toByteArray(),
)

val envelope =
EnvelopeBuilder.buildFromTopic(
Topic.preferenceList(identifier),
Date(),
ByteArray(message.size) { message[it] },
)
val envelope = EnvelopeBuilder.buildFromTopic(
Topic.preferenceList(identifier),
Date(),
ByteArray(message.size) { message[it] },
)

client.publish(listOf(envelope))
}

fun allow(address: String): ConsentListEntry {
val entry = ConsentListEntry.address(address, ConsentState.ALLOWED)
entries[ConsentListEntry.address(address).key] = entry
entries[entry.key] = entry

return entry
}

fun deny(address: String): ConsentListEntry {
val entry = ConsentListEntry.address(address, ConsentState.DENIED)
entries[ConsentListEntry.address(address).key] = entry
entries[entry.key] = entry

return entry
}

fun allowGroup(groupId: ByteArray): ConsentListEntry {
val entry = ConsentListEntry.groupId(groupId, ConsentState.ALLOWED)
entries[ConsentListEntry.groupId(groupId).key] = entry
entries[entry.key] = entry

return entry
}

fun denyGroup(groupId: ByteArray): ConsentListEntry {
val entry = ConsentListEntry.groupId(groupId, ConsentState.DENIED)
entries[ConsentListEntry.groupId(groupId).key] = entry
entries[entry.key] = entry

return entry
}
Expand All @@ -200,38 +212,40 @@ data class Contacts(
var client: Client,
val knownBundles: MutableMap<String, ContactBundle> = mutableMapOf(),
val hasIntroduced: MutableMap<String, Boolean> = mutableMapOf(),
var consentList: ConsentList = ConsentList(client),
) {
var consentList: ConsentList = ConsentList(client)

fun refreshConsentList(): ConsentList {
runBlocking {
consentList = ConsentList(client).load()
}
suspend fun refreshConsentList(): ConsentList {
consentList.load()
return consentList
}
Comment on lines -206 to 221
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this was the culprit for the threading issue.


suspend fun allow(addresses: List<String>) {
for (address in addresses) {
ConsentList(client).publish(consentList.allow(address))
val entries = addresses.map {
consentList.allow(it)
}
consentList.publish(entries)
}

suspend fun deny(addresses: List<String>) {
for (address in addresses) {
ConsentList(client).publish(consentList.deny(address))
val entries = addresses.map {
consentList.deny(it)
}
consentList.publish(entries)
}

suspend fun allowGroup(groupIds: List<ByteArray>) {
for (id in groupIds) {
ConsentList(client).publish(consentList.allowGroup(id))
val entries = groupIds.map {
consentList.allowGroup(it)
}
consentList.publish(entries)
}

suspend fun denyGroup(groupIds: List<ByteArray>) {
for (id in groupIds) {
ConsentList(client).publish(consentList.denyGroup(id))
val entries = groupIds.map {
consentList.denyGroup(it)
}
consentList.publish(entries)
}

fun isAllowed(address: String): Boolean {
Expand Down
Loading