fix: use test container

.github/workflows/alpha-server.yml

@@ -38,7 +38,7 @@ jobs:
       - name: Cache Docker images.
         uses: ScribeMD/docker-cache@0.5.0
         with:
-          key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yaml') }}
+          key: docker-${{ runner.os }}-${{ hashFiles('Dockerfile', 'docker-image-in-test') }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Build mini service

backend/src/main/kotlin/com/storyteller_f/Backend.kt

@@ -1,8 +1,8 @@
 package com.storyteller_f
 
-import com.storyteller_f.index.ElasticTopicDocumentService
-import com.storyteller_f.index.LuceneTopicDocumentService
-import com.storyteller_f.index.TopicDocumentService
+import com.storyteller_f.index.ElasticTopicSearchService
+import com.storyteller_f.index.LuceneTopicSearchService
+import com.storyteller_f.index.TopicSearchService
 import com.storyteller_f.media.FileSystemMediaService
 import com.storyteller_f.media.MediaService
 import com.storyteller_f.media.MinIoMediaService
@@ -20,7 +20,7 @@ import java.util.*
 
 class Backend(
     val config: Config,
-    val topicDocumentService: TopicDocumentService,
+    val topicSearchService: TopicSearchService,
     val mediaService: MediaService,
     val nameService: NameService
 )
@@ -36,8 +36,8 @@ data class ElasticConnection(val url: String, val certFile: String, val name: St
 data class MinIoConnection(val url: String, val user: String, val pass: String)
 data class DatabaseConnection(val uri: String, val driver: String, val user: String, val password: String)
 
-fun readEnv(): Map<out Any, Any> {
-    val map = ClassLoader.getSystemClassLoader().getResourceAsStream(".env")?.use {
+fun readEnv(envResFile: String = ".env"): Map<out Any, Any> {
+    val map = ClassLoader.getSystemClassLoader().getResourceAsStream(envResFile)?.use {
         Properties().apply {
             load(it)
         }
@@ -98,14 +98,14 @@ private fun mediaService(map: Map<out Any, Any>): MediaService {
 
 private fun topicDocumentService(
     map: Map<out Any, Any>,
-): TopicDocumentService {
+): TopicSearchService {
     return when (val type = map["SEARCH_SERVICE"]) {
         "elastic" -> {
             val certFile = map["CERT_FILE"] as String
             val url = map["ELASTIC_URL"] as String
             val name = map["ELASTIC_NAME"] as String
             val pass = map["ELASTIC_PASSWORD"] as String
-            ElasticTopicDocumentService(ElasticConnection(url, certFile, name, pass))
+            ElasticTopicSearchService(ElasticConnection(url, certFile, name, pass))
         }
 
         "lucene" -> {
@@ -114,7 +114,7 @@ private fun topicDocumentService(
             Napier.i {
                 "lucene path $path"
             }
-            LuceneTopicDocumentService(path)
+            LuceneTopicSearchService(path)
         }
 
         else -> throw UnsupportedOperationException("unsupported search service type [$type]")

backend/src/main/kotlin/com/storyteller_f/index/ElasticTopicDocumentService.kt → backend/src/main/kotlin/com/storyteller_f/index/ElasticTopicSearchService.kt

@@ -30,8 +30,13 @@ import org.elasticsearch.client.RestClient
 import java.io.File
 import java.io.FileInputStream
 import java.net.ConnectException
+import javax.net.ssl.SSLContext
+
+class ElasticTopicSearchService(private val connection: ElasticConnection) : TopicSearchService {
+    companion object {
+        var INJECTED_SSL_CONTEXT: SSLContext? = null
+    }
 
-class ElasticTopicDocumentService(private val connection: ElasticConnection) : TopicDocumentService {
     override suspend fun saveDocument(topics: List<TopicDocument>): Result<Unit> {
         return useElasticClient(connection) {
             topics.map { topic ->
@@ -173,12 +178,15 @@ private suspend fun <T> useElasticClient(
     elasticConnection: ElasticConnection,
     block: suspend ElasticsearchAsyncClient.() -> T
 ): Result<T> {
-    val crtStream = withContext(Dispatchers.IO) {
-        Napier.i(message = "cert path ${File(elasticConnection.certFile).canonicalPath}")
-        FileInputStream(elasticConnection.certFile)
+    val sslContext = if (elasticConnection.certFile.isNotBlank()) {
+        val crtStream = withContext(Dispatchers.IO) {
+            Napier.i(message = "cert path ${File(elasticConnection.certFile).canonicalPath}")
+            FileInputStream(elasticConnection.certFile)
+        }
+        TransportUtils.sslContextFromHttpCaCrt(crtStream)
+    } else {
+        null
     }
-    val sslContext = TransportUtils
-        .sslContextFromHttpCaCrt(crtStream)
 
     val credsProv = BasicCredentialsProvider()
     credsProv.setCredentials(

backend/src/main/kotlin/com/storyteller_f/index/LuceneTopicDocumentService.kt → backend/src/main/kotlin/com/storyteller_f/index/LuceneTopicSearchService.kt

@@ -18,7 +18,7 @@ import org.apache.lucene.search.*
 import org.apache.lucene.store.FSDirectory
 import java.nio.file.Path
 
-class LuceneTopicDocumentService(private val path: Path) : TopicDocumentService {
+class LuceneTopicSearchService(private val path: Path) : TopicSearchService {
     private val analyzer = StandardAnalyzer()
 
     override suspend fun saveDocument(topics: List<TopicDocument>): Result<Unit> {

backend/src/main/kotlin/com/storyteller_f/index/TopicDocument.kt

@@ -30,7 +30,7 @@ data class TopicDocument(
     }
 }
 
-interface TopicDocumentService {
+interface TopicSearchService {
     suspend fun saveDocument(topics: List<TopicDocument>): Result<Unit>
 
     suspend fun getDocument(idList: List<PrimaryKey>): Result<List<TopicDocument?>>

backend/src/main/kotlin/com/storyteller_f/media/FileSystemMediaService.kt

@@ -1,7 +1,7 @@
 package com.storyteller_f.media
 
 class FileSystemMediaService : MediaService {
-    override fun upload(bucketName: String, list: List<Pair<String, String>>): Result<Unit> = Result.success(Unit)
+    override fun upload(bucketName: String, list: List<UploadPack>): Result<Unit> = Result.success(Unit)
 
     override fun get(bucketName: String, objList: List<String?>): Result<List<String?>> {
         return Result.success(objList.map {

backend/src/main/kotlin/com/storyteller_f/media/MediaService.kt

@@ -1,7 +1,11 @@
 package com.storyteller_f.media
 
+import java.io.File
+
+data class UploadPack(val name: String, val path: File)
+
 interface MediaService {
-    fun upload(bucketName: String, list: List<Pair<String, String>>): Result<Unit>
+    fun upload(bucketName: String, list: List<UploadPack>): Result<Unit>
 
     fun get(bucketName: String, objList: List<String?>): Result<List<String?>>
 

backend/src/main/kotlin/com/storyteller_f/media/MinIoMediaService.kt

@@ -4,6 +4,7 @@ import com.storyteller_f.MinIoConnection
 import io.minio.*
 import io.minio.http.Method
 import java.util.concurrent.TimeUnit
+import kotlin.Result
 
 class MinIoMediaService(private val connection: MinIoConnection) : MediaService {
     override fun clean(bucketName: String): kotlin.Result<Unit> {
@@ -22,14 +23,16 @@ class MinIoMediaService(private val connection: MinIoConnection) : MediaService
         }
     }
 
-    override fun upload(bucketName: String, list: List<Pair<String, String>>): kotlin.Result<Unit> {
+    override fun upload(bucketName: String, list: List<UploadPack>): Result<Unit> {
         return useMinIoClient(connection) {
             if (!bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) {
                 makeBucket(MakeBucketArgs.builder().bucket(bucketName).build())
             }
             list.forEach { (objName, picFullPath) ->
                 uploadObject(
-                    UploadObjectArgs.builder().bucket(bucketName).`object`(objName).filename(picFullPath).build()
+                    UploadObjectArgs.builder().bucket(
+                        bucketName
+                    ).`object`(objName).filename(picFullPath.absolutePath).build()
                 )
             }
         }

cli/src/main/kotlin/com/storyteller_f/Add.kt

@@ -5,6 +5,7 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
 import com.fasterxml.jackson.module.kotlin.readValue
 import com.perraco.utils.SnowflakeFactory
 import com.storyteller_f.index.TopicDocument
+import com.storyteller_f.media.UploadPack
 import com.storyteller_f.shared.*
 import com.storyteller_f.shared.obj.PresetTopic
 import com.storyteller_f.shared.obj.PresetValue
@@ -84,7 +85,7 @@ class Add : Subcommand("add", "add entry") {
                 Triple(it, null, id)
             } else {
                 val p = "icon/${Uuid.random()}"
-                backend.mediaService.upload("apic", listOf(p to File(parentDir, icon).absolutePath))
+                backend.mediaService.upload("apic", listOf(UploadPack(p, File(parentDir, icon))))
                 Triple(it, p, id)
             }
         }
@@ -93,16 +94,12 @@ class Add : Subcommand("add", "add entry") {
                 it.users + it.admin
             }.distinct().map {
                 User.wrapRow(findUserByAId(it)!!)
-            }.associate {
-                it.aid to it
-            }
+            }.associateBy { it.aid }
             val communityMap = l.mapNotNull {
                 it.community
             }.distinct().map {
                 Community.wrapRow(findCommunityByAId(it)!!)
-            }.associate {
-                it.aid to it
-            }
+            }.associateBy { it.aid }
             val idList = Rooms.batchInsert(data) { (it, p, id) ->
                 this[Rooms.id] = id
                 this[Rooms.aid] = it.id
@@ -132,9 +129,7 @@ class Add : Subcommand("add", "add entry") {
                 it.author
             }.distinct().map {
                 User.wrapRow(findUserByAId(it)!!)
-            }.associate {
-                it.aid!! to it
-            }
+            }.associateBy { it.aid!! }
             data.groupBy {
                 it.community != null
             }.forEach { (t, u) ->
@@ -147,15 +142,12 @@ class Add : Subcommand("add", "add entry") {
         }.getOrThrow()
     }
 
-    @OptIn(ExperimentalUnsignedTypes::class)
     private suspend fun addTopicsIntoRoom(u: List<PresetTopic>, userList: Map<String, User>, parentDir: File) {
         val roomList = u.mapNotNull {
             it.room
         }.distinct().map {
             Room.wrapRow(Room.findRoomByAId(it).firstOrNull()!!)
-        }.associate {
-            it.aid to it
-        }
+        }.associateBy { it.aid }
         val ids = insertRoomTopicBaseLevel(u, userList, roomList)
         // 检查聊天室是属于社区的还是私有的
         val roomIsPrivate = roomList.mapValues { (_, value) ->
@@ -166,7 +158,6 @@ class Add : Subcommand("add", "add entry") {
         insertUnEncryptedTopic(u, roomIsPrivate, parentDir, ids, roomList, userList)
     }
 
-    @OptIn(ExperimentalUnsignedTypes::class)
     private suspend fun insertRoomTopicBaseLevel(
         u: List<PresetTopic>,
         userList: Map<String, User>,
@@ -210,7 +201,6 @@ class Add : Subcommand("add", "add entry") {
         return ids
     }
 
-    @OptIn(ExperimentalUnsignedTypes::class)
     private suspend fun insertUnEncryptedTopic(
         presetTopicList: List<PresetTopic>,
         roomIsPrivate: Map<String, Boolean>,
@@ -226,7 +216,7 @@ class Add : Subcommand("add", "add entry") {
                 null
             }
         }
-        backend.topicDocumentService.saveDocument(
+        backend.topicSearchService.saveDocument(
             topicsPublic.map { (first, second) ->
                 val content = getTopicContent(first, parentDir)
                 val level = first.level
@@ -246,7 +236,6 @@ class Add : Subcommand("add", "add entry") {
         )
     }
 
-    @OptIn(ExperimentalUnsignedTypes::class)
     private suspend fun insertEncryptedTopic(
         roomIsPrivate: Map<String, Boolean>,
         parentDir: File,
@@ -274,9 +263,7 @@ class Add : Subcommand("add", "add entry") {
                 }.map {
                     it.publicKey to it.id
                 }
-        }.associate {
-            it.first to it
-        }
+        }.associateBy { it.first }
         val encrypted = topicsPrivate.map { (addTopic, index) ->
             val (first, aesBytes) = encrypt(getTopicContent(addTopic, parentDir))
             Tuple4(index, first, aesBytes, addTopic)
@@ -299,7 +286,6 @@ class Add : Subcommand("add", "add entry") {
         }
     }
 
-    @OptIn(ExperimentalUnsignedTypes::class)
     private suspend fun addTopicsIntoCommunity(u: List<PresetTopic>, userList: Map<String, User>, parentDir: File) {
         val communityList = u.mapNotNull {
             it.community
@@ -308,15 +294,15 @@ class Add : Subcommand("add", "add entry") {
             if (rowCommunity == null) {
                 error("$it not found")
             } else {
-                Community.wrapRow(rowCommunity).let {
-                    it.aid to it.id
+                Community.wrapRow(rowCommunity).let { community ->
+                    community.aid to community.id
                 }
             }
         }.associate {
             it.first to it.second
         }
         val ids = insertCommunityTopicTopLevel(u, userList, communityList)
-        backend.topicDocumentService.saveDocument(
+        backend.topicSearchService.saveDocument(
             u.mapIndexedNotNull { index, topic ->
                 if (topic.community != null) {
                     val content = getTopicContent(topic, parentDir)
@@ -408,8 +394,7 @@ class Add : Subcommand("add", "add entry") {
                 Tuple5(it, null, derPublicKey, ad, id)
             } else {
                 val p = "icon/${Uuid.random()}"
-                val absolutePath = File(parentDir, icon).absolutePath
-                backend.mediaService.upload("apic", listOf(p to absolutePath))
+                backend.mediaService.upload("apic", listOf(UploadPack(p, File(parentDir, icon))))
                 Tuple5(it, null, derPublicKey, ad, id)
             }
         }
@@ -436,7 +421,7 @@ class Add : Subcommand("add", "add entry") {
                 Triple(it, null, id)
             } else {
                 val p = "icon/${Uuid.random()}"
-                backend.mediaService.upload("apic", listOf(p to File(parentDir, icon).absolutePath))
+                backend.mediaService.upload("apic", listOf(UploadPack(p, File(parentDir, icon))))
                 Triple(it, p, id)
             }
         }

cli/src/main/kotlin/com/storyteller_f/CleanCommand.kt

@@ -14,7 +14,7 @@ class CleanCommand : Subcommand("clean", "clean all data") {
         }
         backend.mediaService.clean("apic")
         runBlocking {
-            backend.topicDocumentService.clean()
+            backend.topicSearchService.clean()
         }
         Napier.i {
             "clean done."

docker-image-in-test

@@ -0,0 +1,2 @@
+docker.elastic.co/elasticsearch/elasticsearch:7.9.2
+minio/minio:RELEASE.2023-09-04T19-57-37Z

gradle/libs.versions.toml

@@ -25,6 +25,7 @@ kotlinxDatetime = "0.6.1"
 ktomlCore = "0.5.1"
 lifecycleViewmodelCompose = "2.8.7"
 luceneCore = "10.0.0"
+minioVersion = "1.20.0"
 mosaicRuntime = "0.14.0"
 multiplatformCryptoLibsodiumBindings = "0.9.2"
 multiplatformMarkdownRenderer = "0.27.0"
@@ -60,6 +61,7 @@ kover = "0.7.4"
 
 [libraries]
 desugar_jdk_libs = { module = "com.android.tools:desugar_jdk_libs", version.ref = "desugar_jdk_libs" }
+elasticsearch = { module = "org.testcontainers:elasticsearch", version.ref = "minioVersion" }
 emoji-compose-m3 = { module = "org.kodein.emoji:emoji-compose-m3", version.ref = "emojiKt" }
 emoji-kt = { module = "org.kodein.emoji:emoji-kt", version.ref = "emojiKt" }
 geoip2 = { module = "com.maxmind.geoip2:geoip2", version.ref = "geoip2" }
@@ -155,6 +157,7 @@ postgresql = { module = "org.postgresql:postgresql", version.ref = "postgresql"
 slf4j-simple = { module = "org.slf4j:slf4j-simple", version.ref = "slf4jSimple" }
 sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqliteJdbc" }
 sunny-chung-composable-table = { module = "io.github.sunny-chung:composable-table", version.ref = "composableTableVersion" }
+testcontainers-minio = { module = "org.testcontainers:minio", version.ref = "minioVersion" }
 uri-kmp = { module = "com.eygraber:uri-kmp", version.ref = "uriKmp" }
 
 [plugins]

server/build.gradle.kts

@@ -30,7 +30,8 @@ dependencies {
     testImplementation(libs.ktor.server.test.host)
     testImplementation(libs.kotlin.test)
     testImplementation(libs.h2)
-    testImplementation(libs.jimfs)
+    testImplementation(libs.testcontainers.minio)
+    testImplementation(libs.elasticsearch)
     implementation(libs.pdfbox)
     implementation(libs.napier)
     implementation(libs.emoji.reader.jvm)

server/src/main/kotlin/com/storyteller_f/a/server/Websocket.kt

@@ -180,7 +180,7 @@ private suspend fun savePlainTopicContent(
 ): Result<TopicInfo> {
     return DatabaseFactory.dbQuery {
         Topic.new(topic)
-        backend.topicDocumentService.saveDocument(
+        backend.topicSearchService.saveDocument(
             listOf(TopicDocument.fromTopic(topic, content))
         ).getOrThrow()