fix: Update search interaction with HLTB API to current version (#25)

This commit fixes search by doing this: - Fixing wrong search code being parsed. Because HLTB changed their script for getting code from `"api/search".concat("code")` to `"api/search".concat("co").concat("de")` proxy only parsed first part ("co") of the "code" I think in the future, it will be problem again because HLTB did it for making scrapping harder. - Updated query schema for current HLTB search version HLTB added "useCache", "lists" fields to the root of schema and "subgenre" field to "root/games/gameplay" - Updated headers By using real API search headers and removing ones that don't affect search success, I left with change "Authority" to "Host" and adding "Accept" headet. Also I updated user-agent
DareFox · Dec 9, 2024 · c500e24 · c500e24
1 parent 61be90b
commit c500e24
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 24 deletions.
diff --git a/src/main/kotlin/io/github/darefox/hltbproxy/hltb/Extensions.kt b/src/main/kotlin/io/github/darefox/hltbproxy/hltb/Extensions.kt
@@ -7,10 +7,11 @@ import org.http4k.format.KotlinxSerialization.auto
 
 fun Request.hltbDefaultHeaders(url: String, bodyIsJson: Boolean = true): Request {
     val headers = mutableListOf(
-        "Authority" to "howlongtobeat.com",
+        "Host" to "howlongtobeat.com",
         "Origin" to "https://howlongtobeat.com",
         "Referer" to "$url",
-        "User-Agent" to "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"
+        "User-Agent" to "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
+        "Accept" to "*/*"
     ).also {
         if (bodyIsJson) it += "Content-Type" to "application/json"
     }

diff --git a/src/main/kotlin/io/github/darefox/hltbproxy/hltb/HLTB.kt b/src/main/kotlin/io/github/darefox/hltbproxy/hltb/HLTB.kt
@@ -31,25 +31,31 @@ object HLTB {
         var keyIsNotUpdated = true
         lateinit var response: Response
         while (true) {
-            val call = client(
-                Request(POST, "https://howlongtobeat.com/api/search/${getSearchKey()}")
-                    .hltbJsonRequest(url, queryObj)
-            )
-
-            if (call.bodyString().startsWith("<!DOCTYPE html>") && keyIsNotUpdated) {
-                log.info("Search key is no longer valid, calling update")
-                updateSearchKey()
-                keyIsNotUpdated = false
+            val request = Request(POST, "https://howlongtobeat.com/api/search/${getSearchKey()}")
+                .hltbJsonRequest(url, queryObj)
+            val call = client(request)
+
+            val isResponseInHtml = call.bodyString().startsWith("<!DOCTYPE html>")
+
+
+            if (call.status.successful) {
+                if (isResponseInHtml) {
+                    if (keyIsNotUpdated) {
+                        log.info("Search key is no longer valid, calling update")
+                        updateSearchKey()
+                        keyIsNotUpdated = false
+                        continue
+                    }
+                    error("Search key was updated, but server returned html. Maybe search API was changed")
+                } else {
+                    response = call
+                    break
+                }
             } else {
-                response = call
-                break
+                error("Server returned not successful code: ${call.status.code}\n\n\nBody: ${call.bodyString()}")
             }
         }
 
-        if (response.bodyString().startsWith("<!DOCTYPE html>")) {
-            updateSearchKey()
-        }
-
         return Body.auto<HltbQueryResponse>().toLens().invoke(response)
     }
 
@@ -102,6 +108,7 @@ object HLTB {
                     val key = findKeyInScripts(codeOfScripts)
 
                     searchKey = key
+                    log.info("Key is $searchKey")
                     key
                 }
             }
@@ -136,11 +143,19 @@ object HLTB {
         }
     }
 
-    private fun findKeyInScripts(codes: Sequence<String>): String {
-        val regex = "(?<=api/search/\"\\.concat\\(\")[a-zA-Z0-9]+".toRegex()
-        for (code in codes) {
-            val result = regex.find(code)?.value ?: continue
-            return result
+    private fun findKeyInScripts(scripts: Sequence<String>): String {
+        val fetchLineRegex = "(?<=api/search/).*?,".toRegex()
+        val concatValuesRegex = "(?<=concat\\(\").*?(?=\")".toRegex()
+        for (script in scripts) {
+            val fetchLine = fetchLineRegex.find(script) ?: continue
+            log.info("found fetch line: ${fetchLine.value}")
+            val concatValues = concatValuesRegex.findAll(fetchLine.value).toList()
+            if (concatValues.isEmpty()) {
+                log.info("but concat values are empty")
+                continue
+            }
+
+            return concatValues.joinToString("") { it.value }
         }
 
         error("Can't find search key")

diff --git a/src/main/kotlin/io/github/darefox/hltbproxy/hltb/HltbQueryRequest.kt b/src/main/kotlin/io/github/darefox/hltbproxy/hltb/HltbQueryRequest.kt
@@ -9,8 +9,16 @@ data class HltbQueryRequest(
     val searchPage: Int,
     val size: Int,
     val searchOptions: SearchOptions,
+    val lists: Lists,
+    val useCache: Boolean = true,
 )
 
+@Serializable
+data class Lists(
+    val sortCategory: String = "follows"
+)
+
+
 @Serializable
 data class SearchOptions(
     val games: Games,
@@ -43,6 +51,7 @@ data class Gameplay(
     val perspective: String,
     val flow: String,
     val genre: String,
+    val subGenre: String
 )
 
 @Serializable
@@ -69,14 +78,15 @@ fun createQueryObj(title: String, page: Int): HltbQueryRequest {
                 sortCategory = "popular",
                 rangeCategory = "main",
                 rangeTime = RangeTime(null, null),
-                gameplay = Gameplay("", "", ""),
+                gameplay = Gameplay("", "", "", ""),
                 rangeYear = RangeYear("", ""),
                 modifier = ""
             ),
             users = Users(sortCategory = "postcount"),
             filter = "",
             sort = 0,
             randomizer = 0
-        )
+        ),
+        lists = Lists()
     )
 }