perf: Refactor code and optimize cache in main.rs and Cargo.toml

- Remove unnecessary dependencies from Cargo.toml
- Optimize cache key type from String to u64 in src/main.rs
- Increase cache expiration time from 60 seconds to 10 minutes
- Replace logging statements with print statements in src/main.rs
- Add a 3-second sleep to allow SPA content to load before fetching it

Author: 7a6163

Cargo.toml

@@ -10,6 +10,3 @@ serde = { version = "1.0", features = ["derive"] }
 tokio = { version = "1", features = ["full"] }
 serde_json = "1.0"
 lru = "0.12"
-select = "0.5"
-log = "0.4"
-env_logger = "0.10"

src/main.rs

@@ -6,23 +6,22 @@ use warp::Filter;
 use serde::Deserialize;
 use std::collections::VecDeque;
 use lru::LruCache;
-use tokio::time::{Duration, Instant};
 use std::num::NonZeroUsize;
-use std::env;
-use log::{info, error};
+use std::time::{Duration, Instant};
+use std::collections::hash_map::DefaultHasher;
+use std::hash::{Hash, Hasher};
+
+const CACHE_TTL: Duration = Duration::from_secs(600); // 10 minutes
 
 #[tokio::main]
 async fn main() {
-    env_logger::init();
-
     let args: Vec<&OsStr> = vec![
         OsStr::new("--no-sandbox"),
         OsStr::new("--disable-gpu"),
         OsStr::new("--disable-dev-shm-usage"),
         OsStr::new("--headless"),
         OsStr::new("--disable-software-rasterizer"),
         OsStr::new("--no-zygote"),
-        OsStr::new("--single-process"),
     ];
 
     let browser = Arc::new(
@@ -36,14 +35,7 @@ async fn main() {
 
     let semaphore = Arc::new(Semaphore::new(4));
     let tab_pool = Arc::new(Mutex::new(VecDeque::new()));
-
-    let cache_maxsize = env::var("CACHE_MAXSIZE")
-        .unwrap_or_else(|_| "100".to_string())
-        .parse::<usize>()
-        .expect("CACHE_MAXSIZE must be a positive integer");
-
-    let cache_size = NonZeroUsize::new(cache_maxsize).expect("CACHE_MAXSIZE must be a non-zero integer");
-    let cache = Arc::new(Mutex::new(LruCache::new(cache_size))); // Cache max size from environment variable
+    let cache = Arc::new(Mutex::new(LruCache::new(NonZeroUsize::new(1000).unwrap())));
 
     let render_route = warp::path("html")
         .and(warp::query::<RenderQuery>())
@@ -53,7 +45,7 @@ async fn main() {
         .and(with_cache(cache.clone()))
         .and_then(render_handler);
 
-    info!("Server running on http://0.0.0.0:8080");
+    println!("Server running on http://0.0.0.0:8080");
     warp::serve(render_route).run(([0, 0, 0, 0], 8080)).await;
 }
 
@@ -81,8 +73,8 @@ fn with_tab_pool(
 }
 
 fn with_cache(
-    cache: Arc<Mutex<LruCache<String, (String, Instant)>>>,
-) -> impl Filter<Extract = (Arc<Mutex<LruCache<String, (String, Instant)>>>,), Error = std::convert::Infallible> + Clone {
+    cache: Arc<Mutex<LruCache<u64, (String, Instant)>>>,
+) -> impl Filter<Extract = (Arc<Mutex<LruCache<u64, (String, Instant)>>>,), Error = std::convert::Infallible> + Clone {
     warp::any().map(move || cache.clone())
 }
 
@@ -96,63 +88,66 @@ async fn render_handler(
     browser: Arc<Browser>,
     semaphore: Arc<Semaphore>,
     tab_pool: Arc<Mutex<VecDeque<Arc<Tab>>>>,
-    cache: Arc<Mutex<LruCache<String, (String, Instant)>>>,
+    cache: Arc<Mutex<LruCache<u64, (String, Instant)>>>,
 ) -> Result<impl warp::Reply, warp::Rejection> {
     let _permit = semaphore.acquire().await;
 
-    let cache_ttl_secs = env::var("CACHE_TTL")
-        .unwrap_or_else(|_| "60".to_string())
-        .parse::<u64>()
-        .expect("CACHE_TTL must be a positive integer");
-    let cache_ttl = Duration::new(cache_ttl_secs, 0);
+    let hash = hash_url(&query.url);
 
-    info!("Initial request to {}", query.url);
-
-    let start_time = Instant::now();
-    
+    // Check if the response is in the cache and if it's still valid.
     let mut cache_guard = cache.lock().await;
-    if let Some((content, timestamp)) = cache_guard.get(&query.url) {
-        if timestamp.elapsed() < cache_ttl {
-            info!("Cache hit for {}", query.url);
-            return Ok(warp::reply::html(content.clone()));
+    if let Some((response, timestamp)) = cache_guard.get(&hash) {
+        if timestamp.elapsed() < CACHE_TTL {
+            return Ok(warp::reply::html(response.clone()));
         }
     }
 
+    // If not, use headless_chrome to get the DOM.
+    drop(cache_guard); // release the lock before using headless_chrome
     let tab = {
         let mut pool = tab_pool.lock().await;
         if let Some(tab) = pool.pop_front() {
             tab
         } else {
             browser.new_tab().map_err(|e| {
-                error!("Failed to create new tab: {:?}", e);
+                eprintln!("Failed to create new tab: {:?}", e);
                 warp::reject::custom(CustomError)
             })?
         }
     };
 
     tab.navigate_to(&query.url).map_err(|e| {
-        error!("Failed to navigate to URL: {:?}", e);
+        eprintln!("Failed to navigate to URL: {:?}", e);
         warp::reject::custom(CustomError)
     })?;
     tab.wait_until_navigated().map_err(|e| {
-        error!("Failed to wait until navigated: {:?}", e);
+        eprintln!("Failed to wait until navigated: {:?}", e);
         warp::reject::custom(CustomError)
     })?;
 
+    // Wait for a bit to allow SPA content to load.
+    tokio::time::sleep(Duration::from_secs(3)).await;
+
     let content = tab.get_content().map_err(|e| {
-        error!("Failed to get content: {:?}", e);
+        eprintln!("Failed to get content: {:?}", e);
         warp::reject::custom(CustomError)
     })?;
 
+    {
+        let mut cache_guard = cache.lock().await;
+        cache_guard.put(hash, (content.clone(), Instant::now()));
+    }
+
     {
         let mut pool = tab_pool.lock().await;
         pool.push_back(Arc::clone(&tab));
     }
 
-    cache_guard.put(query.url.clone(), (content.clone(), Instant::now()));
-
-    let duration = start_time.elapsed();
-    info!("Got {} in {:?} for {}", content.len(), duration, query.url);
-
     Ok(warp::reply::html(content))
-}
+}
+
+fn hash_url(url: &str) -> u64 {
+    let mut hasher = DefaultHasher::new();
+    url.hash(&mut hasher);
+    hasher.finish()
+}