From 3d3281c17936b6c41c05f857a8a79f1f541cde8f Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Mon, 22 Jan 2024 16:53:11 +0000 Subject: [PATCH] Fixes #103 --- src/datasource_database.rs | 41 +++++++++++------------- src/main.rs | 64 +++++++++++--------------------------- 2 files changed, 38 insertions(+), 67 deletions(-) diff --git a/src/datasource_database.rs b/src/datasource_database.rs index e60af94..49ef13c 100644 --- a/src/datasource_database.rs +++ b/src/datasource_database.rs @@ -832,27 +832,27 @@ impl SourceDatabase { let nslist = primary_pagelist.group_by_namespace()?; let mut batches: Vec = vec![]; nslist.iter().for_each(|nsgroup| { - nsgroup.1.chunks(PAGE_BATCH_SIZE*2).for_each(|titles| { - let mut sql = Platform::sql_tuple(); - sql.0 = "SELECT DISTINCT p.page_id,p.page_title,p.page_namespace,(SELECT rev_timestamp FROM revision WHERE rev_id=p.page_latest LIMIT 1) AS page_touched,p.page_len ".to_string() ; - sql.0 += ¶ms.link_count_sql; - sql.0 += " FROM page p"; - if !params.is_before_after_done { - Platform::append_sql(&mut sql, params.sql_before_after.clone()); - } - sql.0 += " WHERE (p.page_namespace="; - sql.0 += &nsgroup.0.to_string(); - sql.0 += " AND p.page_title IN ("; - Platform::append_sql(&mut sql, Platform::prep_quote(&titles)); - sql.0 += "))"; - batches.push(sql); - }); - }); + nsgroup.1.chunks(PAGE_BATCH_SIZE*2).for_each(|titles| { + let mut sql = Platform::sql_tuple(); + sql.0 = "SELECT DISTINCT p.page_id,p.page_title,p.page_namespace,(SELECT rev_timestamp FROM revision WHERE rev_id=p.page_latest LIMIT 1) AS page_touched,p.page_len ".to_string() ; + sql.0 += ¶ms.link_count_sql; + sql.0 += " FROM page p"; + if !params.is_before_after_done { + Platform::append_sql(&mut sql, params.sql_before_after.clone()); + } + sql.0 += " WHERE (p.page_namespace="; + sql.0 += &nsgroup.0.to_string(); + sql.0 += " AND p.page_title IN ("; + Platform::append_sql(&mut sql, Platform::prep_quote(&titles)); + sql.0 += "))"; + batches.push(sql); + }); + }); // Either way, it's done params.is_before_after_done = true; - let wiki = primary_pagelist.wiki()?.ok_or_else(|| "No wiki 12345".to_string())?; + let wiki = primary_pagelist.wiki()?.ok_or_else(|| "No wiki given in datasource_database::get_pages_pagelist".to_string())?; let mut futures : Vec<_> = vec![] ; for sql in batches { @@ -898,8 +898,7 @@ impl SourceDatabase { state: &AppState, primary_pagelist: Option<&PageList>, ) -> Result { - let mut params = - self.get_pages_initialize_query(state, primary_pagelist).await?; + let mut params = self.get_pages_initialize_query(state, primary_pagelist).await?; let mut sql = Platform::sql_tuple(); @@ -995,7 +994,7 @@ impl SourceDatabase { Platform::profile("DSDB::get_pages_for_primary STARTING", Some(sql.1.len())); // Namespaces - if !self.params.namespace_ids.is_empty() { + if !self.params.namespace_ids.is_empty() && primary!="pagelist" { let namespace_ids = &self .params .namespace_ids @@ -1227,8 +1226,6 @@ impl SourceDatabase { Some(sql.1.len()), ); - //println!("{:?}",&sql); - let sql_1_len = sql.1.len() ; let rows = conn.exec_iter(sql.0.as_str(),mysql_async::Params::Positional(sql.1)).await .map_err(|e|format!("{:?}",e))? diff --git a/src/main.rs b/src/main.rs index 3a6c592..f7cf053 100644 --- a/src/main.rs +++ b/src/main.rs @@ -269,6 +269,25 @@ async fn main() -> Result<(), Box> { // Shared state let app_state = Arc::new(AppState::new_from_config(&petscan_config).await) ; + let mut args = std::env::args(); + if args.len()==2 { // ASSUMING PSID, eg 26799233 + let _ = args.next(); + let psid: String = args.next().unwrap(); + let parameter_pairs = std::collections::HashMap::new(); + let mut form_parameters = FormParameters::new_from_pairs ( parameter_pairs ) ; + match app_state.get_query_from_psid(&psid).await { + Ok(psid_query) => { + let psid_params = FormParameters::outcome_from_query(&psid_query).unwrap(); + form_parameters.rebase(&psid_params); + } + Err(e) => panic!("{}", e), + } + let mut platform = Platform::new_from_parameters(&form_parameters, app_state.clone()); + let _ = platform.run().await; + println!("{:?}",platform.result().as_ref().unwrap().entries().read()); + std::process::exit(0); + } + // Run on IP/port let port = petscan_config["http_port"].as_u64().unwrap_or(80) as u16; let ip_address = petscan_config["http_server"].as_str().unwrap_or("0.0.0.0").to_string(); @@ -304,48 +323,3 @@ async fn main() -> Result<(), Box> { }); } } - - -/* - -#[tokio::main] -async fn main() -> Result<(),Error> { - tracing_subscriber::fmt::init(); - - let basedir = env::current_dir() - .expect("Can't get CWD") - .to_str() - .expect("Can't convert CWD to_str") - .to_string(); - let path = basedir.to_owned() + "/config.json"; - let file = File::open(&path).unwrap_or_else(|_| panic!("Can not open config file at {}", &path)); - let petscan_config: Value = - serde_json::from_reader(file).expect("Can not parse JSON from config file"); - - let ip_address = petscan_config["http_server"].as_str().unwrap_or("0.0.0.0").to_string(); - let port = petscan_config["http_port"].as_u64().unwrap_or(80) as u16; - let app_state = Arc::new(AppState::new_from_config(&petscan_config).await) ; - - let ip_address : Vec = ip_address.split('.').map(|s|s.parse::().unwrap()).collect(); - let ip_address = std::net::Ipv4Addr::new(ip_address[0],ip_address[1],ip_address[2],ip_address[3],); - let addr = SocketAddr::from((ip_address, port)); - - let make_service = make_service_fn(move |_| { - let app_state = app_state.clone(); - - async { - Ok::<_, Error>(service_fn(move |req| { - process_request(req,app_state.to_owned()) - })) - } - }); - - let server = Server::bind(&addr).serve(make_service); - - println!("Listening on http://{}", addr); - - server.await?; - - Ok(()) -} - */ \ No newline at end of file