Skip to content

Commit

Permalink
Fixes #103
Browse files Browse the repository at this point in the history
  • Loading branch information
@magnusmanske
magnusmanske committed Jan 22, 2024
1 parent 938c65e commit 3d3281c
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 67 deletions.
41 changes: 19 additions & 22 deletions src/datasource_database.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -832,27 +832,27 @@ impl SourceDatabase {
let nslist = primary_pagelist.group_by_namespace()?;
let mut batches: Vec<SQLtuple> = vec![];
nslist.iter().for_each(|nsgroup| {
nsgroup.1.chunks(PAGE_BATCH_SIZE*2).for_each(|titles| {
let mut sql = Platform::sql_tuple();
sql.0 = "SELECT DISTINCT p.page_id,p.page_title,p.page_namespace,(SELECT rev_timestamp FROM revision WHERE rev_id=p.page_latest LIMIT 1) AS page_touched,p.page_len ".to_string() ;
sql.0 += &params.link_count_sql;
sql.0 += " FROM page p";
if !params.is_before_after_done {
Platform::append_sql(&mut sql, params.sql_before_after.clone());
}
sql.0 += " WHERE (p.page_namespace=";
sql.0 += &nsgroup.0.to_string();
sql.0 += " AND p.page_title IN (";
Platform::append_sql(&mut sql, Platform::prep_quote(&titles));
sql.0 += "))";
batches.push(sql);
});
});
nsgroup.1.chunks(PAGE_BATCH_SIZE*2).for_each(|titles| {
let mut sql = Platform::sql_tuple();
sql.0 = "SELECT DISTINCT p.page_id,p.page_title,p.page_namespace,(SELECT rev_timestamp FROM revision WHERE rev_id=p.page_latest LIMIT 1) AS page_touched,p.page_len ".to_string() ;
sql.0 += &params.link_count_sql;
sql.0 += " FROM page p";
if !params.is_before_after_done {
Platform::append_sql(&mut sql, params.sql_before_after.clone());
}
sql.0 += " WHERE (p.page_namespace=";
sql.0 += &nsgroup.0.to_string();
sql.0 += " AND p.page_title IN (";
Platform::append_sql(&mut sql, Platform::prep_quote(&titles));
sql.0 += "))";
batches.push(sql);
});
});

// Either way, it's done
params.is_before_after_done = true;

let wiki = primary_pagelist.wiki()?.ok_or_else(|| "No wiki 12345".to_string())?;
let wiki = primary_pagelist.wiki()?.ok_or_else(|| "No wiki given in datasource_database::get_pages_pagelist".to_string())?;

let mut futures : Vec<_> = vec![] ;
for sql in batches {
Expand Down Expand Up @@ -898,8 +898,7 @@ impl SourceDatabase {
state: &AppState,
primary_pagelist: Option<&PageList>,
) -> Result<PageList, String> {
let mut params =
self.get_pages_initialize_query(state, primary_pagelist).await?;
let mut params = self.get_pages_initialize_query(state, primary_pagelist).await?;

let mut sql = Platform::sql_tuple();

Expand Down Expand Up @@ -995,7 +994,7 @@ impl SourceDatabase {
Platform::profile("DSDB::get_pages_for_primary STARTING", Some(sql.1.len()));

// Namespaces
if !self.params.namespace_ids.is_empty() {
if !self.params.namespace_ids.is_empty() && primary!="pagelist" {
let namespace_ids = &self
.params
.namespace_ids
Expand Down Expand Up @@ -1227,8 +1226,6 @@ impl SourceDatabase {
Some(sql.1.len()),
);

//println!("{:?}",&sql);

let sql_1_len = sql.1.len() ;
let rows = conn.exec_iter(sql.0.as_str(),mysql_async::Params::Positional(sql.1)).await
.map_err(|e|format!("{:?}",e))?
Expand Down
64 changes: 19 additions & 45 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,25 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Shared state
let app_state = Arc::new(AppState::new_from_config(&petscan_config).await) ;

let mut args = std::env::args();
if args.len()==2 { // ASSUMING PSID, eg 26799233
let _ = args.next();
let psid: String = args.next().unwrap();
let parameter_pairs = std::collections::HashMap::new();
let mut form_parameters = FormParameters::new_from_pairs ( parameter_pairs ) ;
match app_state.get_query_from_psid(&psid).await {
Ok(psid_query) => {
let psid_params = FormParameters::outcome_from_query(&psid_query).unwrap();
form_parameters.rebase(&psid_params);
}
Err(e) => panic!("{}", e),
}
let mut platform = Platform::new_from_parameters(&form_parameters, app_state.clone());
let _ = platform.run().await;
println!("{:?}",platform.result().as_ref().unwrap().entries().read());
std::process::exit(0);
}

// Run on IP/port
let port = petscan_config["http_port"].as_u64().unwrap_or(80) as u16;
let ip_address = petscan_config["http_server"].as_str().unwrap_or("0.0.0.0").to_string();
Expand Down Expand Up @@ -304,48 +323,3 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
});
}
}


/*
#[tokio::main]
async fn main() -> Result<(),Error> {
tracing_subscriber::fmt::init();
let basedir = env::current_dir()
.expect("Can't get CWD")
.to_str()
.expect("Can't convert CWD to_str")
.to_string();
let path = basedir.to_owned() + "/config.json";
let file = File::open(&path).unwrap_or_else(|_| panic!("Can not open config file at {}", &path));
let petscan_config: Value =
serde_json::from_reader(file).expect("Can not parse JSON from config file");
let ip_address = petscan_config["http_server"].as_str().unwrap_or("0.0.0.0").to_string();
let port = petscan_config["http_port"].as_u64().unwrap_or(80) as u16;
let app_state = Arc::new(AppState::new_from_config(&petscan_config).await) ;
let ip_address : Vec<u8> = ip_address.split('.').map(|s|s.parse::<u8>().unwrap()).collect();
let ip_address = std::net::Ipv4Addr::new(ip_address[0],ip_address[1],ip_address[2],ip_address[3],);
let addr = SocketAddr::from((ip_address, port));
let make_service = make_service_fn(move |_| {
let app_state = app_state.clone();
async {
Ok::<_, Error>(service_fn(move |req| {
process_request(req,app_state.to_owned())
}))
}
});
let server = Server::bind(&addr).serve(make_service);
println!("Listening on http://{}", addr);
server.await?;
Ok(())
}
*/

0 comments on commit 3d3281c

Please sign in to comment.