From 02800b564aad9ca1d096db57ab62816473d22662 Mon Sep 17 00:00:00 2001 From: Steven Fackler Date: Mon, 29 Sep 2014 19:56:17 -0700 Subject: [PATCH 1/6] Use a temp table for encode_many Just using IN will be faster for small numbers of crates, so you might want to switch implementations depending on crates.len(). The best solution would be to use COPY, but I haven't implemented that yet. I haven't actually tested this, but it compiles! --- src/db.rs | 9 ++++++++- src/krate.rs | 43 ++++++++++++++++++++++++++++++------------- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/db.rs b/src/db.rs index 4f6573d1e3c..3f95a3e4c76 100644 --- a/src/db.rs +++ b/src/db.rs @@ -3,7 +3,7 @@ use std::mem; use std::sync::Arc; use pg; -use pg::{PostgresConnection, PostgresStatement, PostgresResult}; +use pg::{PostgresConnection, PostgresTransaction, PostgresStatement, PostgresResult}; use pg::types::ToSql; use r2d2::{mod, LoggingErrorHandler}; use r2d2_postgres::PostgresPoolManager; @@ -154,6 +154,7 @@ impl<'a> RequestTransaction<'a> for &'a Request + 'a { pub trait Connection { fn prepare<'a>(&'a self, query: &str) -> PostgresResult>; fn execute(&self, query: &str, params: &[&ToSql]) -> PostgresResult; + fn transaction<'a>(&'a self) -> PostgresResult>; } impl Connection for pg::PostgresConnection { @@ -163,6 +164,9 @@ impl Connection for pg::PostgresConnection { fn execute(&self, query: &str, params: &[&ToSql]) -> PostgresResult { self.execute(query, params) } + fn transaction<'a>(&'a self) -> PostgresResult> { + self.transaction() + } } // // impl Connection for pg::pool::PooledPostgresConnection { @@ -181,4 +185,7 @@ impl<'a> Connection for pg::PostgresTransaction<'a> { fn execute(&self, query: &str, params: &[&ToSql]) -> PostgresResult { self.execute(query, params) } + fn transaction<'a>(&'a self) -> PostgresResult> { + self.transaction() + } } diff --git a/src/krate.rs b/src/krate.rs index 74c233258d2..18cfe3fbf2c 100644 --- a/src/krate.rs +++ b/src/krate.rs @@ -133,19 +133,36 @@ impl Crate { pub fn encode_many(conn: &Connection, crates: Vec) -> CargoResult> { - // TODO: can rust-postgres do this escaping? - let crateids: Vec = crates.iter().map(|p| p.id).collect(); - let mut map = HashMap::new(); - let query = format!("'{{{:#}}}'::int[]", crateids.as_slice()); - let stmt = try!(conn.prepare(format!("SELECT id, crate_id FROM versions \ - WHERE crate_id = ANY({})", - query).as_slice())); - for row in try!(stmt.query(&[])) { - match map.entry(row.get("crate_id")) { - Occupied(e) => e.into_mut(), - Vacant(e) => e.set(Vec::new()), - }.push(row.get("id")); - } + let trans = try!(conn.transaction()); + + try!(trans.execute("CREATE TEMPORARY TABLE crateids ( + id INT PRIMARY KEY + ) ON COMMIT DROP", [])); + + let mut map = { + let mut query = "INSERT INTO crateids (id) VALUES (".to_string(); + let mut crateids: Vec<&ToSql> = vec![]; + for (i, krate) in crates.iter().enumerate() { + query.push_str(format!("${}", i+1).as_slice()); + crateids.push(&krate.id); + } + query.push_str(")"); + try!(trans.execute(query.as_slice(), crateids.as_slice())); + + let stmt = try!(conn.prepare("SELECT v.id, v.crate_id FROM versions v + INNER JOIN crateids c ON v.id = c.id")); + + let mut map = HashMap::new(); + for row in try!(stmt.query(&[])) { + match map.entry(row.get("crate_id")) { + Occupied(e) => e.into_mut(), + Vacant(e) => e.set(Vec::new()), + }.push(row.get("id")); + } + map + }; + + try!(trans.finish()); Ok(crates.into_iter().map(|p| { let id = p.id; From 26192b70a767d333979141cf60ba97d3efb3208d Mon Sep 17 00:00:00 2001 From: Steven Fackler Date: Mon, 29 Sep 2014 20:03:06 -0700 Subject: [PATCH 2/6] Herp derp --- src/krate.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/krate.rs b/src/krate.rs index 18cfe3fbf2c..1fa5a01d8a6 100644 --- a/src/krate.rs +++ b/src/krate.rs @@ -142,7 +142,12 @@ impl Crate { let mut map = { let mut query = "INSERT INTO crateids (id) VALUES (".to_string(); let mut crateids: Vec<&ToSql> = vec![]; + let mut first = true; for (i, krate) in crates.iter().enumerate() { + if !first { + query.push_str(", "); + } + first = false; query.push_str(format!("${}", i+1).as_slice()); crateids.push(&krate.id); } From a214a9669eb2ce5d8ade8e4b828cf1f8802cca97 Mon Sep 17 00:00:00 2001 From: Steven Fackler Date: Mon, 29 Sep 2014 20:18:06 -0700 Subject: [PATCH 3/6] Avoid intermediate string allocation --- src/krate.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/krate.rs b/src/krate.rs index 1fa5a01d8a6..5703058b17e 100644 --- a/src/krate.rs +++ b/src/krate.rs @@ -1,3 +1,4 @@ +use std::io::MemWriter; use std::collections::hashmap::{HashMap, Occupied, Vacant}; use std::sync::Arc; use serialize::json; @@ -140,18 +141,20 @@ impl Crate { ) ON COMMIT DROP", [])); let mut map = { - let mut query = "INSERT INTO crateids (id) VALUES (".to_string(); + let mut query = MemWriter::new(); + let _ = write!(query, "INSERT INTO crateids (id) VALUES ("); let mut crateids: Vec<&ToSql> = vec![]; let mut first = true; for (i, krate) in crates.iter().enumerate() { if !first { - query.push_str(", "); + let _ = write!(query, ", "); } first = false; - query.push_str(format!("${}", i+1).as_slice()); + let _ = write!(query, "${}", i+1); crateids.push(&krate.id); } - query.push_str(")"); + let _ = write!(query, ")"); + let query = String::from_utf8(query.unwrap()).unwrap(); try!(trans.execute(query.as_slice(), crateids.as_slice())); let stmt = try!(conn.prepare("SELECT v.id, v.crate_id FROM versions v From 009d5c2839ad4cb765670d90fa0d154d3b6f4e80 Mon Sep 17 00:00:00 2001 From: Steven Fackler Date: Mon, 29 Sep 2014 20:22:40 -0700 Subject: [PATCH 4/6] Early exit on empty crates list --- src/krate.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/krate.rs b/src/krate.rs index 5703058b17e..f843b32b32b 100644 --- a/src/krate.rs +++ b/src/krate.rs @@ -134,6 +134,10 @@ impl Crate { pub fn encode_many(conn: &Connection, crates: Vec) -> CargoResult> { + if crates.is_empty() { + return Ok(vec![]); + } + let trans = try!(conn.transaction()); try!(trans.execute("CREATE TEMPORARY TABLE crateids ( From 9ca61caada3c1185146455d6a6a3cdecf1368186 Mon Sep 17 00:00:00 2001 From: Steven Fackler Date: Mon, 29 Sep 2014 20:27:57 -0700 Subject: [PATCH 5/6] Use not-broken syntax :( --- src/krate.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/krate.rs b/src/krate.rs index f843b32b32b..216676544ea 100644 --- a/src/krate.rs +++ b/src/krate.rs @@ -146,7 +146,7 @@ impl Crate { let mut map = { let mut query = MemWriter::new(); - let _ = write!(query, "INSERT INTO crateids (id) VALUES ("); + let _ = write!(query, "INSERT INTO crateids (id) VALUES "); let mut crateids: Vec<&ToSql> = vec![]; let mut first = true; for (i, krate) in crates.iter().enumerate() { @@ -154,10 +154,9 @@ impl Crate { let _ = write!(query, ", "); } first = false; - let _ = write!(query, "${}", i+1); + let _ = write!(query, "(${})", i+1); crateids.push(&krate.id); } - let _ = write!(query, ")"); let query = String::from_utf8(query.unwrap()).unwrap(); try!(trans.execute(query.as_slice(), crateids.as_slice())); From 0c547d19fe83636a9f6e1d022efaf04387b3e6f1 Mon Sep 17 00:00:00 2001 From: Steven Fackler Date: Mon, 29 Sep 2014 20:28:45 -0700 Subject: [PATCH 6/6] Don't use conn --- src/krate.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/krate.rs b/src/krate.rs index 216676544ea..4b700c78b50 100644 --- a/src/krate.rs +++ b/src/krate.rs @@ -160,8 +160,8 @@ impl Crate { let query = String::from_utf8(query.unwrap()).unwrap(); try!(trans.execute(query.as_slice(), crateids.as_slice())); - let stmt = try!(conn.prepare("SELECT v.id, v.crate_id FROM versions v - INNER JOIN crateids c ON v.id = c.id")); + let stmt = try!(trans.prepare("SELECT v.id, v.crate_id FROM versions v + INNER JOIN crateids c ON v.id = c.id")); let mut map = HashMap::new(); for row in try!(stmt.query(&[])) {