Compare commits

...

5 Commits

Author SHA1 Message Date
Amaury
4530a1f7e8 fix clippy 2023-10-24 18:30:53 +02:00
Amaury
ce2923b5a8 Update comments 2023-10-24 18:20:59 +02:00
Amaury
f34b01265e Fix compiler 2023-10-24 18:17:32 +02:00
Amaury
f05fa47994 Merge branch 'master' of ssh://github.com/reacherhq/check-if-email-exists into am/update 2023-10-24 17:11:42 +02:00
Amaury
f4a496692c Move file 2023-10-24 17:11:35 +02:00
2 changed files with 254 additions and 257 deletions

View File

@ -14,53 +14,18 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! This file implements the /bulk/{id}/results endpoints.
use std::convert::{TryFrom, TryInto};
use check_if_email_exists::LOG_TARGET;
use csv::WriterBuilder;
use serde::{Deserialize, Serialize};
use sqlx::{Executor, Pool, Postgres, Row};
use warp::Filter;
use super::{
db::with_db,
error::{BulkError, CsvError},
};
/// Defines the download format, passed in as a query param.
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
enum JobResultResponseFormat {
Json,
Csv,
}
// limit and offset are optional in the request
// if they are unspecified their default values
// are 50 and 0 respectively
#[derive(Serialize, Deserialize)]
struct JobResultRequest {
format: Option<JobResultResponseFormat>,
limit: Option<u64>,
offset: Option<u64>,
}
#[derive(Serialize, Deserialize)]
struct JobResultJsonResponse {
results: Vec<serde_json::Value>,
}
use serde::Serialize;
use std::convert::TryFrom;
/// Wrapper for serde json value to convert
/// into a csv response
#[derive(Debug)]
struct CsvWrapper(serde_json::Value);
pub struct CsvWrapper(pub serde_json::Value);
/// Simplified output of `CheckEmailOutput` struct
/// for csv fields.
#[derive(Debug, Serialize)]
struct JobResultCsvResponse {
pub struct JobResultCsvResponse {
input: String,
is_reachable: String,
#[serde(rename = "misc.is_disposable")]
@ -238,221 +203,3 @@ impl TryFrom<CsvWrapper> for JobResultCsvResponse {
})
}
}
async fn job_result(
job_id: i32,
conn_pool: Pool<Postgres>,
req: JobResultRequest,
) -> Result<impl warp::Reply, warp::Rejection> {
// Throw an error if the job is still running.
// Is there a way to combine these 2 requests in one?
let total_records = sqlx::query!(
r#"SELECT total_records FROM bulk_jobs WHERE id = $1;"#,
job_id
)
.fetch_one(&conn_pool)
.await
.map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to fetch total_records for [job={}] with [error={}]",
job_id,
e
);
BulkError::from(e)
})?
.total_records;
let total_processed = sqlx::query!(
r#"SELECT COUNT(*) FROM email_results WHERE job_id = $1;"#,
job_id
)
.fetch_one(&conn_pool)
.await
.map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to get total_processed for [job={}] with [error={}]",
job_id,
e
);
BulkError::from(e)
})?
.count
.unwrap_or(0);
if total_processed < total_records as i64 {
return Err(BulkError::JobInProgress.into());
}
let format = req.format.unwrap_or(JobResultResponseFormat::Json);
match format {
JobResultResponseFormat::Json => {
let data = job_result_json(
job_id,
req.limit.unwrap_or(50),
req.offset.unwrap_or(0),
conn_pool,
)
.await?;
let reply =
serde_json::to_vec(&JobResultJsonResponse { results: data }).map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to convert json results to string for [job={}] with [error={}]",
job_id,
e
);
BulkError::Json(e)
})?;
Ok(warp::reply::with_header(
reply,
"Content-Type",
"application/json",
))
}
JobResultResponseFormat::Csv => {
let data = job_result_csv(
job_id,
req.limit.unwrap_or(5000),
req.offset.unwrap_or(0),
conn_pool,
)
.await?;
Ok(warp::reply::with_header(data, "Content-Type", "text/csv"))
}
}
}
async fn job_result_json(
job_id: i32,
limit: u64,
offset: u64,
conn_pool: Pool<Postgres>,
) -> Result<Vec<serde_json::Value>, warp::Rejection> {
let query = sqlx::query!(
r#"
SELECT result FROM email_results
WHERE job_id = $1
ORDER BY id
LIMIT $2 OFFSET $3
"#,
job_id,
limit as i64,
offset as i64
);
let rows: Vec<serde_json::Value> = conn_pool
.fetch_all(query)
.await
.map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to get results for [job={}] [limit={}] [offset={}] with [error={}]",
job_id,
limit,
offset,
e
);
BulkError::from(e)
})?
.iter()
.map(|row| row.get("result"))
.collect();
Ok(rows)
}
async fn job_result_csv(
job_id: i32,
limit: u64,
offset: u64,
conn_pool: Pool<Postgres>,
) -> Result<Vec<u8>, warp::Rejection> {
let query = sqlx::query!(
r#"
SELECT result FROM email_results
WHERE job_id = $1
ORDER BY id
LIMIT $2 OFFSET $3
"#,
job_id,
limit as i64,
offset as i64
);
let mut wtr = WriterBuilder::new().has_headers(true).from_writer(vec![]);
for json_value in conn_pool
.fetch_all(query)
.await
.map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to get results for [job={}] with [error={}]",
job_id,
e
);
BulkError::from(e)
})?
.iter()
.map(|row| row.get("result"))
{
let result_csv: JobResultCsvResponse = CsvWrapper(json_value).try_into().map_err(|e: &'static str| {
log::error!(
target: LOG_TARGET,
"Failed to convert json to csv output struct for [job={}] [limit={}] [offset={}] to csv with [error={}]",
job_id,
limit,
offset,
e
);
BulkError::Csv(CsvError::Parse(e))
})?;
wtr.serialize(result_csv).map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to serialize result for [job={}] [limit={}] [offset={}] to csv with [error={}]",
job_id,
limit,
offset,
e
);
BulkError::Csv(CsvError::CsvLib(e))
})?;
}
let data = wtr.into_inner().map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to convert results for [job={}] [limit={}] [offset={}] to csv with [error={}]",
job_id,
limit,
offset,
e
);
BulkError::Csv(CsvError::CsvLibWriter(Box::new(e)))
})?;
Ok(data)
}
pub fn get_bulk_job_result(
o: Option<Pool<Postgres>>,
) -> impl Filter<Extract = (impl warp::Reply,), Error = warp::Rejection> + Clone {
warp::path!("v0" / "bulk" / i32 / "results")
.and(warp::get())
.and(with_db(o))
.and(warp::query::<JobResultRequest>())
.and_then(job_result)
// View access logs by setting `RUST_LOG=reacher`.
.with(warp::log(LOG_TARGET))
}

View File

@ -0,0 +1,250 @@
// Reacher - Email Verification
// Copyright (C) 2018-2022 Reacher
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! This file implements the /bulk/{id}/results endpoints.
use check_if_email_exists::LOG_TARGET;
use csv::WriterBuilder;
use serde::{Deserialize, Serialize};
use sqlx::{Executor, Pool, Postgres, Row};
use std::convert::TryInto;
use std::iter::Iterator;
use warp::Filter;
use super::{
db::with_db,
error::{BulkError, CsvError},
};
use csv_helper::{CsvWrapper, JobResultCsvResponse};
mod csv_helper;
/// Defines the download format, passed in as a query param.
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
enum JobResultResponseFormat {
Json,
Csv,
}
// limit and offset are optional in the request
// If unspecified, offset will default to 0.
#[derive(Serialize, Deserialize)]
struct JobResultRequest {
format: Option<JobResultResponseFormat>,
limit: Option<u64>,
offset: Option<u64>,
}
#[derive(Serialize, Deserialize)]
struct JobResultJsonResponse {
results: Vec<serde_json::Value>,
}
async fn job_result(
job_id: i32,
conn_pool: Pool<Postgres>,
req: JobResultRequest,
) -> Result<impl warp::Reply, warp::Rejection> {
// Throw an error if the job is still running.
// Is there a way to combine these 2 requests in one?
let total_records = sqlx::query!(
r#"SELECT total_records FROM bulk_jobs WHERE id = $1;"#,
job_id
)
.fetch_one(&conn_pool)
.await
.map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to fetch total_records for [job={}] with [error={}]",
job_id,
e
);
BulkError::from(e)
})?
.total_records;
let total_processed = sqlx::query!(
r#"SELECT COUNT(*) FROM email_results WHERE job_id = $1;"#,
job_id
)
.fetch_one(&conn_pool)
.await
.map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to get total_processed for [job={}] with [error={}]",
job_id,
e
);
BulkError::from(e)
})?
.count
.unwrap_or(0);
if total_processed < total_records as i64 {
return Err(BulkError::JobInProgress.into());
}
let format = req.format.unwrap_or(JobResultResponseFormat::Json);
match format {
JobResultResponseFormat::Json => {
let data =
job_result_json(job_id, req.limit, req.offset.unwrap_or(0), conn_pool).await?;
let reply =
serde_json::to_vec(&JobResultJsonResponse { results: data }).map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to convert json results to string for [job={}] with [error={}]",
job_id,
e
);
BulkError::Json(e)
})?;
Ok(warp::reply::with_header(
reply,
"Content-Type",
"application/json",
))
}
JobResultResponseFormat::Csv => {
let data =
job_result_csv(job_id, req.limit, req.offset.unwrap_or(0), conn_pool).await?;
Ok(warp::reply::with_header(data, "Content-Type", "text/csv"))
}
}
}
async fn job_result_as_iter(
job_id: i32,
limit: Option<u64>,
offset: u64,
conn_pool: Pool<Postgres>,
) -> Result<Box<dyn Iterator<Item = serde_json::Value>>, BulkError> {
let query = sqlx::query!(
r#"
SELECT result FROM email_results
WHERE job_id = $1
ORDER BY id
LIMIT $2 OFFSET $3
"#,
job_id,
limit.map(|l| l as i64),
offset as i64
);
let rows = conn_pool.fetch_all(query).await.map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to get results for [job={}] [limit={}] [offset={}] with [error={}]",
job_id,
limit.map(|s| s.to_string()).unwrap_or_else(|| "n/a".into()),
offset,
e
);
BulkError::from(e)
})?;
Ok(Box::new(
rows.into_iter()
.map(|row| row.get::<serde_json::Value, &str>("result")),
))
}
async fn job_result_json(
job_id: i32,
limit: Option<u64>,
offset: u64,
conn_pool: Pool<Postgres>,
) -> Result<Vec<serde_json::Value>, warp::Rejection> {
// For JSON responses, we don't want ot return more than 50 results at a
// time, to avoid having a too big payload (unless client specifies a limit)
Ok(
job_result_as_iter(job_id, limit.or(Some(50)), offset, conn_pool)
.await?
.collect(),
)
}
async fn job_result_csv(
job_id: i32,
limit: Option<u64>,
offset: u64,
conn_pool: Pool<Postgres>,
) -> Result<Vec<u8>, warp::Rejection> {
let rows = job_result_as_iter(job_id, limit, offset, conn_pool).await?;
let mut wtr = WriterBuilder::new().has_headers(true).from_writer(vec![]);
for json_value in rows {
let result_csv: JobResultCsvResponse = CsvWrapper(json_value).try_into().map_err(|e: &'static str| {
log::error!(
target: LOG_TARGET,
"Failed to convert json to csv output struct for [job={}] [limit={}] [offset={}] to csv with [error={}]",
job_id,
limit.map(|s| s.to_string()).unwrap_or_else(|| "n/a".into()),
offset,
e
);
BulkError::Csv(CsvError::Parse(e))
})?;
wtr.serialize(result_csv).map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to serialize result for [job={}] [limit={}] [offset={}] to csv with [error={}]",
job_id,
limit.map(|s| s.to_string()).unwrap_or_else(|| "n/a".into()),
offset,
e
);
BulkError::Csv(CsvError::CsvLib(e))
})?;
}
let data = wtr.into_inner().map_err(|e| {
log::error!(
target: LOG_TARGET,
"Failed to convert results for [job={}] [limit={}] [offset={}] to csv with [error={}]",
job_id,
limit.map(|s| s.to_string()).unwrap_or_else(|| "n/a".into()),
offset,
e
);
BulkError::Csv(CsvError::CsvLibWriter(Box::new(e)))
})?;
Ok(data)
}
pub fn get_bulk_job_result(
o: Option<Pool<Postgres>>,
) -> impl Filter<Extract = (impl warp::Reply,), Error = warp::Rejection> + Clone {
warp::path!("v0" / "bulk" / i32 / "results")
.and(warp::get())
.and(with_db(o))
.and(warp::query::<JobResultRequest>())
.and_then(job_result)
// View access logs by setting `RUST_LOG=reacher`.
.with(warp::log(LOG_TARGET))
}