Compare commits

...

3 Commits

Author SHA1 Message Date
Amaury
50c8b6241f Ignore tests 2023-10-25 14:56:29 +02:00
Amaury
f5cbf6491a Fix docker 2023-10-25 14:53:00 +02:00
Amaury
20355716d2 feat: Yahoo account recovery via headless 2023-10-25 14:49:25 +02:00
16 changed files with 431 additions and 198 deletions

View File

@ -7,7 +7,7 @@
RUST_LOG=reacher=info
SQLX_OFFLINE=1
RCH_HOTMAIL_USE_HEADLESS=http://localhost:4444
RCH_WEBDRIVER_ADDR=http://localhost:9515
# To enable bulk email verification, set the value below to 1, and fill out all
# other env variables below.

3
.vscode/settings.json vendored Normal file
View File

@ -0,0 +1,3 @@
{
"rust-analyzer.cargo.features": ["headless"]
}

View File

@ -42,7 +42,7 @@ USER chrome
ENV RUST_LOG=reacher=info
ENV RCH_HTTP_HOST=0.0.0.0
ENV PORT=8080
ENV RCH_HOTMAIL_USE_HEADLESS=http://localhost:9515
ENV RCH_WEBDRIVER_ADDR=http://localhost:9515
# Bulk verification is disabled by default. Set to 1 to enable it.
ENV RCH_ENABLE_BULK=0

View File

@ -48,21 +48,21 @@ Then send a `POST http://localhost:8080/v0/check_email` request with the followi
These are the environment variables used to configure the HTTP server. To pass them to the Docker container, use the `-e {ENV_VAR}={VALUE}` flag.
| Env Var | Required? | Description | Default |
| ----------------------------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------- |
| `RUST_LOG` | No | One of `trace,debug,warn,error,info`. 💡 PRO TIP: `RUST_LOG=debug` is very handful for debugging purposes. | not defined |
| `RCH_ENABLE_BULK` | No | If set to `1`, then bulk verification endpoints will be added to the backend. | 0 |
| `DATABASE_URL` | Yes if `RCH_ENABLE_BULK==1` | [Bulk] Database connection string for storing results and task queue | not defined |
| `RCH_DATABASE_MAX_CONNECTIONS` | No | [Bulk] Connections created for the database pool | 5 |
| `RCH_MINIMUM_TASK_CONCURRENCY` | No | [Bulk] Minimum number of concurrent running tasks below which more tasks are fetched | 10 |
| `RCH_MAXIMUM_CONCURRENT_TASK_FETCH` | No | [Bulk] Maximum number of tasks fetched at once | 20 |
| `RCH_HTTP_HOST` | No | The host name to bind the HTTP server to. | `127.0.0.1` |
| `PORT` | No | The port to bind the HTTP server to, often populated by the cloud provider. | `8080` |
| `RCH_SENTRY_DSN` | No | If set, bug reports will be sent to this [Sentry](https://sentry.io) DSN. | not defined |
| `RCH_HEADER_SECRET` | No | If set, then all HTTP requests must have the `x-reacher-secret` header set to this value. This is used to protect the backend against public unwanted HTTP requests. | undefined |
| `RCH_FROM_EMAIL` | No | Email to use in the `<MAIL FROM:>` SMTP step. Can be overwritten by each API request's `from_email` field. | reacher.email@gmail.com |
| `RCH_HELLO_NAME` | No | Name to use in the `<EHLO>` SMTP step. Can be overwritten by each API request's `hello_name` field. | gmail.com |
| `RCH_HOTMAIL_USE_HEADLESS` | No | Set to a running WebDriver process endpoint (e.g. `http://localhost:4444`) to use a headless navigator to Hotmail's password recovery page to check Hotmail/Outlook addresses. We recommend `chromedriver` as it allows parallel requests. | not defined |
| Env Var | Required? | Description | Default |
| ----------------------------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- |
| `RUST_LOG` | No | One of `trace,debug,warn,error,info`. 💡 PRO TIP: `RUST_LOG=debug` is very handful for debugging purposes. | not defined |
| `RCH_ENABLE_BULK` | No | If set to `1`, then bulk verification endpoints will be added to the backend. | 0 |
| `DATABASE_URL` | Yes if `RCH_ENABLE_BULK==1` | [Bulk] Database connection string for storing results and task queue | not defined |
| `RCH_DATABASE_MAX_CONNECTIONS` | No | [Bulk] Connections created for the database pool | 5 |
| `RCH_MINIMUM_TASK_CONCURRENCY` | No | [Bulk] Minimum number of concurrent running tasks below which more tasks are fetched | 10 |
| `RCH_MAXIMUM_CONCURRENT_TASK_FETCH` | No | [Bulk] Maximum number of tasks fetched at once | 20 |
| `RCH_HTTP_HOST` | No | The host name to bind the HTTP server to. | `127.0.0.1` |
| `PORT` | No | The port to bind the HTTP server to, often populated by the cloud provider. | `8080` |
| `RCH_SENTRY_DSN` | No | If set, bug reports will be sent to this [Sentry](https://sentry.io) DSN. | not defined |
| `RCH_HEADER_SECRET` | No | If set, then all HTTP requests must have the `x-reacher-secret` header set to this value. This is used to protect the backend against public unwanted HTTP requests. | undefined |
| `RCH_FROM_EMAIL` | No | Email to use in the `<MAIL FROM:>` SMTP step. Can be overwritten by each API request's `from_email` field. | reacher.email@gmail.com |
| `RCH_HELLO_NAME` | No | Name to use in the `<EHLO>` SMTP step. Can be overwritten by each API request's `hello_name` field. | gmail.com |
| `RCH_WEBDRIVER_ADDR` | No | Set to a running WebDriver process endpoint (e.g. `http://localhost:9515`) to use a headless navigator to password recovery pages to check Yahoo and Hotmail/Outlook addresses. We recommend `chromedriver` as it allows parallel requests. | not defined |
## REST API Documentation

View File

@ -26,7 +26,6 @@ use super::sentry_util;
/// Same as `check-if-email-exists`'s check email, but adds some additional
/// inputs and error handling.
pub async fn check_email(input: CheckEmailInput) -> CheckEmailOutput {
let hotmail_use_headless = env::var("RCH_HOTMAIL_USE_HEADLESS").ok();
let from_email =
env::var("RCH_FROM_EMAIL").unwrap_or_else(|_| CheckEmailInput::default().from_email);
let hello_name =
@ -35,7 +34,6 @@ pub async fn check_email(input: CheckEmailInput) -> CheckEmailOutput {
let input = CheckEmailInput {
// If we want to override core check-if-email-exists's default values
// for CheckEmailInput for the backend, we do it here.
hotmail_use_headless,
from_email,
hello_name,
..input

View File

@ -58,15 +58,22 @@ pub struct Cli {
#[clap(long, env, default_value = "true", parse(try_from_str))]
pub yahoo_use_api: bool,
/// For Yahoo addresses, use a headless browser to connect to the
/// Yahoo account recovery page. Requires a webdriver instance
/// listening on RCH_WEBDRIVER_ADDR.
#[clap(long, env)]
pub yahoo_use_headless: bool,
/// For Gmail email addresses, use Gmail's API instead of connecting
/// directly to their SMTP servers.
#[clap(long, env, default_value = "false", parse(try_from_str))]
pub gmail_use_api: bool,
/// For Hotmail addresses, use a headless browser to connect to the
/// Microsoft account recovery page.
/// Microsoft account recovery page. Requires a webdriver instance
/// listening on RCH_WEBDRIVER_ADDR.
#[clap(long, env)]
pub hotmail_use_headless: Option<String>,
pub hotmail_use_headless: bool,
/// For Microsoft 365 email addresses, use OneDrive's API instead of
/// connecting directly to their SMTP servers.
@ -100,10 +107,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
.set_hello_name(CONF.hello_name.clone())
.set_smtp_port(CONF.smtp_port)
.set_yahoo_use_api(CONF.yahoo_use_api)
.set_yahoo_use_headless(CONF.yahoo_use_headless)
.set_gmail_use_api(CONF.gmail_use_api)
.set_microsoft365_use_api(CONF.microsoft365_use_api)
.set_check_gravatar(CONF.check_gravatar)
.set_hotmail_use_headless(CONF.hotmail_use_headless.clone())
.set_hotmail_use_headless(CONF.hotmail_use_headless)
.set_haveibeenpwned_api_key(CONF.haveibeenpwned_api_key.clone());
if let Some(proxy_host) = &CONF.proxy_host {

View File

@ -374,7 +374,7 @@ pub async fn check_smtp_with_retry(
// be non-callable, as this function only deals with actual SMTP
// connection errors.
#[cfg(feature = "headless")]
Err(SmtpError::HotmailError(_)) => result,
Err(SmtpError::HeadlessError(_)) => result,
Err(SmtpError::YahooError(_)) => result,
Err(SmtpError::GmailError(_)) => result,
// Only retry if the SMTP error was unknown.

View File

@ -16,7 +16,7 @@
use super::gmail::GmailError;
#[cfg(feature = "headless")]
use super::outlook::hotmail::HotmailError;
use super::headless::HeadlessError;
use super::outlook::microsoft365::Microsoft365Error;
use super::parser;
use super::yahoo::YahooError;
@ -41,9 +41,11 @@ pub enum SmtpError {
GmailError(GmailError),
/// Error when verifying a Hotmail email via headless browser.
#[cfg(feature = "headless")]
HotmailError(HotmailError),
HeadlessError(HeadlessError),
/// Error when verifying a Microsoft 365 email via HTTP request.
Microsoft365Error(Microsoft365Error),
/// Headless Navigator not running.
NoHeadlessNavigator,
/// Email is in the `skipped_domains` parameter.
SkippedDomain(String),
}
@ -67,9 +69,9 @@ impl From<GmailError> for SmtpError {
}
#[cfg(feature = "headless")]
impl From<HotmailError> for SmtpError {
fn from(e: HotmailError) -> Self {
SmtpError::HotmailError(e)
impl From<HeadlessError> for SmtpError {
fn from(e: HeadlessError) -> Self {
SmtpError::HeadlessError(e)
}
}

69
core/src/smtp/headless.rs Normal file
View File

@ -0,0 +1,69 @@
// check-if-email-exists
// Copyright (C) 2018-2023 Reacher
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use fantoccini::{
error::{CmdError, NewSessionError},
Client, ClientBuilder,
};
use serde::Serialize;
use serde_json::Map;
use crate::util::ser_with_display::ser_with_display;
#[derive(Debug, Serialize)]
pub enum HeadlessError {
#[serde(serialize_with = "ser_with_display")]
Cmd(CmdError),
#[serde(serialize_with = "ser_with_display")]
NewSession(NewSessionError),
}
impl From<CmdError> for HeadlessError {
fn from(e: CmdError) -> Self {
Self::Cmd(e)
}
}
impl From<NewSessionError> for HeadlessError {
fn from(e: NewSessionError) -> Self {
Self::NewSession(e)
}
}
pub async fn create_headless_client(webdriver: &str) -> Result<Client, HeadlessError> {
// Running in a Docker container, I run into the following error:
// Failed to move to new namespace: PID namespaces supported, Network namespace supported, but failed: errno = Operation not permitted
// In searching around I found a few different workarounds:
// - Enable namespaces: https://github.com/jessfraz/dockerfiles/issues/65#issuecomment-266532289
// - Run it with a custom seccomp: https://github.com/jessfraz/dockerfiles/issues/65#issuecomment-217214671
// - Run with --no-sandbox: https://github.com/karma-runner/karma-chrome-launcher/issues/125#issuecomment-312668593
// For now I went with the --no-sandbox.
//
// TODO Look into security implications...
let mut caps = Map::new();
let opts = serde_json::json!({
"args": ["--headless", "--disable-gpu", "--no-sandbox", "--disable-dev-shm-usage"],
});
caps.insert("goog:chromeOptions".to_string(), opts);
// Connect to WebDriver instance that is listening on `webdriver`
let c = ClientBuilder::native()
.capabilities(caps)
.connect(webdriver)
.await?;
Ok(c)
}

View File

@ -17,12 +17,15 @@
mod connect;
mod error;
mod gmail;
#[cfg(feature = "headless")]
mod headless;
mod http_api;
mod outlook;
mod parser;
mod yahoo;
use std::default::Default;
use std::env;
use async_smtp::EmailAddress;
use serde::{Deserialize, Serialize};
@ -62,7 +65,7 @@ pub async fn check_smtp(
domain: &str,
input: &CheckEmailInput,
) -> Result<SmtpDetails, SmtpError> {
let host = host.to_string();
let host: String = host.to_string();
if input.skipped_domains.iter().any(|d| host.contains(d)) {
return Err(SmtpError::SkippedDomain(format!(
@ -70,17 +73,45 @@ pub async fn check_smtp(
)));
}
if input.yahoo_use_api && is_yahoo(&host) {
return yahoo::check_yahoo(to_email, input)
.await
.map_err(|err| err.into());
// Headless checks. Please note that they take precedence over API checks.
#[cfg(feature = "headless")]
{
let webdriver_addr = env::var("RCH_WEBDRIVER_ADDR");
if is_outlook(&host) {
match &webdriver_addr {
Ok(a) => {
return outlook::headless::check_password_recovery(
to_email.to_string().as_str(),
a,
)
.await
.map_err(|err| err.into());
}
_ => return Err(SmtpError::NoHeadlessNavigator),
}
} else if is_yahoo(&host) {
match &webdriver_addr {
Ok(a) => {
return yahoo::check_headless(to_email.to_string().as_str(), a)
.await
.map_err(|err| err.into());
}
_ => return Err(SmtpError::NoHeadlessNavigator),
}
}
}
// API checks
if input.gmail_use_api && is_gmail(&host) {
return gmail::check_gmail(to_email, input)
.await
.map_err(|err| err.into());
}
if input.microsoft365_use_api && is_microsoft365(&host) {
} else if input.yahoo_use_api && is_yahoo(&host) {
return yahoo::check_api(to_email, input)
.await
.map_err(|err| err.into());
} else if input.microsoft365_use_api && is_microsoft365(&host) {
match outlook::microsoft365::check_microsoft365_api(to_email, input).await {
Ok(Some(smtp_details)) => return Ok(smtp_details),
// Continue in the event of an error/ambiguous result.
@ -95,14 +126,6 @@ pub async fn check_smtp(
_ => {}
}
}
#[cfg(feature = "headless")]
if let Some(webdriver) = &input.hotmail_use_headless {
if is_outlook(&host) {
return outlook::hotmail::check_password_recovery(to_email, webdriver)
.await
.map_err(|err| err.into());
}
}
check_smtp_with_retry(to_email, &host, port, domain, input, input.retries).await
}

View File

@ -16,46 +16,26 @@
use std::{thread::sleep, time::Duration};
use async_smtp::EmailAddress;
use async_std::prelude::FutureExt;
use fantoccini::{
error::{CmdError, NewSessionError},
ClientBuilder, Locator,
};
use fantoccini::Locator;
use futures::TryFutureExt;
use serde::Serialize;
use serde_json::Map;
use crate::{smtp::SmtpDetails, util::ser_with_display::ser_with_display, LOG_TARGET};
#[derive(Debug, Serialize)]
pub enum HotmailError {
#[serde(serialize_with = "ser_with_display")]
Cmd(CmdError),
#[serde(serialize_with = "ser_with_display")]
NewSession(NewSessionError),
}
impl From<CmdError> for HotmailError {
fn from(e: CmdError) -> Self {
Self::Cmd(e)
}
}
impl From<NewSessionError> for HotmailError {
fn from(e: NewSessionError) -> Self {
Self::NewSession(e)
}
}
use crate::{
smtp::{
headless::{create_headless_client, HeadlessError},
SmtpDetails,
},
LOG_TARGET,
};
/// Check if a Hotmail/Outlook email exists by connecting to the password
/// recovery page https://account.live.com/password/reset using a headless
/// browser. Make sure you have a WebDriver server running locally before
/// running this, or this will error.
pub async fn check_password_recovery(
to_email: &EmailAddress,
to_email: &str,
webdriver: &str,
) -> Result<SmtpDetails, HotmailError> {
) -> Result<SmtpDetails, HeadlessError> {
let to_email = to_email.to_string();
log::debug!(
target: LOG_TARGET,
@ -63,26 +43,7 @@ pub async fn check_password_recovery(
to_email,
);
// Running in a Docker container, I run into the following error:
// Failed to move to new namespace: PID namespaces supported, Network namespace supported, but failed: errno = Operation not permitted
// In searching around I found a few different workarounds:
// - Enable namespaces: https://github.com/jessfraz/dockerfiles/issues/65#issuecomment-266532289
// - Run it with a custom seccomp: https://github.com/jessfraz/dockerfiles/issues/65#issuecomment-217214671
// - Run with --no-sandbox: https://github.com/karma-runner/karma-chrome-launcher/issues/125#issuecomment-312668593
// For now I went with the --no-sandbox.
//
// TODO Look into security implications...
let mut caps = Map::new();
let opts = serde_json::json!({
"args": ["--headless", "--disable-gpu", "--no-sandbox", "--disable-dev-shm-usage"],
});
caps.insert("goog:chromeOptions".to_string(), opts);
// Connect to WebDriver instance that is listening on `webdriver`
let c = ClientBuilder::native()
.capabilities(caps)
.connect(webdriver)
.await?;
let c = create_headless_client(webdriver).await?;
// Navigate to Microsoft password recovery page.
c.goto("https://account.live.com/password/reset").await?;
@ -147,37 +108,29 @@ pub async fn check_password_recovery(
#[cfg(test)]
mod tests {
use super::check_password_recovery;
use async_smtp::EmailAddress;
use async_std::prelude::FutureExt;
use std::str::FromStr;
// Ignoring this test as it requires a local process of WebDriver running on
// "http://localhost:4444". To debug the headless password recovery page,
// "http://localhost:9515". To debug the headless password recovery page,
// run chromedriver and remove the "#[ignore]".
// Also see: https://github.com/jonhoo/fantoccini
#[tokio::test]
#[ignore]
#[ignore = "Run a webdriver server locally to test this"]
async fn test_hotmail_address() {
// This email does not exist.
let email = EmailAddress::from_str("test42134@hotmail.com").unwrap();
// Run 10 headless sessions with the above fake email (not deliverable).
// Run 10 headless sessions with dummy emails.
// It should not error.
for _ in 0..10 {
let res = check_password_recovery(&email, "http://localhost:4444")
// This email does not exist.
let res = check_password_recovery(&"test42134@hotmail.com", "http://localhost:9515")
.await
.unwrap();
assert!(!res.is_deliverable)
}
assert!(!res.is_deliverable);
// This email does exist.
let email = EmailAddress::from_str("test@hotmail.com").unwrap();
// Run 10 headless sessions with the above fake email (not deliverable).
// It should not error.
for _ in 0..10 {
let res = check_password_recovery(&email, "http://localhost:4444")
// This email does exist.
let res = check_password_recovery("test@hotmail.com", "http://localhost:9515")
.await
.unwrap();
assert!(res.is_deliverable)
assert!(res.is_deliverable);
}
}
@ -186,13 +139,11 @@ mod tests {
// but will fail with geckodriver.
// ref: https://github.com/jonhoo/fantoccini/issues/111#issuecomment-727650629
#[tokio::test]
#[ignore]
#[ignore = "Run a **chromedriver** server locally to test this"]
async fn test_parallel() {
// This email does not exist.
let email = EmailAddress::from_str("foo@bar.baz").unwrap();
let f1 = check_password_recovery(&email, "http://localhost:4444");
let f2 = check_password_recovery(&email, "http://localhost:4444");
let f1 = check_password_recovery("foo@bar.baz", "http://localhost:9515");
let f2 = check_password_recovery("foo@bar.baz", "http://localhost:9515");
let f = f1.try_join(f2).await;
assert!(f.is_ok(), "{:?}", f);

View File

@ -1,5 +1,5 @@
#[cfg(feature = "headless")]
pub mod hotmail;
pub mod headless;
pub mod microsoft365;
/// Check if a MX host is from outlook (includes @hotmail.*, @outlook.* and

View File

@ -14,19 +14,14 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use super::SmtpDetails;
use super::YahooError;
use crate::{
smtp::http_api::create_client,
util::{
constants::LOG_TARGET, input_output::CheckEmailInput, ser_with_display::ser_with_display,
},
smtp::{http_api::create_client, SmtpDetails},
util::{constants::LOG_TARGET, input_output::CheckEmailInput},
};
use async_smtp::EmailAddress;
use regex::Regex;
use reqwest::Error as ReqwestError;
use serde::{Deserialize, Serialize};
use serde_json::error::Error as SerdeError;
use std::fmt;
const SIGNUP_PAGE: &str = "https://login.yahoo.com/account/create?specId=yidReg&lang=en-US&src=&done=https%3A%2F%2Fwww.yahoo.com&display=login";
const SIGNUP_API: &str = "https://login.yahoo.com/account/module/create?validateField=yid";
@ -79,56 +74,20 @@ struct FormResponse {
errors: Vec<FormResponseItem>,
}
/// Possible errors when checking Yahoo email addresses.
#[derive(Debug, Serialize)]
pub enum YahooError {
/// Cannot find "acrumb" field in cookie.
NoAcrumb,
/// Cannot find "sessionIndex" hidden input in body
NoSessionIndex,
/// Cannot find cookie in Yahoo response.
NoCookie,
/// Error when serializing or deserializing HTTP requests and responses.
#[serde(serialize_with = "ser_with_display")]
ReqwestError(ReqwestError),
/// Error when serializing or deserializing HTTP requests and responses.
#[serde(serialize_with = "ser_with_display")]
SerdeError(SerdeError),
}
impl fmt::Display for YahooError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
// Customize so only `x` and `y` are denoted.
write!(f, "{self:?}")
}
}
impl From<ReqwestError> for YahooError {
fn from(error: ReqwestError) -> Self {
YahooError::ReqwestError(error)
}
}
impl From<SerdeError> for YahooError {
fn from(error: SerdeError) -> Self {
YahooError::SerdeError(error)
}
}
/// Use well-crafted HTTP requests to verify if a Yahoo email address exists.
/// Inspired by https://github.com/hbattat/verifyEmail.
pub async fn check_yahoo(
pub async fn check_api(
to_email: &EmailAddress,
input: &CheckEmailInput,
) -> Result<SmtpDetails, YahooError> {
let response = create_client(input, "yahoo")?
let res = create_client(input, "yahoo")?
.get(SIGNUP_PAGE)
.header("User-Agent", USER_AGENT)
.send()
.await?;
// Get the cookies from the response.
let cookies = match response.headers().get("Set-Cookie") {
let cookies = match res.headers().get("Set-Cookie") {
Some(x) => x.to_owned(),
_ => {
return Err(YahooError::NoCookie);
@ -138,18 +97,11 @@ pub async fn check_yahoo(
let to_email = to_email.to_string();
log::debug!(
target: LOG_TARGET,
"[email={}] Yahoo 1st response: {:?}",
"[email={}] Yahoo succesfully got cookies after response",
to_email,
response
);
log::debug!(
target: LOG_TARGET,
"[email={}] Yahoo cookies: {:?}",
to_email,
cookies
);
let body = response.text().await?;
let body = res.text().await?;
let username = to_email
.split('@')
@ -182,7 +134,7 @@ pub async fn check_yahoo(
};
// Mimic a real HTTP request.
let response = create_client(input, "yahoo")?
let res = create_client(input, "yahoo")?
.post(SIGNUP_API)
.header("Origin", "https://login.yahoo.com")
.header("X-Requested-With", "XMLHttpRequest")
@ -210,10 +162,10 @@ pub async fn check_yahoo(
target: LOG_TARGET,
"[email={}] Yahoo 2nd response: {:?}",
to_email,
response
res
);
let username_exists = response.errors.iter().any(|item| {
let username_exists = res.errors.iter().any(|item| {
item.name == "userId"
&& (item.error == "IDENTIFIER_NOT_AVAILABLE" || item.error == "IDENTIFIER_EXISTS")
});
@ -224,11 +176,3 @@ pub async fn check_yahoo(
..Default::default()
})
}
/// Check if the MX host is from Yahoo.
/// Examples:
/// - mta7.am0.yahoodns.net.
/// - mx-eu.mail.am0.yahoodns.net.
pub fn is_yahoo(host: &str) -> bool {
host.to_lowercase().ends_with(".yahoodns.net.")
}

View File

@ -0,0 +1,141 @@
// check-if-email-exists
// Copyright (C) 2018-2023 Reacher
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::{thread::sleep, time::Duration};
use async_std::prelude::FutureExt;
use fantoccini::Locator;
use futures::TryFutureExt;
use crate::smtp::headless::{create_headless_client, HeadlessError};
use crate::{smtp::SmtpDetails, LOG_TARGET};
/// Check if a Hotmail/Outlook email exists by connecting to the password
/// recovery page https://account.live.com/password/reset using a headless
/// browser. Make sure you have a WebDriver server running locally before
/// running this, or this will error.
pub async fn check_headless(to_email: &str, webdriver: &str) -> Result<SmtpDetails, HeadlessError> {
log::debug!(
target: LOG_TARGET,
"[email={}] Using Yahoo password recovery in headless navigator",
to_email,
);
let c = create_headless_client(webdriver).await?;
// Navigate to Microsoft password recovery page.
c.goto("https://login.yahoo.com/forgot").await?;
// Wait for network/javascript/dom to make the input-box available
// and click it.
let input = c.wait().for_element(Locator::Id("username")).await?;
// Sometimes I get "input ... is not reachable by keyboard". Addind this
// small sleep helps.
sleep(Duration::from_millis(200));
input.send_keys(to_email).await?;
// Click on "Continue"
c.find(Locator::Css("button[name=\"verifyYid\"]"))
.await?
.click()
.await?;
// Depending on what answers we have on the account recovery page, return
// the relevant (is_deliverable, is_disabled) tuple.
// "Sorry, we don't recognise that email address or phone number".
let f1 = c
.wait()
.for_element(Locator::Css(".error-msg"))
.and_then(|_| async { Ok((false, false)) });
// "This account has been deactivated due to inactivity, but we would love to welcome you back!"
let f2 = c
.wait()
.for_element(Locator::Css(".ctx-account_is_locked"))
.and_then(|_| async { Ok((false, true)) });
// Recaptcha
let f3 = c
.wait()
.for_element(Locator::Css(".recaptcha-challenge"))
.and_then(|_| async { Ok((true, false)) });
// "Enter verification code sent to your email address"
let f4 = c
.wait()
.for_element(Locator::Id("email-verify-challenge"))
.and_then(|_| async { Ok((true, false)) });
let (is_deliverable, is_disabled) = f1.try_race(f2).try_race(f3).try_race(f4).await?;
if is_deliverable {
log::debug!(
target: LOG_TARGET,
"[email={}] Did not find error message in password recovery, email exists",
to_email,
);
} else {
log::debug!(
target: LOG_TARGET,
"[email={}] Found error message in password recovery, email does not exist",
to_email,
);
}
c.close().await?;
Ok(SmtpDetails {
can_connect_smtp: true,
has_full_inbox: false,
is_catch_all: false,
is_deliverable,
is_disabled,
})
}
#[cfg(test)]
mod tests {
use super::check_headless;
// Ignoring this test as it requires a local process of WebDriver running on
// "http://localhost:9515". To debug the headless password recovery page,
// run chromedriver and remove the "#[ignore]".
// Also see: https://github.com/jonhoo/fantoccini
#[tokio::test]
#[ignore = "Run a webdriver server locally to test this"]
async fn test_yahoo_address() {
// Run 5 headless sessions with the below dummy emails.
for _ in 0..5 {
// Email does not exist.
let res = check_headless("test42134@yahoo.com", "http://localhost:9515")
.await
.unwrap();
assert!(!res.is_deliverable);
// Disabled email.
let res = check_headless("amaury@yahoo.com", "http://localhost:9515")
.await
.unwrap();
assert!(!res.is_deliverable);
assert!(res.is_disabled);
// OK email.
let res = check_headless("test2@yahoo.com", "http://localhost:9515")
.await
.unwrap();
assert!(res.is_deliverable);
assert!(!res.is_disabled);
}
}
}

View File

@ -0,0 +1,73 @@
// check-if-email-exists
// Copyright (C) 2018-2023 Reacher
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
mod api;
#[cfg(feature = "headless")]
mod headless;
use crate::util::ser_with_display::ser_with_display;
use reqwest::Error as ReqwestError;
use serde::Serialize;
use serde_json::error::Error as SerdeError;
use std::fmt;
pub use api::check_api;
#[cfg(feature = "headless")]
pub use headless::check_headless;
/// Possible errors when checking Yahoo email addresses.
#[derive(Debug, Serialize)]
pub enum YahooError {
/// Cannot find "acrumb" field in cookie.
NoAcrumb,
/// Cannot find "sessionIndex" hidden input in body
NoSessionIndex,
/// Cannot find cookie in Yahoo response.
NoCookie,
/// Error when serializing or deserializing HTTP requests and responses.
#[serde(serialize_with = "ser_with_display")]
ReqwestError(ReqwestError),
/// Error when serializing or deserializing HTTP requests and responses.
#[serde(serialize_with = "ser_with_display")]
SerdeError(SerdeError),
}
impl fmt::Display for YahooError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
// Customize so only `x` and `y` are denoted.
write!(f, "{self:?}")
}
}
impl From<ReqwestError> for YahooError {
fn from(error: ReqwestError) -> Self {
YahooError::ReqwestError(error)
}
}
impl From<SerdeError> for YahooError {
fn from(error: SerdeError) -> Self {
YahooError::SerdeError(error)
}
}
/// Check if the MX host is from Yahoo.
/// Examples:
/// - mta7.am0.yahoodns.net.
/// - mx-eu.mail.am0.yahoodns.net.
pub fn is_yahoo(host: &str) -> bool {
host.to_lowercase().ends_with(".yahoodns.net.")
}

View File

@ -101,8 +101,19 @@ pub struct CheckEmailInput {
/// For Yahoo email addresses, use Yahoo's API instead of connecting
/// directly to their SMTP servers.
///
/// Defaults to true.
/// Defaults to false.
pub yahoo_use_api: bool,
/// For Yahoo email addresses, use Yahoo's account recovery page instead
/// of connecting directly to their SMTP servers.
///
/// This assumes you have a WebDriver compatible process running, then pass
/// its endpoint, usually http://localhost:9515, into the environment
/// variable RCH_WEBDRIVER_ADDR. We recommend running chromedriver (and not
/// geckodriver) as it allows parallel requests.
///
/// Defaults to true.
#[cfg(feature = "headless")]
pub yahoo_use_headless: bool,
/// For Gmail email addresses, use Gmail's API instead of connecting
/// directly to their SMTP servers.
///
@ -122,13 +133,15 @@ pub struct CheckEmailInput {
pub haveibeenpwned_api_key: Option<String>,
/// For Hotmail/Outlook email addresses, use a headless navigator
/// connecting to the password recovery page instead of the SMTP server.
/// This assumes you have a WebDriver compatible process running, then pass
/// its endpoint, usually http://localhost:4444. We recommend running
/// chromedriver (and not geckodriver) as it allows parallel requests.
///
/// Defaults to None.
/// This assumes you have a WebDriver compatible process running, then pass
/// its endpoint, usually http://localhost:9515, into the environment
/// variable RCH_WEBDRIVER_ADDR. We recommend running chromedriver (and not
/// geckodriver) as it allows parallel requests.
///
/// Defaults to true.
#[cfg(feature = "headless")]
pub hotmail_use_headless: Option<String>,
pub hotmail_use_headless: bool,
/// Number of retries of SMTP connections to do.
///
/// Defaults to 2 to avoid greylisting.
@ -165,12 +178,17 @@ impl Default for CheckEmailInput {
from_email: "reacher.email@gmail.com".into(), // Unused, owned by Reacher
hello_name: "gmail.com".into(),
#[cfg(feature = "headless")]
hotmail_use_headless: None,
hotmail_use_headless: true,
proxy: None,
smtp_port: 25,
smtp_security: SmtpSecurity::default(),
smtp_timeout: Some(Duration::from_secs(12)),
#[cfg(not(feature = "headless"))]
yahoo_use_api: true,
#[cfg(feature = "headless")]
yahoo_use_api: false,
#[cfg(feature = "headless")]
yahoo_use_headless: true,
gmail_use_api: false,
microsoft365_use_api: false,
check_gravatar: false,
@ -285,6 +303,14 @@ impl CheckEmailInput {
self
}
/// Set whether or not to use a headless navigator to navigate to Yahoo's
/// password recovery page to check if an email exists.
#[cfg(feature = "headless")]
pub fn set_yahoo_use_headless(&mut self, use_headless: bool) -> &mut CheckEmailInput {
self.yahoo_use_headless = use_headless;
self
}
/// Set whether to use Gmail's API or connecting directly to their SMTP
/// servers. Defaults to false.
pub fn set_gmail_use_api(&mut self, use_api: bool) -> &mut CheckEmailInput {
@ -307,21 +333,16 @@ impl CheckEmailInput {
}
/// Whether to haveibeenpwned' API for the given email
/// check only if the api_key is set
/// check only if the api_key is set.
pub fn set_haveibeenpwned_api_key(&mut self, api_key: Option<String>) -> &mut CheckEmailInput {
self.haveibeenpwned_api_key = api_key;
self
}
/// Set whether or not to use a headless navigator to navigate to Hotmail's
/// password recovery page to check if an email exists. If set to
/// `Some(<endpoint>)`, this endpoint must point to a WebDriver process,
/// usually listening on http://localhost:4444. Defaults to None.
/// password recovery page to check if an email exists.
#[cfg(feature = "headless")]
pub fn set_hotmail_use_headless(
&mut self,
use_headless: Option<String>,
) -> &mut CheckEmailInput {
pub fn set_hotmail_use_headless(&mut self, use_headless: bool) -> &mut CheckEmailInput {
self.hotmail_use_headless = use_headless;
self
}