diff --git a/Cargo.lock b/Cargo.lock index 6e682c5..5c84b7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -544,6 +544,19 @@ dependencies = [ "memchr", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.9.0" @@ -918,6 +931,12 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.32" @@ -1029,6 +1048,26 @@ dependencies = [ "walkdir", ] +[[package]] +name = "governor" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68a7f542ee6b35af73b06abc0dad1c1bae89964e4e253bc4b587b91c9637867b" +dependencies = [ + "cfg-if", + "dashmap", + "futures", + "futures-timer", + "no-std-compat", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand 0.8.5", + "smallvec", + "spinning_top", +] + [[package]] name = "h2" version = "0.3.27" @@ -1067,6 +1106,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.15.5" @@ -1390,7 +1435,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "system-configuration 0.6.1", "tokio", "tower-service", @@ -2012,6 +2057,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + [[package]] name = "normpath" version = "1.5.0" @@ -2465,6 +2522,21 @@ dependencies = [ "yansi", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "querying" version = "0.1.0" @@ -2502,7 +2574,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror", "tokio", "tracing", @@ -2539,7 +2611,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -2641,6 +2713,15 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.10.0", +] + [[package]] name = "rayon" version = "1.11.0" @@ -3306,6 +3387,15 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.7.3" @@ -4325,6 +4415,7 @@ dependencies = [ "dotenvy", "env_logger", "flate2", + "governor", "log", "querying", "reports", @@ -4354,6 +4445,22 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -4363,6 +4470,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows" version = "0.48.0" diff --git a/website/Cargo.toml b/website/Cargo.toml index 6453387..86c62b7 100644 --- a/website/Cargo.toml +++ b/website/Cargo.toml @@ -15,6 +15,7 @@ env_logger = "0.11.10" rocket-cache-response = "0.6.4" log = { workspace = true } dotenvy = { version = "0.15.7" } +governor = { version = "0.6", features = ["dashmap"] } [build-dependencies] reqwest = { version = "0.12", features = ["blocking", "json"] } diff --git a/website/src/api.rs b/website/src/api.rs new file mode 100644 index 0000000..c722504 --- /dev/null +++ b/website/src/api.rs @@ -0,0 +1,133 @@ +use crate::db::{check_whitelist, save_query, WhitelistedEntry}; +use governor::clock::DefaultClock; +use governor::state::keyed::DefaultKeyedStateStore; +use governor::{Quota, RateLimiter}; +use log::warn; +use querying::asn::AsnInfo; +use querying::geoip::IpInfo; +use querying::lists::NetworkRecord; +use querying::target::Target; +use querying::{Check, CheckError, CheckVerdict, Checker}; +use rocket::http::Status; +use rocket::serde::json::Json; +use rocket::tokio::sync::RwLock; +use rocket::State; +use rocket_client_addr::ClientRealAddr; +use serde::Serialize; +use sqlx::postgres::PgPool; +use std::collections::HashMap; +use std::net::IpAddr; +use std::num::NonZeroU32; +use std::sync::Arc; + +pub type ApiRateLimiter = + RateLimiter, DefaultClock>; + +pub fn build_rate_limiter(per_minute: u32) -> ApiRateLimiter { + RateLimiter::keyed(Quota::per_minute( + NonZeroU32::new(per_minute).expect("rate limit must be > 0"), + )) +} + +#[derive(Serialize)] +pub struct ApiCheckResponse { + pub id: Option, + pub target: String, + pub target_type: String, + pub blocked: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub rkn_domain: Option, + pub ips: Vec, + pub blocked_subnets: Vec, + pub cdn_providers: HashMap>, + pub geo: IpInfo, + #[serde(skip_serializing_if = "Option::is_none")] + pub asn_info: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub whitelist: Option, +} + +fn build_response( + id: Option, + target: &Target, + check: Check, + whitelist: Option, +) -> ApiCheckResponse { + let (blocked, rkn_domain, cdn_providers) = match check.verdict { + CheckVerdict::Blocked { + rkn_domain, + cdn_provider_subnets, + } => { + let providers: HashMap> = cdn_provider_subnets + .into_iter() + .map(|(k, v)| (k, v.into_iter().collect())) + .collect(); + (true, rkn_domain, providers) + } + CheckVerdict::Clear => (false, None, HashMap::new()), + }; + + ApiCheckResponse { + id, + target: target.to_query(), + target_type: target.readable_type().to_string(), + blocked, + rkn_domain, + ips: check.ips.iter().map(|ip| ip.to_string()).collect(), + blocked_subnets: check.rkn_subnets.iter().map(|n| n.to_string()).collect(), + cdn_providers, + geo: check.geo, + asn_info: check.asn_info, + whitelist, + } +} + +#[get("/check?")] +pub async fn check( + target: &str, + checker: &State>>, + addr: &ClientRealAddr, + pool: &State, + limiter: &State>, +) -> Result, Status> { + if limiter.check_key(&addr.ip).is_err() { + return Err(Status::TooManyRequests); + } + + let target = Target::from(target.trim()); + let check = checker.read().await.check(target.clone()).await; + + let mut db = pool + .acquire() + .await + .map_err(|_| Status::InternalServerError)?; + + let id: Option = if let Ok(check) = &check { + match save_query(&mut *db, &target, check, addr, checker.read().await).await { + Ok(id) => Some(id.to_string()), + Err(e) => { + warn!("api: failed to save check: {:?}", e); + None + } + } + } else { + None + }; + + let whitelist: Option = if let Target::Domain(domain) = &target { + check_whitelist(domain, &mut *db) + .await + .map_err(|_| Status::InternalServerError)? + } else { + None + }; + + match check { + Err(CheckError::NotFound) => Err(Status::NotFound), + Ok(check) => Ok(Json(build_response(id, &target, check, whitelist))), + Err(e) => { + log::error!("api check failed {:?}", e); + Err(Status::InternalServerError) + } + } +} diff --git a/website/src/main.rs b/website/src/main.rs index f8c2b73..64e8e53 100644 --- a/website/src/main.rs +++ b/website/src/main.rs @@ -1,6 +1,7 @@ #[macro_use] extern crate rocket; mod agency; +mod api; mod db; mod whitelist; @@ -298,6 +299,12 @@ async fn rocket() -> _ { } }); + let rate_limit_rpm: u32 = std::env::var("API_RATE_LIMIT_RPM") + .unwrap_or("30".to_string()) + .parse() + .unwrap_or(30); + let api_limiter = std::sync::Arc::new(api::build_rate_limiter(rate_limit_rpm)); + let pool = sqlx::postgres::PgPoolOptions::new() .max_connections(std::env::var("DATABASE_MAX_CONNECTIONS") .unwrap_or("100".to_string()) @@ -316,9 +323,12 @@ async fn rocket() -> _ { rocket::build() .manage(checker) .manage(pool) + .manage(api_limiter) .attach(AdHoc::try_on_ignite("SQLx Migrations", run_migrations)) .mount("/", routes![index, check, healthcheck, page, feedback]) .mount("/vendor", routes![lucide, chartjs, chartjs_datalabels]) + .mount("/api/v1", routes![api::check]) + .register("/api", catchers![api_error]) .mount("/agency", routes![agency::upload_report]) .mount("/whitelist", routes![whitelist::histogram, whitelist::export_csv]) .register("/agency", catchers![api_error])