feat: add reporter, database

This commit is contained in:
Lowder
2025-12-10 04:50:14 +05:00
parent 2926da13a2
commit 3523dd30c8
33 changed files with 2625 additions and 166 deletions
+1
View File
@@ -1 +1,2 @@
.git/
target/
+1
View File
@@ -1,2 +1,3 @@
/target
.idea/
.env
Generated
+926 -45
View File
File diff suppressed because it is too large Load Diff
+5 -1
View File
@@ -1,5 +1,8 @@
[workspace]
members = [ "querying",
members = [
"querying",
"reporter",
"reports",
"website",
]
resolver = "3"
@@ -8,3 +11,4 @@ resolver = "3"
tokio = { version = "1.48.0", features = ["full"] }
reqwest = { version = "0.12.24", features = ["stream", "rustls-tls-webpki-roots"] }
log = "0.4.28"
serde = { version = "1", features = ["derive"] }
+6 -4
View File
@@ -33,14 +33,16 @@
Для проверки используются списки [123jjck/cdn-ip-ranges](https://github.com/123jjck/cdn-ip-ranges), [antifilter.download](https://antifilter.download/) и [antifilter.network](https://antifilter.network).
Мы собираем собственные белые списки с помощью [Cheburcheck Reporter](reporter/README.md).
---
## Структура проекта
* `src/` — исходный код приложения
* `templates/` — Tera-шаблоны для фронтенда
* `static/` — статические файлы
* `Cargo.toml` — зависимости и конфигурация Rust-проекта
* `querying` — модуль проверки сайтов по базам данных
* `reporter` — [Cheburcheck Reporter](reporter/README.md)
* `reports` — общий протокол для отправки отчетов
* `website` — исходный код веб-сайта
---
+3
View File
@@ -1,2 +1,5 @@
[default]
template_dir = "website/templates/"
[global.limits]
msgpack = 5242880
+30 -13
View File
@@ -1,11 +1,11 @@
use std::{io};
use crate::updater::{fetch_db, Updatable};
use async_trait::async_trait;
use maxminddb::geoip2::{city, country, City, Country};
use maxminddb::{geoip2, MaxMindDbError};
use serde::Serialize;
use std::io::Error;
use std::net::IpAddr;
use async_trait::async_trait;
use maxminddb::{geoip2, MaxMindDbError};
use maxminddb::geoip2::{city, country, City, Country};
use serde::Serialize;
use crate::updater::{fetch_db, Updatable};
use std::io;
pub struct GeoIp {
asn: Option<maxminddb::Reader<Vec<u8>>>,
@@ -15,16 +15,20 @@ pub struct GeoIp {
#[derive(Serialize, Debug)]
pub struct IpInfo {
asn: String,
organisation: String,
pub asn: Option<String>,
pub country_code: Option<String>,
pub organisation: Option<String>,
pub city_geo_name_id: Option<u32>,
location: String,
}
impl Default for IpInfo {
fn default() -> IpInfo {
IpInfo {
asn: "-".to_string(),
organisation: "-".to_string(),
asn: None,
country_code: None,
organisation: None,
city_geo_name_id: None,
location: "-".to_string(),
}
}
@@ -57,6 +61,18 @@ impl GeoIp {
db.lookup::<Country>(ip)?
} else { None };
let country_code = country.as_ref().map(|c| c.country.as_ref()
.map(|c| c.iso_code
.map(|c| c.to_string()))
.flatten())
.flatten();
let city_geo_name_id = city.as_ref()
.map(|c| c.city.as_ref()
.map(|c| c.geoname_id)
.flatten())
.flatten();
let location = match (city, country) {
(Some(City { city: Some(city::City { names: Some(city), .. }),
country: Some(country::Country { names: Some(country), .. }), .. }), _) => {
@@ -72,11 +88,12 @@ impl GeoIp {
Ok(IpInfo {
location,
country_code,
city_geo_name_id,
asn: asn.clone().and_then(|asn| asn.autonomous_system_number)
.map(|asn| format!("AS{}", asn))
.unwrap_or("-".to_string()),
.map(|asn| format!("AS{}", asn)),
organisation: asn.and_then(|asn| asn.autonomous_system_organization)
.unwrap_or("-").to_string(),
.map(|org| org.to_string()),
})
}
}
+13 -8
View File
@@ -1,16 +1,17 @@
use std::collections::{HashMap, HashSet};
use std::net::IpAddr;
use std::sync::Arc;
use chrono::{DateTime, Utc};
use ipnet::IpNet;
use log::error;
use thiserror::Error;
use tokio::sync::{watch, RwLock};
use crate::geoip::{GeoIp, IpInfo};
use crate::lists::{CdnList, NetworkRecord, RuBlacklist};
use crate::resolver::{ResolveError, Resolver};
use crate::target::Target;
use crate::updater::Updatable;
use chrono::{DateTime, Utc};
use ipnet::IpNet;
use log::error;
use std::collections::{HashMap, HashSet};
use std::net::IpAddr;
use std::sync::Arc;
use maxminddb::MaxMindDbError;
use thiserror::Error;
use tokio::sync::{watch, RwLock};
pub mod geoip;
pub mod lists;
@@ -66,6 +67,10 @@ impl Checker {
}
}
pub async fn geo_ip(&self, ip: IpAddr) -> Result<IpInfo, MaxMindDbError> {
self.geo_ip.read().await.lookup(ip)
}
pub async fn check(&self, target: Target) -> Result<Check, CheckError> {
let ips = match target.resolve(&self.resolver).await {
Ok(ips) => ips,
+6 -6
View File
@@ -1,15 +1,15 @@
use std::collections::VecDeque;
use std::io;
use std::io::{BufRead, Error, Read};
use std::net::IpAddr;
use std::str::FromStr;
use crate::updater::{fetch_db, Updatable};
use async_trait::async_trait;
use ipnet::IpNet;
use ipnet_trie::IpnetTrie;
use log::info;
use serde::{de, Deserialize, Deserializer, Serializer};
use std::collections::VecDeque;
use std::io;
use std::io::{BufRead, Error, Read};
use std::net::IpAddr;
use std::str::FromStr;
use trie_rs::map::{Trie, TrieBuilder};
use crate::updater::{fetch_db, Updatable};
pub struct CdnList {
trie: IpnetTrie<NetworkRecord>,
+2 -2
View File
@@ -1,7 +1,7 @@
use hickory_resolver::config::{LookupIpStrategy, ResolverConfig, ResolverOpts};
use hickory_resolver::name_server::TokioConnectionProvider;
use std::io::{Error, ErrorKind};
use std::net::IpAddr;
use hickory_resolver::config::{LookupIpStrategy, ResolverConfig, ResolverOpts};
use hickory_resolver::name_server::{TokioConnectionProvider};
use thiserror::Error;
pub struct Resolver {
+1 -1
View File
@@ -1,6 +1,6 @@
use crate::resolver::{ResolveError, Resolver};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use url::Url;
use crate::resolver::{ResolveError, Resolver};
#[derive(Debug, Clone)]
pub enum Target {
+5 -5
View File
@@ -1,11 +1,11 @@
use async_trait::async_trait;
use futures_util::StreamExt;
use indicatif::{ProgressBar, ProgressStyle};
use log::info;
use reqwest::IntoUrl;
use std::fmt::Display;
use std::io;
use std::io::Error;
use async_trait::async_trait;
use indicatif::{ProgressBar, ProgressStyle};
use futures_util::StreamExt;
use log::info;
use reqwest::IntoUrl;
pub async fn fetch_db<T: IntoUrl + Display>(url: T) -> Result<Vec<u8>, Error> {
info!("Fetching {}", url);
+25
View File
@@ -0,0 +1,25 @@
[package]
name = "reporter"
version = "0.1.0"
edition = "2024"
[dependencies]
tokio = { workspace = true }
reqwest = { workspace = true }
log = { workspace = true }
reports = { path = "../reports" }
anyhow = "1.0"
env_logger = "0.11.8"
futures = "0.3"
clap = { version = "4.5.53", features = ["derive", "env"] }
csv = "1.4.0"
indicatif = "0.18.3"
rmp-serde = "1.3.0"
serde = { version = "1.0.228", features = ["derive"] }
[target.'cfg(unix)'.dependencies]
libc = "0.2.177"
[build-dependencies]
reqwest = { workspace = true }
getrandom = "0.3.4"
+43
View File
@@ -0,0 +1,43 @@
# Cheburcheck Reporter
Инструмент для автоматического сканирования и анализа блокировок.
Позволяет сканировать большое количество доменов за короткий срок за счет параллелизации запросов.
## Использование
В программе заранее установлены настройки по-умолчанию для сканирования топ-100,000 доменов из [Tranco list от 26 ноября 2025](https://tranco-list.eu/list/2NPQ9).
Для этого достаточно запустить его:
1. Сохранить результаты в CSV
```shell
reporter output.csv
```
2. Отправить результаты в **Cheburcheck Agency**
```shell
reporter -k <API-ключ>
```
> Для того чтобы получить API-ключ,
> обратитесь по адресу [agency@cheburcheck.ru](mailto:agency@cheburcheck.ru).
>
> В письме укажите ваш интернет-провайдер и регион.
## Конфигурация
| Параметр | Описание | Значение по-умолчанию |
|-------------------------------------|-----------------------------------------------------------------------------------------|--------------------------------------|
| `-f, --fake <FAKE>` | Использовать один домен для всех запросов | |
| `-c, --count <COUNT>` | Количество доменов, которые будут проверены (берутся с начала списка) | 100,000 |
| `-t, --timeout-secs <TIMEOUT_SECS>` | Максимальное время ожидания ответа от сервера | 5 секунд |
| `-p, --probes <PROBE_COUNT>` | Максимальное количество одновременных запросов | 1000 |
| `-v, --verbosity <VERBOSITY>` | Отображение результатов сканирования в консоли [silent, error, block, all] | silent |
| `-r, --retry-count <RETRY_COUNT>` | Количество попыток запросов на один домен | 2 |
| `-H, --http` | Использовать plain-HTTP (без TLS) | |
| `-x, --tx` | Отправлять мусорные данные 64кб на сервер | |
| `-i, --ip <IP>` | IP-адрес сервера, на который будут идти запросы (должен отвечать >64kb на любые домены) | 5.78.7.195 |
| `-P, --path <PATH>` | Путь к файлу на сервере | 100MB.bin |
| `-a, --endpoint <AGENCY_ENDPOINT>` | Адрес сервера, на который будут загружены результаты сканирования | https://cheburcheck.ru/agency/report |
| `-k, --key <KEY>` | API-ключ | |
+25
View File
@@ -0,0 +1,25 @@
use std::{env, fs, path::Path};
use reqwest::blocking::Client;
fn main() {
let out_dir = env::var("OUT_DIR").expect("OUT_DIR not set");
let junk_path = Path::new(&out_dir).join("junk.bin");
let mut data = vec![0u8; 64 * 1024];
getrandom::fill(&mut data).expect("Failed to generate random bytes");
fs::write(&junk_path, &data).expect("Failed to write random data");
let client = Client::new();
println!("cargo:rerun-if-env-changed=DIST_DOMAIN_COUNT");
let data = client.get(
format!("https://tranco-list.eu/download/2NPQ9/{}", option_env!("DIST_DOMAIN_COUNT")
.map(|x| x.parse::<u32>().expect("DIST_DOMAIN_COUNT is not a number"))
.unwrap_or(1_000_000)))
.send().expect("Failed to send request")
.bytes().expect("Failed to read bytes");
let list_csv = Path::new(&out_dir).join("list.csv");
fs::write(&list_csv, &data).expect("Failed to write list");
println!("cargo:rerun-if-changed=build.rs");
}
+66
View File
@@ -0,0 +1,66 @@
use std::collections::HashMap;
use std::fmt::Display;
use std::path::PathBuf;
use log::info;
use reports::Evidence;
use crate::Verbosity;
#[derive(Default)]
pub struct Counter {
ok: usize,
block: usize,
err: usize,
pub early: usize,
pub results: HashMap<String, Evidence>,
}
impl Counter {
pub fn save_results(&self, output: &PathBuf) -> anyhow::Result<()> {
let mut out = csv::WriterBuilder::new().from_path(output)?;
out.write_record(&["target", "evidence"])?;
for (target, evidence) in &self.results {
out.write_record(&[target, &evidence.to_string()])?;
}
info!("Saved results to {:?}", output);
Ok(())
}
pub fn print_results(&self, verbosity: &Verbosity) {
if verbosity > &Verbosity::Silent {
info!("Results:");
for (target, evidence) in &self.results {
match evidence {
Evidence::Ok if verbosity >= &Verbosity::All => println!(" [Ok] {}", target),
Evidence::Blocked if verbosity >= &Verbosity::Block => println!(" [Blocked] {}", target),
Evidence::ConnectError if verbosity >= &Verbosity::Error => println!(" [ConnectError] {}", target),
_ => {}
}
}
}
}
pub fn total(&self) -> usize {
self.ok + self.block + self.err
}
pub fn add(&mut self, target: &str, evidence: Evidence) {
match evidence {
Evidence::Ok => self.ok += 1,
Evidence::Blocked => self.block += 1,
Evidence::ConnectError | Evidence::Error => self.err += 1,
}
self.results.insert(target.to_string(), evidence);
}
}
impl Display for Counter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let total = self.total();
write!(f, "OK {} ({:.2}%) | Blocked {} (early: {}) ({:.2}%) | Error {} ({:.2}%)",
self.ok,
self.ok as f32 / total as f32 * 100.0,
self.block, self.early,
self.block as f32 / total as f32 * 100.0,
self.err,
self.err as f32 / total as f32 * 100.0)
}
}
+307
View File
@@ -0,0 +1,307 @@
mod resolver;
mod counter;
use crate::resolver::Resolver;
use anyhow::Result;
use clap::{Parser, ValueEnum};
use futures::stream::FuturesUnordered;
use futures::StreamExt;
use indicatif::{ProgressIterator, ProgressStyle};
use log::{error, info, warn, LevelFilter};
use reports::{AgencyReport, Evidence, ReporterConfig};
use reqwest::redirect::Policy;
use reqwest::Client;
use serde::Serialize;
use std::collections::HashMap;
use std::fmt::Display;
use std::net::IpAddr;
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::time::Instant;
use counter::Counter;
const JUNK: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/junk.bin"));
#[derive(Serialize, Debug, Ord, PartialOrd, Eq, PartialEq, Clone, ValueEnum)]
#[serde(rename_all = "kebab-case")]
enum Verbosity {
Silent,
Error,
Block,
All,
}
#[derive(Parser, Debug, Clone)]
#[command(author, version, about = "DPI probe: checks blockage of domains by SNI")]
struct Args {
/// Output results file
#[arg(required = false)]
output: Option<PathBuf>,
/// Fake target to establish probing settings.
/// Use with caution - it might trigger TLS block
#[arg(short, long, required = false)]
fake: Option<String>,
/// Take first N targets
#[arg(short, long, default_value_t = 100_000)]
count: usize,
/// Read timeout in seconds
#[arg(short, long, default_value_t = 5)]
timeout_secs: u64,
/// Maximum concurrent probes. Make sure that it doesn't exceed 'ulimit -n'
#[arg(short, long = "probes", default_value_t = 1000)]
probe_count: usize,
/// Display probing results in console
#[arg(short, long, default_value_t = Verbosity::Silent, value_enum)]
verbosity: Verbosity,
/// Attempts to establish connection
#[arg(short, long, default_value_t = 2)]
retry_count: usize,
/// Try using plain HTTP without TLS
#[arg(short = 'H', long, default_value_t = false)]
http: bool,
/// Send 64kb junk to server
#[arg(short = 'x', long, default_value_t = false)]
tx: bool,
/// Target IP to probe with.
/// It should be included in IP-ranges of interest.
/// The server must respond to any SNI/Host with a response larger than 64kb.
#[arg(short, long, default_value = "5.78.7.195", value_parser = |v: &str| v.parse::<IpAddr>())]
ip: IpAddr,
/// File name on the server to test
#[arg(short = 'P', long, default_value = "100MB.bin")]
path: String,
/// Custom agency endpoint address
#[arg(short, long = "endpoint", default_value_t = option_env!("AGENCY_ENDPOINT")
.unwrap_or("https://cheburcheck.ru/agency/upload")
.to_string())]
agency_endpoint: String,
/// Agency endpoint API key
#[arg(short, long, env = "AGENCY_KEY")]
key: Option<String>,
}
impl Args {
fn to_reporter_config(&self) -> ReporterConfig {
ReporterConfig {
http: self.http,
tx_junk: self.tx,
ip: self.ip.clone(),
path: self.path.clone(),
retry_count: self.retry_count,
timeout_secs: self.timeout_secs,
probe_count: self.probe_count,
}
}
}
fn build_client(args: &Args, attempt: usize) -> reqwest::Result<Client> {
let client = Client::builder()
.danger_accept_invalid_certs(true)
.redirect(Policy::none())
.use_rustls_tls()
.dns_resolver(Arc::new(Resolver::new(args.ip)))
.read_timeout(Duration::from_secs(args.timeout_secs * attempt as u64))
.timeout(Duration::from_secs(15));
Ok(client.build()?)
}
#[tokio::main]
async fn main() -> Result<()> {
let args = Args::parse();
env_logger::builder().filter_level(LevelFilter::Info).init();
#[cfg(target_family = "unix")]
{
let file_limit: Option<usize> = unsafe { libc::getdtablesize() }.try_into().ok();
if matches!(file_limit, Some(file_limit) if file_limit <= args.probe_count + 128) {
warn!("Open file limit is too low ({})! Consider increasing it using `ulimit -n`.", file_limit.unwrap());
}
}
let api_client = Client::new();
info!("Loading targets list...");
let targets = include_str!(concat!(env!("OUT_DIR"), "/list.csv"));
let targets: Vec<String> = targets.lines().take(args.count)
.map(|s| s.split(",").last().unwrap().to_string()).collect();
info!("Probing {} domains with {} concurrent probes...", targets.len(), args.probe_count);
let sem = Arc::new(tokio::sync::Semaphore::new(args.probe_count));
let cancelled = wait_for_ctrlc();
let start = Instant::now();
let mut futs = FuturesUnordered::new();
for target in targets.into_iter().progress()
.with_style(ProgressStyle::default_bar()
.template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {human_pos}/{human_len} ({eta}, {per_sec})")?
.progress_chars("#>-")) {
if cancelled() {
break;
}
let permit = sem.clone().acquire_owned().await?;
let args = args.clone();
let fake_target = args.fake.clone();
futs.push(tokio::spawn(async move {
let res = check_target(&args, fake_target.as_ref().unwrap_or(&target)).await;
drop(permit);
(target, res)
}));
}
info!("Collecting results...");
let mut counter = Counter::default();
while let Some(res) = futs.next().await {
match res {
Ok((target, Ok(Verdict::Accepted))) => {
counter.add(&target, Evidence::Ok);
}
Ok((target, Ok(Verdict::Blocked { early }))) => {
counter.add(&target, Evidence::Blocked);
if early {
counter.early += 1;
}
}
Ok((target, Err(e))) if e.is_connect() => {
counter.add(&target, Evidence::ConnectError);
if args.verbosity >= Verbosity::Error {
println!("{e:?}");
}
}
Ok((target, Err(_))) => {
counter.add(&target, Evidence::Error);
}
Err(join_err) => {
error!("Task join error: {}", join_err);
}
};
}
counter.print_results(&args.verbosity);
if let Some(output) = &args.output {
counter.save_results(output)?;
}
info!("Probed {} domains in {}s! \nSummary: {counter}", counter.total(), start.elapsed().as_secs());
if let Err(e) = upload_results(&args, &api_client, counter.results).await {
warn!("Upload failed: {}", e);
}
Ok(())
}
async fn upload_results(args: &Args, api_client: &Client, results: HashMap<String, Evidence>) -> Result<()> {
info!("Uploading to {}", args.agency_endpoint);
let uploaded = api_client.post(&args.agency_endpoint)
.header("Content-Type", "application/msgpack")
.body(rmp_serde::to_vec(&AgencyReport {
version: env!("CARGO_PKG_VERSION").to_string(),
config: args.to_reporter_config(),
data: results,
})?);
let uploaded = if let Some(key) = &args.key {
uploaded.header("Authorization", format!("Bearer {key}"))
} else { uploaded };
let uploaded = uploaded.send().await?;
// let uploaded = uploaded.error_for_status()?;
if uploaded.status().is_success() {
info!("Uploaded ({})!", uploaded.status().to_string());
} else {
warn!("Upload failed: {}", uploaded.status().to_string());
}
info!("Agency response: {}", uploaded.text().await?);
Ok(())
}
fn wait_for_ctrlc() -> impl Fn() -> bool {
let cancelled = Arc::new(AtomicUsize::new(0));
let cancelled_ctrlc = cancelled.clone();
tokio::spawn(async move {
loop {
let _ = tokio::signal::ctrl_c().await;
match cancelled_ctrlc.fetch_add(1, Ordering::SeqCst) {
0 => warn!("Ctrl-C received. Finishing up and saving..."),
_ => {
warn!("Forcing exit.");
std::process::exit(130);
}
}
}
});
move || {
cancelled.load(Ordering::SeqCst) != 0
}
}
enum Verdict {
Blocked { early: bool },
Accepted,
}
async fn check_target(args: &Args, target: &str) -> Result<Verdict, reqwest::Error> {
let url = format!("http{}://{target}/{}", if args.http {""} else {"s"}, args.path);
let mut attempts = 0;
loop {
attempts += 1;
let client = build_client(&args, 1)?;
let mut resp = client.get(&url)
.header("Range", "bytes=0-65536");
if args.tx {
resp = resp.body(JUNK)
}
let resp = resp.send()
.await;
let resp = match resp {
Ok(resp) => match (resp.status(), resp.bytes().await) {
(status, Ok(b)) => Ok((status, b)),
(_, Err(e)) => Err((e, false)),
},
Err(e) => Err((e, true)),
};
return match resp {
Ok((status, bytes)) => {
if !status.is_success() {
warn!("Domain {target} returned non-OK code: {status}");
}
if bytes.len() < 65535 {
warn!("Domain {target} completed with {} bytes: \n{}", bytes.len(), String::from_utf8_lossy(bytes.as_ref()));
}
Ok(Verdict::Accepted)
}
Err((e, early)) => {
if attempts < args.retry_count {
continue;
}
if e.is_timeout() {
Ok(Verdict::Blocked { early })
} else {
error!("{} -> Error: {:?}", target, e);
Err(e)
}
},
}
}
}
+23
View File
@@ -0,0 +1,23 @@
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
use std::net::{IpAddr, SocketAddr};
pub struct Resolver {
ip: SocketAddr,
}
impl Resolver {
pub fn new(ip: IpAddr) -> Resolver {
Resolver {
ip: SocketAddr::from(SocketAddr::new(ip, 0)),
}
}
}
impl Resolve for Resolver {
fn resolve(&self, _: Name) -> Resolving {
let ip = self.ip.clone();
Box::pin(async move {
Ok(Addrs::from(Box::new(vec![ip].into_iter())))
})
}
}
+7
View File
@@ -0,0 +1,7 @@
[package]
name = "reports"
version = "0.1.0"
edition = "2024"
[dependencies]
serde = { workspace = true }
+42
View File
@@ -0,0 +1,42 @@
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt::Display;
use std::net::IpAddr;
#[derive(Debug, Serialize, Deserialize)]
pub struct AgencyReport {
pub version: String,
pub config: ReporterConfig,
pub data: HashMap<String, Evidence>,
}
#[derive(Debug, Serialize, Deserialize)]
pub enum Evidence {
Ok,
Blocked,
ConnectError,
Error,
}
impl Display for Evidence {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let str = match self {
Evidence::Ok => "ok",
Evidence::Blocked => "blocked",
Evidence::ConnectError => "connect_error",
Evidence::Error => "unknown_error",
};
write!(f, "{}", str)
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ReporterConfig {
pub http: bool,
pub tx_junk: bool,
pub ip: IpAddr,
pub path: String,
pub retry_count: usize,
pub timeout_secs: u64,
pub probe_count: usize,
}
+2
View File
@@ -9,7 +9,9 @@ RUN apt update && apt install -y libssl-dev pkg-config
RUN --mount=type=cache,target=/build/target \
--mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=secret,id=DATABASE_URL \
set -eux; \
export DATABASE_URL=$(cat /run/secrets/DATABASE_URL); \
cargo build --release --package website; \
objcopy --compress-debug-sections target/release/website ./main
+8 -3
View File
@@ -1,16 +1,21 @@
[package]
name = "website"
version = "0.1.4"
version = "0.2.0"
edition = "2024"
[dependencies]
rocket = "0.5.1"
rocket = { version = "0.5.1", features = ["msgpack", "json"] }
rocket_db_pools = { version = "0.2.0", features = ["sqlx_postgres"]}
rocket_dyn_templates = { version = "0.2.0", features = ["tera"] }
serde = { version = "1", features = ["derive"] }
sqlx = { version = "0.7", features = ["runtime-tokio", "postgres", "migrate", "chrono", "uuid"] }
rocket-client-addr = "0.5.4"
serde = { workspace = true }
reports = { path = "../reports" }
querying = { path = "../querying" }
env_logger = "0.11.8"
rocket-cache-response = "0.6.4"
log = { workspace = true }
dotenvy = { version = "0.15.7" }
[build-dependencies]
reqwest = { version = "0.12", features = ["blocking", "json"] }
+36 -18
View File
@@ -1,10 +1,11 @@
use std::{
env,
fs::{File},
fs::File,
io::{self, Cursor},
path::Path,
};
use std::collections::HashMap;
use std::path::PathBuf;
use flate2::read::GzDecoder;
use reqwest::blocking::Client;
use serde::Deserialize;
@@ -21,13 +22,35 @@ struct Dist {
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let out_dir = env::var("OUT_DIR")?;
let lucide_js_path = Path::new(&out_dir).join("lucide.js");
let client = Client::new();
println!("cargo:rerun-if-changed=migrations");
let out_dir = PathBuf::from(env::var("OUT_DIR")?);
println!("cargo:rerun-if-env-changed=LUCIDE_VERSION");
let metadata_url = format!("https://registry.npmjs.org/lucide/{}", option_env!("LUCIDE_VERSION").unwrap_or("latest"));
install_npm("lucide", option_env!("LUCIDE_VERSION"), HashMap::from([
(PathBuf::from("package/dist/umd/lucide.min.js"), out_dir.join("lucide.js")),
]))?;
println!("cargo:rerun-if-env-changed=CHARTJS_VERSION");
install_npm("chart.js", option_env!("CHARTJS_VERSION"), HashMap::from([
(PathBuf::from("package/dist/chart.umd.min.js"), out_dir.join("chart.js")),
]))?;
println!("cargo:rerun-if-env-changed=CHARTDATALABELS_VERSION");
install_npm("chartjs-plugin-datalabels", option_env!("CHARTDATALABELS_VERSION"), HashMap::from([
(PathBuf::from("package/dist/chartjs-plugin-datalabels.min.js"), out_dir.join("chartjs-plugin-datalabels.js")),
]))?;
println!("cargo:rerun-if-changed=build.rs");
Ok(())
}
fn install_npm(package: &str, version: Option<&str>, mut entries: HashMap<PathBuf, PathBuf>) -> Result<(), Box<dyn std::error::Error>> {
let client = Client::new();
let metadata_url = format!(
"https://registry.npmjs.org/{package}/{}",
version.unwrap_or("latest")
);
println!("Fetching npm metadata from {}", metadata_url);
let metadata: NpmMetadata = client.get(metadata_url).send()?.json()?;
@@ -38,24 +61,19 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
let tar = GzDecoder::new(Cursor::new(tarball_bytes));
let mut archive = Archive::new(tar);
let mut lucide_js_found = false;
for entry in archive.entries()? {
let mut entry = entry?;
let path = entry.path()?;
if path == Path::new("package/dist/umd/lucide.min.js") {
println!("Found lucide.min.js, extracting to {:?}", lucide_js_path);
let mut out_file = File::create(&lucide_js_path)?;
let path = PathBuf::from(entry.path()?);
if let Some(dest) = entries.remove(&path) {
println!("Found {:?}, extracting to {:?}", path.file_name(), dest);
let mut out_file = File::create(&dest)?;
io::copy(&mut entry, &mut out_file)?;
lucide_js_found = true;
break;
}
}
if !lucide_js_found {
panic!("Could not find dist/lucide.min.js inside the tarball");
if !entries.is_empty() {
panic!("Not all entries found! {:?}", entries);
}
println!("cargo:rerun-if-changed=build.rs");
Ok(())
}
@@ -0,0 +1,130 @@
-- Queries
CREATE TABLE IF NOT EXISTS queries
(
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
query VARCHAR(255) NOT NULL,
source_ip VARCHAR(39) NOT NULL,
source_country_code VARCHAR(5),
source_city_geo_name_id INT,
target_country_code VARCHAR(5),
target_asn VARCHAR(32),
target_provider VARCHAR(255),
resolved_ips VARCHAR(39)[],
cdn_networks VARCHAR(43)[],
cdn_providers VARCHAR(255)[],
rkn_domain VARCHAR(255),
date TIMESTAMP DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS queries_query_date_idx ON queries (query, date);
-- Human reports
CREATE TABLE IF NOT EXISTS human_reports
(
id UUID PRIMARY KEY REFERENCES queries (id),
source_ip VARCHAR(39) NOT NULL,
date TIMESTAMP DEFAULT NOW(),
works BOOLEAN
);
CREATE INDEX IF NOT EXISTS human_reports_date_idx ON human_reports (date);
CREATE INDEX IF NOT EXISTS human_reports_works_idx ON human_reports (works);
-- Domain rankings list
CREATE TABLE IF NOT EXISTS domains
(
domain VARCHAR(255) PRIMARY KEY,
rank INT
);
CREATE INDEX IF NOT EXISTS domains_domain_rank_idx ON domains (domain, rank);
-- Agency reporters
CREATE TABLE IF NOT EXISTS reporters
(
id SERIAL PRIMARY KEY,
token VARCHAR(255) NOT NULL,
name VARCHAR(255) NOT NULL
);
CREATE INDEX IF NOT EXISTS reporters_token_idx ON reporters (token);
-- Agency reports
CREATE TABLE IF NOT EXISTS reports
(
id SERIAL PRIMARY KEY,
reporter SERIAL,
reporter_ip VARCHAR(39) NOT NULL,
date TIMESTAMP DEFAULT NOW(),
version VARCHAR(32) NOT NULL,
http BOOLEAN,
tx_junk BOOLEAN,
ip VARCHAR(39),
path VARCHAR(255),
retry_count INT,
timeout_secs INT,
probe_count INT,
FOREIGN KEY (reporter) REFERENCES reporters (id)
);
CREATE INDEX IF NOT EXISTS reports_reporter_date_id_idx ON reports (reporter, date DESC, id);
DO
$$
BEGIN
CREATE TYPE evidence AS ENUM ('ok', 'blocked', 'connection_error', 'unknown_error');
EXCEPTION
WHEN duplicate_object THEN null;
END
$$;
CREATE TABLE IF NOT EXISTS report_row
(
id BIGSERIAL PRIMARY KEY,
report_id SERIAL,
domain VARCHAR(255),
evidence evidence,
FOREIGN KEY (report_id) REFERENCES reports (id) ON DELETE CASCADE
);
CREATE UNIQUE INDEX IF NOT EXISTS report_row_id_domain_evidence_idx ON report_row (report_id, domain, evidence);
-- Whitelist view
CREATE MATERIALIZED VIEW IF NOT EXISTS
whitelist AS
WITH ranked_reports AS (SELECT rr.domain,
rr.evidence,
r.date,
ROW_NUMBER() OVER (
PARTITION BY
rr.domain
ORDER BY
r.date DESC
) AS rn
FROM report_row rr
JOIN reports r ON rr.report_id = r.id
WHERE r.reporter = 1)
SELECT rr.domain,
d.rank,
MAX(
CASE
WHEN rr.evidence = 'ok' THEN rr.date
END
) AS last_ok
FROM ranked_reports rr
LEFT JOIN domains d ON d.domain = rr.domain
WHERE rr.rn <= 5
GROUP BY rr.domain,
d.rank
HAVING COUNT(*) FILTER (
WHERE
rr.evidence = 'ok'
) >= COUNT(*) / 2.0
ORDER BY d.rank;
+85
View File
@@ -0,0 +1,85 @@
use crate::Db;
use reports::AgencyReport;
use rocket::http::Status;
use rocket::serde::json::serde_json::json;
use rocket::serde::json::{Json, Value};
use rocket::serde::msgpack::MsgPack;
use rocket_client_addr::ClientRealAddr;
use rocket_db_pools::Connection;
use sqlx::Acquire;
pub struct Agency {
pub id: i32,
pub name: String,
}
#[rocket::post("/report", format = "application/msgpack", data = "<report>")]
pub async fn upload_report(
report: MsgPack<AgencyReport>,
addr: &ClientRealAddr,
agency: Agency,
mut db: Connection<Db>,
) -> Result<Json<Value>, (Status, String)> {
let mut tx = db
.begin()
.await
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
let report = report.into_inner();
let report_id: i32 = sqlx::query_scalar(
"INSERT INTO reports (
reporter,
reporter_ip,
version,
http,
tx_junk,
ip,
path,
retry_count,
timeout_secs,
probe_count
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id",
)
.bind(agency.id)
.bind(addr.ip.to_string())
.bind(report.version)
.bind(report.config.http)
.bind(report.config.tx_junk)
.bind(report.config.ip.to_string())
.bind(report.config.path)
.bind(report.config.retry_count as i32)
.bind(report.config.timeout_secs as i64)
.bind(report.config.probe_count as i32)
.fetch_one(&mut *tx)
.await
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
let mut copy_in = tx
.copy_in_raw("COPY report_row (report_id, evidence, domain) FROM STDIN (FORMAT CSV)")
.await
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
for (domain, evidence) in report.data {
let line = format!("{},{},{}\n", report_id, evidence, domain);
copy_in
.send(line.as_bytes())
.await
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
}
copy_in
.finish()
.await
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
sqlx::query!("REFRESH MATERIALIZED VIEW whitelist")
.execute(&mut *tx)
.await
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
tx.commit()
.await
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
Ok(Json(json!({ "ok": true, "id": report_id })))
}
+179
View File
@@ -0,0 +1,179 @@
use crate::agency::Agency;
use crate::Db;
use querying::target::Target;
use querying::{Check, CheckVerdict, Checker};
use rocket::http::Status;
use rocket::outcome::{try_outcome, IntoOutcome};
use rocket::request::{FromRequest, Outcome};
use rocket::tokio::sync::RwLockReadGuard;
use rocket::Request;
use rocket_client_addr::ClientRealAddr;
use rocket_db_pools::Connection;
use serde::Serialize;
use sqlx::types::chrono::NaiveDateTime;
use sqlx::types::Uuid;
pub async fn save_query(
db: &mut Connection<Db>,
target: &Target,
check: &Check,
addr: &ClientRealAddr,
checker: RwLockReadGuard<'_, Checker>,
) -> Result<Uuid, sqlx::Error> {
let (cdn_networks, cdn_providers, rkn_domain): (Vec<_>, Vec<_>, Option<_>) =
if let CheckVerdict::Blocked {
cdn_provider_subnets,
rkn_domain,
..
} = &check.verdict
{
(
cdn_provider_subnets
.values()
.flatten()
.map(|n| n.cidr.to_string())
.collect(),
cdn_provider_subnets.keys().map(|p| p.to_string()).collect(),
rkn_domain.clone(),
)
} else {
(vec![], vec![], None)
};
let id = sqlx::query_scalar(
"INSERT INTO queries (
query,
source_ip,
source_country_code,
source_city_geo_name_id,
target_country_code,
target_asn,
target_provider,
resolved_ips,
cdn_networks,
cdn_providers,
rkn_domain
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING id",
)
.bind(target.to_query())
.bind(addr.ip.to_string())
.bind(
checker
.geo_ip(addr.ip)
.await
.map(|i| i.country_code)
.ok()
.flatten(),
)
.bind(check.geo.city_geo_name_id.map(|id| id as i32))
.bind(check.geo.country_code.clone())
.bind(check.geo.asn.clone())
.bind(check.geo.organisation.clone())
.bind(
check
.ips
.iter()
.map(|i| i.to_string())
.collect::<Vec<String>>(),
)
.bind(cdn_networks)
.bind(cdn_providers)
.bind(rkn_domain)
.fetch_one(&mut ***db)
.await?;
Ok(id)
}
#[rocket::async_trait]
impl<'r> FromRequest<'r> for Agency {
type Error = Option<rocket_db_pools::Error<sqlx::Error>>;
async fn from_request(request: &'r Request<'_>) -> Outcome<Self, Self::Error> {
let mut db = try_outcome!(Connection::<Db>::from_request(request).await);
let token = request.headers().get_one("Authorization");
let token = try_outcome!(
token
.and_then(|t| t.split_once(" "))
.map(|(_, tok)| tok.to_string())
.or_forward(Status::Unauthorized)
);
let agency = try_outcome!(
sqlx::query!("SELECT id, name FROM reporters WHERE token = $1", token)
.fetch_optional(&mut **db)
.await
.map_err(|e| Some(rocket_db_pools::Error::Get(e)))
.or_forward(Status::InternalServerError)
);
agency
.map(|r| Agency {
id: r.id,
name: r.name,
})
.or_forward(Status::Unauthorized)
}
}
#[derive(Serialize, Debug, sqlx::FromRow)]
pub struct WhitelistedEntry {
domain: Option<String>,
rank: Option<i32>,
last_ok: Option<NaiveDateTime>,
}
pub async fn check_whitelist(
domain: &str,
db: &mut Connection<Db>,
) -> Result<Option<WhitelistedEntry>, sqlx::Error> {
sqlx::query_as!(
WhitelistedEntry,
"SELECT *
FROM whitelist
WHERE $1 = domain
OR $1 LIKE CONCAT('%.', domain)
ORDER BY LENGTH(domain) DESC
LIMIT 1",
domain
)
.fetch_optional(&mut ***db)
.await
.into()
}
#[derive(Debug, Serialize, sqlx::FromRow)]
pub struct WhitelistHistogramBin {
pub bin_id: Option<i32>,
pub bin_min_rank: Option<i32>,
pub bin_max_rank: Option<i32>,
pub count: Option<i64>,
}
pub async fn collect_histogram(
db: &mut Connection<Db>,
bins: i32,
limit: i32,
filter: bool,
) -> Result<Vec<WhitelistHistogramBin>, sqlx::Error> {
sqlx::query_as!(
WhitelistHistogramBin,
"WITH bins AS (
SELECT generate_series(0, $1 - 1) AS bin
)
SELECT
b.bin as bin_id,
b.bin * $2 + 1 AS bin_min_rank,
(b.bin + 1) * $2 AS bin_max_rank,
COUNT(case when not $3 or w.domain not like '%.co.uk' then 1 end) AS count
FROM bins b
LEFT JOIN whitelist w
ON FLOOR(w.rank / $2) = b.bin
GROUP BY b.bin
ORDER BY b.bin;", bins, limit / bins, filter
)
.fetch_all(&mut ***db)
.await
.into()
}
+180 -54
View File
@@ -1,21 +1,34 @@
#[macro_use] extern crate rocket;
#[macro_use]
extern crate rocket;
mod agency;
mod db;
mod whitelist;
use crate::db::{check_whitelist, save_query};
use log::error;
use querying::resolver::Resolver;
use querying::target::Target;
use querying::{Check, CheckError, CheckVerdict, Checker};
use rocket::fairing::AdHoc;
use rocket::fs::FileServer;
use rocket::http::Status;
use rocket::response::content::RawJavaScript;
use rocket::tokio::sync::RwLock;
use rocket::tokio::time;
use rocket::{fairing, tokio, Build, Request, Rocket, State};
use rocket_cache_response::CacheResponse;
use rocket_client_addr::ClientRealAddr;
use rocket_db_pools::{Connection, Database};
use rocket_dyn_templates::{context, Metadata, Template};
use serde::Serialize;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use rocket::fs::FileServer;
use rocket::http::Status;
use rocket::{tokio, Request, State};
use rocket::response::content::RawJavaScript;
use rocket_dyn_templates::{context, Metadata, Template};
use serde::Serialize;
use log::error;
use rocket::tokio::sync::RwLock;
use rocket::tokio::time;
use rocket_cache_response::CacheResponse;
use querying::{Check, CheckError, CheckVerdict, Checker};
use querying::resolver::Resolver;
use querying::target::Target;
use sqlx::types::Uuid;
#[derive(rocket_db_pools::Database)]
#[database("cheburcheck")]
struct Db(sqlx::PgPool);
#[derive(Serialize)]
struct GlobalContext {
@@ -24,19 +37,24 @@ struct GlobalContext {
impl GlobalContext {
fn new() -> Self {
GlobalContext { version: env!("CARGO_PKG_VERSION") }
GlobalContext {
version: env!("CARGO_PKG_VERSION"),
}
}
}
#[get("/")]
async fn index(checker: &State<Arc<RwLock<Checker>>>) -> Template {
let checker_ref = checker.read().await;
Template::render("index", context! {
global: GlobalContext::new(),
domain_count: format_number(checker_ref.total_domains().await),
v4_count: format_number(checker_ref.total_v4s().await),
last_update: checker_ref.last_update(),
})
Template::render(
"index",
context! {
global: GlobalContext::new(),
domain_count: format_number(checker_ref.total_domains().await),
v4_count: format_number(checker_ref.total_v4s().await),
last_update: checker_ref.last_update(),
},
)
}
#[get("/kb/<page>")]
@@ -46,9 +64,12 @@ fn page(metadata: Metadata, page: &str) -> Option<Template> {
return None;
}
Some(Template::render(page, context! {
global: GlobalContext::new(),
}))
Some(Template::render(
page,
context! {
global: GlobalContext::new(),
},
))
}
#[get("/healthcheck")]
@@ -60,32 +81,86 @@ async fn healthcheck(checker: &State<Arc<RwLock<Checker>>>) -> (Status, String)
}
}
#[post("/feedback/<uuid>/<works>")]
async fn feedback(uuid: &str, works: bool, mut db: Connection<Db>, addr: &ClientRealAddr) -> Result<(), Status> {
sqlx::query!(
"INSERT INTO human_reports (id, source_ip, works) VALUES ($1, $2, $3)",
Uuid::try_parse(uuid).map_err(|_| Status::BadRequest)?,
addr.ip.to_string(),
works
).execute(&mut **db).await.map_err(|_| Status::InternalServerError)?;
Ok(())
}
#[get("/check?<target>")]
async fn check(target: &str, checker: &State<Arc<RwLock<Checker>>>) -> Result<Template, Status> {
async fn check(
target: &str,
checker: &State<Arc<RwLock<Checker>>>,
addr: &ClientRealAddr,
mut db: Connection<Db>,
) -> Result<Template, Status> {
let target = Target::from(target);
match checker.read().await.check(target.clone()).await {
Err(CheckError::NotFound) =>
Ok(Template::render("empty", context! {
global: GlobalContext::new(),
target: target.to_query(),
target_type: target.readable_type(),
})),
Ok(Check { verdict: CheckVerdict::Clear, geo, ips }) =>
Ok(Template::render("result", context! {
let check = checker.read().await.check(target.clone()).await;
let id = if let Ok(check) = &check {
match save_query(&mut db, &target, check, addr, checker.read().await).await {
Ok(id) => Some(id.to_string()),
Err(e) => {
warn!("Failed to save check: {:?}", e);
None
}
}
} else {
None
};
let whitelist = if let Target::Domain(domain) = &target {
check_whitelist(domain, &mut db)
.await
.map_err(|_| Status::InternalServerError)?
} else {
None
};
match check {
Err(CheckError::NotFound) => Ok(Template::render(
"empty",
context! {
global: GlobalContext::new(),
target: target.to_query(),
target_type: target.readable_type(),
},
)),
Ok(Check {
verdict: CheckVerdict::Clear,
geo,
ips,
}) => Ok(Template::render(
"result",
context! {
id,
global: GlobalContext::new(),
found: false,
target: target.to_query(),
target_type: target.readable_type(),
whitelist,
ips,
geo,
})),
},
)),
Ok(Check {
verdict: CheckVerdict::Blocked {
rkn_domain,
rkn_subnets,
cdn_provider_subnets
}, geo, ips }) =>
Ok(Template::render("result", context! {
verdict:
CheckVerdict::Blocked {
rkn_domain,
rkn_subnets,
cdn_provider_subnets,
},
geo,
ips,
}) => Ok(Template::render(
"result",
context! {
id,
global: GlobalContext::new(),
found: true,
domain: rkn_domain,
@@ -95,9 +170,11 @@ async fn check(target: &str, checker: &State<Arc<RwLock<Checker>>>) -> Result<Te
.collect::<Vec<_>>(),
target: target.to_query(),
target_type: target.readable_type(),
whitelist,
ips,
geo,
})),
},
)),
Err(e) => {
error!("check failed {:?}", e);
Err(Status::InternalServerError)
@@ -107,14 +184,17 @@ async fn check(target: &str, checker: &State<Arc<RwLock<Checker>>>) -> Result<Te
#[catch(default)]
fn default(status: Status, _req: &Request) -> Template {
Template::render("error", context! {
global: GlobalContext::new(),
status: status.code,
reason: status.reason_lossy(),
})
Template::render(
"error",
context! {
global: GlobalContext::new(),
status: status.code,
reason: status.reason_lossy(),
},
)
}
#[rocket::get("/vendor/lucide.js")]
#[rocket::get("/lucide.js")]
fn lucide() -> CacheResponse<RawJavaScript<&'static [u8]>> {
CacheResponse::Public {
responder: RawJavaScript(include_bytes!(concat!(env!("OUT_DIR"), "/lucide.js"))),
@@ -122,9 +202,26 @@ fn lucide() -> CacheResponse<RawJavaScript<&'static [u8]>> {
must_revalidate: false,
}
}
#[rocket::get("/chart.js")]
fn chartjs() -> CacheResponse<RawJavaScript<&'static [u8]>> {
CacheResponse::Public {
responder: RawJavaScript(include_bytes!(concat!(env!("OUT_DIR"), "/chart.js"))),
max_age: 604800,
must_revalidate: false,
}
}
#[rocket::get("/chartjs-plugin-datalabels.js")]
fn chartjs_datalabels() -> CacheResponse<RawJavaScript<&'static [u8]>> {
CacheResponse::Public {
responder: RawJavaScript(include_bytes!(concat!(env!("OUT_DIR"), "/chartjs-plugin-datalabels.js"))),
max_age: 604800,
must_revalidate: false,
}
}
fn format_number(number: usize) -> String {
number.to_string()
number
.to_string()
.as_bytes()
.rchunks(3)
.rev()
@@ -134,12 +231,31 @@ fn format_number(number: usize) -> String {
.join(" ")
}
async fn run_migrations(rocket: Rocket<Build>) -> fairing::Result {
match Db::fetch(&rocket) {
Some(db) => match sqlx::migrate!("./migrations").run(&**db).await {
Ok(_) => Ok(rocket),
Err(e) => {
error!("Failed to run database migrations: {}", e);
Err(rocket)
}
},
None => Err(rocket),
}
}
#[launch]
async fn rocket() -> _ {
env_logger::builder().filter_level(log::LevelFilter::Info).init();
env_logger::builder()
.filter_level(log::LevelFilter::Info)
.init();
let mut interval = time::interval(Duration::from_secs(std::env::var("DATABASE_INTERVAL_SECONDS")
.unwrap_or("21600".to_string()).parse().unwrap()));
let mut interval = time::interval(Duration::from_secs(
std::env::var("DATABASE_INTERVAL_SECONDS")
.unwrap_or("21600".to_string())
.parse()
.unwrap(),
));
let checker = Arc::new(RwLock::new(Checker::new().await));
@@ -154,10 +270,20 @@ async fn rocket() -> _ {
}
});
rocket::build()
let figment = rocket::Config::figment().merge((
"databases.cheburcheck.url",
dotenvy::var("DATABASE_URL").expect("DATABASE_URL must be set"),
));
rocket::custom(figment)
.manage(Resolver::new().await)
.manage(checker)
.mount("/", routes![index, lucide, check, healthcheck, page])
.attach(Db::init())
.attach(AdHoc::try_on_ignite("SQLx Migrations", run_migrations))
.mount("/", routes![index, check, healthcheck, page, feedback])
.mount("/vendor", routes![lucide, chartjs, chartjs_datalabels])
.mount("/agency", routes![agency::upload_report])
.mount("/whitelist", routes![whitelist::histogram, whitelist::export_csv])
.register("/", catchers![default])
.mount("/", FileServer::from(PathBuf::from("static")))
.attach(Template::fairing())
+65
View File
@@ -0,0 +1,65 @@
use crate::Db;
use rocket::futures::StreamExt;
use rocket::http::{ContentType, Status};
use rocket::request::FromParam;
use rocket_cache_response::CacheResponse;
use rocket_db_pools::Connection;
use std::io;
use rocket::serde::json::Json;
use crate::db::{collect_histogram, WhitelistHistogramBin};
enum ExportType {
Full,
Domains,
}
impl<'r> FromParam<'r> for ExportType {
type Error = &'r str;
fn from_param(param: &'r str) -> Result<Self, Self::Error> {
match param {
"full.csv" => Ok(ExportType::Full),
"domains.csv" => Ok(ExportType::Domains),
_ => Err(param),
}
}
}
#[get("/<export_type>")]
pub async fn export_csv(
export_type: ExportType,
mut db: Connection<Db>,
) -> Result<CacheResponse<(ContentType, Vec<u8>)>, io::Error> {
let query = match export_type {
ExportType::Full => {
"COPY (SELECT domain, rank, last_ok FROM whitelist) TO STDOUT WITH (FORMAT CSV, HEADER, ENCODING 'UTF8')"
}
ExportType::Domains => {
"COPY (SELECT domain FROM whitelist) TO STDOUT WITH (FORMAT CSV, ENCODING 'UTF8')"
}
};
let mut stream = db
.copy_out_raw(query)
.await
.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
let mut data = Vec::default();
while let Some(bytes_result) = stream.next().await {
let bytes = bytes_result.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
data.extend(bytes)
}
Ok(CacheResponse::Public {
responder: (ContentType::CSV, data),
max_age: 86400,
must_revalidate: false,
})
}
#[get("/histogram?<filter>")]
pub async fn histogram(mut db: Connection<Db>, filter: Option<bool>) -> Result<Json<Vec<WhitelistHistogramBin>>, Status> {
Ok(Json(collect_histogram(&mut db, 50, 100_000, filter.is_some()).await
.map_err(|e| Status::InternalServerError)?))
}
+106 -1
View File
@@ -15,6 +15,10 @@
--red-bg-subtle: rgba(69, 10, 10, 0.2);
--red-border: rgba(127, 29, 29, 0.3);
--yellow-color: #f0b100; /* red-500 */
--yellow-bg-subtle: rgba(69, 64, 10, 0.2);
--yellow-border: rgba(127, 114, 29, 0.3);
--green-color: #22c55e; /* green-500 */
--green-bg-subtle: rgba(5, 46, 22, 0.2);
--green-border: rgba(20, 83, 45, 0.3);
@@ -22,6 +26,11 @@
--font-mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
}
::-webkit-scrollbar { width: 8px; }
::-webkit-scrollbar-track { background: #09090b; }
::-webkit-scrollbar-thumb { background: var(--border-color); }
::-webkit-scrollbar-thumb:hover { background: var(--input-border); }
* {
box-sizing: border-box;
margin: 0;
@@ -262,7 +271,19 @@ h1.error-code {
.clean-theme .alert-box {
border: 1px solid var(--green-border);
background: var(--green-bg-subtle);
color: #4ade80;
color: var(--green-color);
}
.whitelist-theme {
border-color: var(--yellow-border);
background-color: rgba(46, 45, 5, 0.1);
}
.whitelist-theme h2, .whitelist-theme .status-icon, .whitelist-theme .subheading { color: var(--yellow-color); }
.whitelist-theme .reason-header { border-bottom: 1px solid var(--yellow-border); color: var(--yellow-color); }
.whitelist-theme .alert-box {
border: 1px solid var(--yellow-border);
background: var(--yellow-bg-subtle);
color: var(--yellow-color);
}
.result-header {
@@ -335,6 +356,12 @@ h1.error-code {
text-align: right;
}
.row-value.alert { color: var(--red-color); }
.row-value.success { color: var(--green-color); }
.hint {
text-decoration: underline;
text-decoration-style: dotted;
}
.alert-box {
padding: 1rem;
@@ -413,6 +440,84 @@ h1.error-code {
text-align: center;
}
.code {
border-left: 2px solid var(--border-color);
padding-left: 1em;
font-family: inherit;
overflow-x: scroll;
color: var(--text-muted);
}
.user-feedback-section {
margin-top: 1.5rem;
padding-top: 1rem;
border-top: 1px dashed var(--border-color);
}
.feedback-prompt {
color: var(--text-muted);
font-size: 0.875rem;
margin-bottom: 0.75rem;
}
.feedback-buttons {
display: flex;
gap: 0.75rem;
}
.feedback-btn {
padding: 0.5rem 1rem;
border: 1px solid transparent;
font-weight: bold;
font-size: 0.875rem;
cursor: pointer;
display: flex;
align-items: center;
gap: 0.5rem;
transition: all 0.2s;
flex-grow: 1;
justify-content: center;
font-family: inherit;
}
.blocked-theme .feedback-works {
background-color: var(--green-color);
color: var(--bg-color);
}
.blocked-theme .feedback-not-works, .whitelist-theme .feedback-not-works {
background-color: transparent;
color: var(--red-color);
border: 1px solid var(--red-color);
}
.blocked-theme .feedback-works:hover { background-color: #34d399; }
.blocked-theme .feedback-not-works:hover { background-color: rgba(239, 68, 68, 0.1); }
.clean-theme .feedback-works, .whitelist-theme .feedback-works {
background-color: transparent;
color: var(--green-color);
border: 1px solid var(--green-color);
}
.clean-theme .feedback-not-works {
background-color: var(--red-color);
color: var(--text-bright);
}
.clean-theme .feedback-works:hover { background-color: rgba(34, 197, 94, 0.1); }
.clean-theme .feedback-not-works:hover { background-color: #f87171; }
.feedback-status {
display: flex;
align-items: center;
gap: 0.5rem;
font-size: 0.875rem;
padding: 0.5rem 0;
color: #d4d4d4;
}
.blocked-theme .feedback-status { color: #f87171; }
.clean-theme .feedback-status { color: var(--green-color); }
.hidden {
display: none;
}
@media (min-width: 640px) {
body { padding: 2rem; }
+137
View File
@@ -0,0 +1,137 @@
{% macro head() %}
<script src="/vendor/chart.js"></script>
<script src="/vendor/chartjs-plugin-datalabels.js"></script>
{% endmacro %}
{% macro histogram(id="histogram",
endpoint="/whitelist/histogram",
data_label="Количество ресурсов",
x_label="Ранг домена",
y_label="Количество ресурсов") %}
<canvas id="{{ id }}"></canvas>
<style>
#{{ id }} {
max-height: 480px;
}
</style>
<script>
(async function () {
function formatNumber(num) {
return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
}
const TEXT_LIGHT = '#d4d4d4';
const RED_ACCENT = '#ef4444';
const AXIS_COLOR = '#404040';
const GRID_COLOR = 'rgba(64, 64, 64, 0.2)';
const ctx = document.getElementById('{{ id }}').getContext('2d');
const response = await fetch("{{ endpoint | safe }}");
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const data = await response.json();
const labels = data.map(item =>
`${formatNumber(Math.round(item.bin_min_rank))}-${formatNumber(Math.round(item.bin_max_rank))}`
);
const counts = data.map(item => item.count);
Chart.register(ChartDataLabels);
new Chart(ctx, {
type: 'bar',
data: {
labels: labels,
datasets: [{
label: '{{ data_label }}',
data: counts,
backgroundColor: RED_ACCENT,
borderColor: RED_ACCENT,
borderWidth: 1,
borderRadius: 6,
hoverBackgroundColor: `${RED_ACCENT}cc`,
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
plugins: {
datalabels: {
anchor: 'end',
align: 'top',
offset: 8,
color: TEXT_LIGHT,
font: {
weight: 'bold',
size: 10
},
formatter: (value) => value > 0 ? formatNumber(value) : '',
},
legend: {
display: false,
},
tooltip: {
callbacks: {
label: function(context) {
let label = context.dataset.label || '';
if (label) {
label += ': ';
}
if (context.parsed.y !== null) {
label += formatNumber(context.parsed.y);
}
return label;
}
},
backgroundColor: 'rgba(30, 30, 30, 0.9)',
titleColor: TEXT_LIGHT,
bodyColor: TEXT_LIGHT,
}
},
scales: {
x: {
title: {
display: true,
text: '{{ x_label }}',
color: TEXT_LIGHT
},
ticks: {
color: TEXT_LIGHT,
maxRotation: 45,
minRotation: 45
},
grid: {
color: GRID_COLOR,
drawBorder: false
},
border: {
color: AXIS_COLOR
}
},
y: {
type: 'linear',
title: {
display: true,
text: '{{ y_label }}',
color: TEXT_LIGHT
},
ticks: {
color: TEXT_LIGHT,
callback: () => ''
},
grid: {
color: GRID_COLOR,
drawBorder: false
},
border: {
color: AXIS_COLOR
}
}
}
}
});
})();
</script>
{% endmacro metadata %}
+7
View File
@@ -33,6 +33,13 @@
и <a href="https://antifilter.network/">antifilter.network</a>.
</p>
{{ typography::heading(title="Что такое «белый список»?") }}
<p>
РКН формирует свой белый список доменов, которые доступны,
даже если их адреса находятся в списке заблокированных подсетей.
</p>
<a href="/kb/whitelist">Подробнее...</a>
{{ typography::heading(title='Мне написало, что сайт "Заблокирован", но у меня все работает!') }}
<p>
Мы не можем с точностью утверждать, заблокирован сайт или нет.
@@ -0,0 +1,93 @@
{% extends 'page' %}
{% import 'page' as page %}
{% import 'histogram' as histogram %}
{% block head %}
{{ super() }}
{{ histogram::head() }}
{% endblock head %}
{% block metadata %}
{{ page::metadata(title="Белые списки", description="Анализ данных из белых списков блокировок", keywords="белые списки, whitelist", url="/kb/whitelist") }}
{% endblock metadata %}
{% block page_text %}
<h1>Белые списки доменов</h1>
<p>
Российские операторы связи начали применять новый тип блокировок CDN,
при котором загрузка контента обрывается после передачи примерно 16–20 килобайт данных,
из‑за чего большинство сайтов на Cloudflare и других зарубежных платформах становятся
практически неработоспособными для пользователей в России.
Одновременно сохраняются и развиваются белые списки с популярными и «социально значимыми» доменами,
которые старательно выводятся из‑под подобных ограничений.
</p>
<p>
Суть схемы в том, что соединение технически устанавливается, первые килобайты HTML,
CSS или скриптов передаются, но затем трафик режется или соединение принудительно сбрасывается на уровне
российских операторов, обычно после примерно 16 килобайт (10–14 пакетов в зависимости от протокола).
Этого объёма достаточно, чтобы страница начала загружаться и создавалась иллюзия доступности,
но ключевые части сайта (основной HTML, JS‑бандлы, стили, API‑запросы) не проходят, и сервис фактически
перестаёт работать.
</p>
<pre class="code">$ curl -k https://cheburcheck.ru/100MB.bin -o/dev/null -r 0-65536 --resolve cheburcheck.ru:443:5.78.7.195 --max-time 5
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
24 65537 24 16101 0 0 3220 0 0:00:20 0:00:04 0:00:16 0
curl: (28) Operation timed out after 5000 milliseconds with 16101 out of 65537 bytes received</pre>
<p>
Определение домена и выбор, «разрешить или задушить» запрос, в текущей схеме делается на связке DPI‑фильтров.
По умолчанию, все HTTP(s) запросы на адреса из подсетей хостеров блокируются. Однако, запросы
к некоторым доменам из тех же подсетей проходят нормально &mdash; они обнаруживаются по открытому расширению
TLS SNI, либо по заголовку Host (в случае с plain http).
</p>
<p>
Проверить нахождение домена в белом списке можно с помощью простого HTTPS-запроса с подменой SNI на адрес из
заблокированных диапазонов:
</p>
<pre class="code">$ curl -k https://ok.ru/100MB.bin -o/dev/null -r 0-65536 --resolve ok.ru:443:5.78.7.195
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 65537 100 65537 0 0 55226 0 0:00:01 0:00:01 --:--:-- 55258</pre>
<p>
Также, стоит отметить, что домены белого списка добавляются с маской по поддоменам:
</p>
<pre class="code">$ curl -k https://cheburcheck.ok.ru/100MB.bin -o/dev/null -r 0-65536 --resolve cheburcheck.ok.ru:443:
5.78.7.195
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 65537 100 65537 0 0 53192 0 0:00:01 0:00:01 --:--:-- 53195</pre>
{{ typography::heading(title="Автоматическое сканирование") }}
<p>
Используя методы, указанные выше, мы разработали инструмент для автоматического сканирования и анализа
блокировок
<b><a href="https://github.com/LowderPlay/cheburcheck/tree/master/reporter">Cheburcheck Reporter</a></b>.
Он позволяет сканировать большое количество доменов
за короткий срок за счет параллелизации запросов.
</p>
<p>
Исходя из анализа 1,000,000 доменов из рейтинга
<a href="https://tranco-list.eu/list/2NPQ9">Tranco list от 26 ноября 2025</a>,
в белом списке содержится около 1000 доменов. Все* домены из списка входят в топ-100,000 рейтинга.
</p>
<i>
* &mdash; Мы не включаем в это число домены из зоны .co.uk, так как по какой-то причине, они все находятся в белом списке.
</i>
<i class="caption">Гистограмма количества доменов относительно их положения в рейтинге</i>
{{ histogram::histogram(endpoint="/whitelist/histogram") }}
<br>
<i class="caption">Гистограмма без учета доменов .co.uk</i>
{{ histogram::histogram(id="filtered", endpoint="/whitelist/histogram?filter") }}
{{ typography::heading(title="Скачать списки") }}
<p>
Мы публикуем результаты наших сканирований в виде CSV-файлов:
</p>
<div>
<i data-lucide="file-symlink" width="16" height="16"></i> <a href="/whitelist/full.csv">Полный список (CSV)</a>
</div>
<div>
<i data-lucide="file-symlink" width="16" height="16"></i> <a href="/whitelist/domains.csv">Только домены (CSV)</a>
</div>
{% endblock page_text %}
+60 -5
View File
@@ -3,16 +3,27 @@
{% block content %}
{% include 'search-form' %}
<div class="result-panel {% if found %}blocked-theme{% else %}clean-theme{% endif %}">
{% set warning = whitelist and not domain %}
<div class="result-panel {% if warning %}whitelist-theme{% elif found %}blocked-theme{% else %}clean-theme{% endif %}">
<div class="result-header">
<div class="icon-box status-icon">
{% if found %}
{% if whitelist %}
<i data-lucide="shield-alert" width="32" height="32"></i>
{% elif blocked %}
<i data-lucide="shield-x" width="32" height="32"></i>
{% else %}
<i data-lucide="shield-check" width="32" height="32"></i>
{% endif %}
</div>
{% if found %}
{% if warning %}
<div>
<h2>Белый список</h2>
<p class="subheading text-sm">
Ресурс находится в белом списке
</p>
</div>
{% elif found %}
<div>
<h2>Заблокирован</h2>
<p class="subheading text-sm">Ресурс был найден в списках блокировок</p>
@@ -44,7 +55,7 @@
</div>
<div class="detail-row">
<span class="row-label">Хостинг / ISP</span>
<span class="row-value">{{ geo.organisation }}</span>
<span class="row-value">{% if geo.organisation %}{{ geo.organisation }}{% else %}-{% endif %}</span>
</div>
<div class="detail-row">
<span class="row-label">Локация</span>
@@ -52,7 +63,7 @@
</div>
<div class="detail-row">
<span class="row-label">ASN</span>
<span class="row-value">{{ geo.asn }}</span>
<span class="row-value">{% if geo.asn %}{{ geo.asn }}{% else %}-{% endif %}</span>
</div>
</div>
@@ -78,6 +89,21 @@
</div>
{% endfor %}
{% endif %}
{% if whitelist %}
<div class="detail-row">
<a href="/kb/whitelist" class="row-label">Белый список (?)</a>
<span class="row-value success">
НАЙДЕН - <span class="hint"
title="Дата последнего сканирования, когда данный домен был найден в белом списке">
<script>
document.write(new Date("{{ whitelist.last_ok }}").toLocaleDateString());
</script>
</span>
</span>
</div>
{% endif %}
<div class="detail-row">
<span class="row-label">Реестр РКН</span>
@@ -106,7 +132,36 @@
{% endif %}
</div>
</div>
<div class="user-feedback-section">
<p class="feedback-prompt">У вас работает этот ресурс?</p>
<div class="feedback-buttons">
<button class="feedback-btn feedback-works" onclick="sendFeedback(true)">
<i data-lucide="thumbs-up" width="16" height="16"></i>
Работает
</button>
<button class="feedback-btn feedback-not-works" onclick="sendFeedback(false)">
<i data-lucide="thumbs-down" width="16" height="16"></i>
Не работает
</button>
</div>
<div class="feedback-status hidden">
<i data-lucide="thumbs-up" width="16" height="16"></i>
<span>Спасибо за ваш отзыв!</span>
</div>
</div>
</div>
<script>
function sendFeedback(works) {
document.querySelector('.feedback-buttons').classList.add('hidden');
document.querySelector('.feedback-prompt').classList.add('hidden');
document.querySelector('.feedback-status').classList.remove('hidden');
fetch(`/feedback/{{ id }}/${works}`, {
method: 'POST',
});
}
</script>
{% endblock content %}