mirror of
https://github.com/LowderPlay/cheburcheck.git
synced 2026-09-29 02:08:08 +03:00
feat: add reporter, database
This commit is contained in:
@@ -1 +1,2 @@
|
||||
.git/
|
||||
target/
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
/target
|
||||
.idea/
|
||||
.env
|
||||
|
||||
Generated
+926
-45
File diff suppressed because it is too large
Load Diff
+5
-1
@@ -1,5 +1,8 @@
|
||||
[workspace]
|
||||
members = [ "querying",
|
||||
members = [
|
||||
"querying",
|
||||
"reporter",
|
||||
"reports",
|
||||
"website",
|
||||
]
|
||||
resolver = "3"
|
||||
@@ -8,3 +11,4 @@ resolver = "3"
|
||||
tokio = { version = "1.48.0", features = ["full"] }
|
||||
reqwest = { version = "0.12.24", features = ["stream", "rustls-tls-webpki-roots"] }
|
||||
log = "0.4.28"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
|
||||
@@ -33,14 +33,16 @@
|
||||
|
||||
Для проверки используются списки [123jjck/cdn-ip-ranges](https://github.com/123jjck/cdn-ip-ranges), [antifilter.download](https://antifilter.download/) и [antifilter.network](https://antifilter.network).
|
||||
|
||||
Мы собираем собственные белые списки с помощью [Cheburcheck Reporter](reporter/README.md).
|
||||
|
||||
---
|
||||
|
||||
## Структура проекта
|
||||
|
||||
* `src/` — исходный код приложения
|
||||
* `templates/` — Tera-шаблоны для фронтенда
|
||||
* `static/` — статические файлы
|
||||
* `Cargo.toml` — зависимости и конфигурация Rust-проекта
|
||||
* `querying` — модуль проверки сайтов по базам данных
|
||||
* `reporter` — [Cheburcheck Reporter](reporter/README.md)
|
||||
* `reports` — общий протокол для отправки отчетов
|
||||
* `website` — исходный код веб-сайта
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,2 +1,5 @@
|
||||
[default]
|
||||
template_dir = "website/templates/"
|
||||
|
||||
[global.limits]
|
||||
msgpack = 5242880
|
||||
|
||||
+30
-13
@@ -1,11 +1,11 @@
|
||||
use std::{io};
|
||||
use crate::updater::{fetch_db, Updatable};
|
||||
use async_trait::async_trait;
|
||||
use maxminddb::geoip2::{city, country, City, Country};
|
||||
use maxminddb::{geoip2, MaxMindDbError};
|
||||
use serde::Serialize;
|
||||
use std::io::Error;
|
||||
use std::net::IpAddr;
|
||||
use async_trait::async_trait;
|
||||
use maxminddb::{geoip2, MaxMindDbError};
|
||||
use maxminddb::geoip2::{city, country, City, Country};
|
||||
use serde::Serialize;
|
||||
use crate::updater::{fetch_db, Updatable};
|
||||
use std::io;
|
||||
|
||||
pub struct GeoIp {
|
||||
asn: Option<maxminddb::Reader<Vec<u8>>>,
|
||||
@@ -15,16 +15,20 @@ pub struct GeoIp {
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
pub struct IpInfo {
|
||||
asn: String,
|
||||
organisation: String,
|
||||
pub asn: Option<String>,
|
||||
pub country_code: Option<String>,
|
||||
pub organisation: Option<String>,
|
||||
pub city_geo_name_id: Option<u32>,
|
||||
location: String,
|
||||
}
|
||||
|
||||
impl Default for IpInfo {
|
||||
fn default() -> IpInfo {
|
||||
IpInfo {
|
||||
asn: "-".to_string(),
|
||||
organisation: "-".to_string(),
|
||||
asn: None,
|
||||
country_code: None,
|
||||
organisation: None,
|
||||
city_geo_name_id: None,
|
||||
location: "-".to_string(),
|
||||
}
|
||||
}
|
||||
@@ -57,6 +61,18 @@ impl GeoIp {
|
||||
db.lookup::<Country>(ip)?
|
||||
} else { None };
|
||||
|
||||
let country_code = country.as_ref().map(|c| c.country.as_ref()
|
||||
.map(|c| c.iso_code
|
||||
.map(|c| c.to_string()))
|
||||
.flatten())
|
||||
.flatten();
|
||||
|
||||
let city_geo_name_id = city.as_ref()
|
||||
.map(|c| c.city.as_ref()
|
||||
.map(|c| c.geoname_id)
|
||||
.flatten())
|
||||
.flatten();
|
||||
|
||||
let location = match (city, country) {
|
||||
(Some(City { city: Some(city::City { names: Some(city), .. }),
|
||||
country: Some(country::Country { names: Some(country), .. }), .. }), _) => {
|
||||
@@ -72,11 +88,12 @@ impl GeoIp {
|
||||
|
||||
Ok(IpInfo {
|
||||
location,
|
||||
country_code,
|
||||
city_geo_name_id,
|
||||
asn: asn.clone().and_then(|asn| asn.autonomous_system_number)
|
||||
.map(|asn| format!("AS{}", asn))
|
||||
.unwrap_or("-".to_string()),
|
||||
.map(|asn| format!("AS{}", asn)),
|
||||
organisation: asn.and_then(|asn| asn.autonomous_system_organization)
|
||||
.unwrap_or("-").to_string(),
|
||||
.map(|org| org.to_string()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+13
-8
@@ -1,16 +1,17 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use chrono::{DateTime, Utc};
|
||||
use ipnet::IpNet;
|
||||
use log::error;
|
||||
use thiserror::Error;
|
||||
use tokio::sync::{watch, RwLock};
|
||||
use crate::geoip::{GeoIp, IpInfo};
|
||||
use crate::lists::{CdnList, NetworkRecord, RuBlacklist};
|
||||
use crate::resolver::{ResolveError, Resolver};
|
||||
use crate::target::Target;
|
||||
use crate::updater::Updatable;
|
||||
use chrono::{DateTime, Utc};
|
||||
use ipnet::IpNet;
|
||||
use log::error;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use maxminddb::MaxMindDbError;
|
||||
use thiserror::Error;
|
||||
use tokio::sync::{watch, RwLock};
|
||||
|
||||
pub mod geoip;
|
||||
pub mod lists;
|
||||
@@ -66,6 +67,10 @@ impl Checker {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn geo_ip(&self, ip: IpAddr) -> Result<IpInfo, MaxMindDbError> {
|
||||
self.geo_ip.read().await.lookup(ip)
|
||||
}
|
||||
|
||||
pub async fn check(&self, target: Target) -> Result<Check, CheckError> {
|
||||
let ips = match target.resolve(&self.resolver).await {
|
||||
Ok(ips) => ips,
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::io;
|
||||
use std::io::{BufRead, Error, Read};
|
||||
use std::net::IpAddr;
|
||||
use std::str::FromStr;
|
||||
use crate::updater::{fetch_db, Updatable};
|
||||
use async_trait::async_trait;
|
||||
use ipnet::IpNet;
|
||||
use ipnet_trie::IpnetTrie;
|
||||
use log::info;
|
||||
use serde::{de, Deserialize, Deserializer, Serializer};
|
||||
use std::collections::VecDeque;
|
||||
use std::io;
|
||||
use std::io::{BufRead, Error, Read};
|
||||
use std::net::IpAddr;
|
||||
use std::str::FromStr;
|
||||
use trie_rs::map::{Trie, TrieBuilder};
|
||||
use crate::updater::{fetch_db, Updatable};
|
||||
|
||||
pub struct CdnList {
|
||||
trie: IpnetTrie<NetworkRecord>,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use hickory_resolver::config::{LookupIpStrategy, ResolverConfig, ResolverOpts};
|
||||
use hickory_resolver::name_server::TokioConnectionProvider;
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::net::IpAddr;
|
||||
use hickory_resolver::config::{LookupIpStrategy, ResolverConfig, ResolverOpts};
|
||||
use hickory_resolver::name_server::{TokioConnectionProvider};
|
||||
use thiserror::Error;
|
||||
|
||||
pub struct Resolver {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use crate::resolver::{ResolveError, Resolver};
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
||||
use url::Url;
|
||||
use crate::resolver::{ResolveError, Resolver};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Target {
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
use async_trait::async_trait;
|
||||
use futures_util::StreamExt;
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use log::info;
|
||||
use reqwest::IntoUrl;
|
||||
use std::fmt::Display;
|
||||
use std::io;
|
||||
use std::io::Error;
|
||||
use async_trait::async_trait;
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use futures_util::StreamExt;
|
||||
use log::info;
|
||||
use reqwest::IntoUrl;
|
||||
|
||||
pub async fn fetch_db<T: IntoUrl + Display>(url: T) -> Result<Vec<u8>, Error> {
|
||||
info!("Fetching {}", url);
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "reporter"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
tokio = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
log = { workspace = true }
|
||||
reports = { path = "../reports" }
|
||||
anyhow = "1.0"
|
||||
env_logger = "0.11.8"
|
||||
futures = "0.3"
|
||||
clap = { version = "4.5.53", features = ["derive", "env"] }
|
||||
csv = "1.4.0"
|
||||
indicatif = "0.18.3"
|
||||
rmp-serde = "1.3.0"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = "0.2.177"
|
||||
|
||||
[build-dependencies]
|
||||
reqwest = { workspace = true }
|
||||
getrandom = "0.3.4"
|
||||
@@ -0,0 +1,43 @@
|
||||
# Cheburcheck Reporter
|
||||
|
||||
Инструмент для автоматического сканирования и анализа блокировок.
|
||||
Позволяет сканировать большое количество доменов за короткий срок за счет параллелизации запросов.
|
||||
|
||||
## Использование
|
||||
|
||||
В программе заранее установлены настройки по-умолчанию для сканирования топ-100,000 доменов из [Tranco list от 26 ноября 2025](https://tranco-list.eu/list/2NPQ9).
|
||||
Для этого достаточно запустить его:
|
||||
|
||||
1. Сохранить результаты в CSV
|
||||
|
||||
```shell
|
||||
reporter output.csv
|
||||
```
|
||||
|
||||
2. Отправить результаты в **Cheburcheck Agency**
|
||||
|
||||
```shell
|
||||
reporter -k <API-ключ>
|
||||
```
|
||||
|
||||
> Для того чтобы получить API-ключ,
|
||||
> обратитесь по адресу [agency@cheburcheck.ru](mailto:agency@cheburcheck.ru).
|
||||
>
|
||||
> В письме укажите ваш интернет-провайдер и регион.
|
||||
|
||||
## Конфигурация
|
||||
|
||||
| Параметр | Описание | Значение по-умолчанию |
|
||||
|-------------------------------------|-----------------------------------------------------------------------------------------|--------------------------------------|
|
||||
| `-f, --fake <FAKE>` | Использовать один домен для всех запросов | |
|
||||
| `-c, --count <COUNT>` | Количество доменов, которые будут проверены (берутся с начала списка) | 100,000 |
|
||||
| `-t, --timeout-secs <TIMEOUT_SECS>` | Максимальное время ожидания ответа от сервера | 5 секунд |
|
||||
| `-p, --probes <PROBE_COUNT>` | Максимальное количество одновременных запросов | 1000 |
|
||||
| `-v, --verbosity <VERBOSITY>` | Отображение результатов сканирования в консоли [silent, error, block, all] | silent |
|
||||
| `-r, --retry-count <RETRY_COUNT>` | Количество попыток запросов на один домен | 2 |
|
||||
| `-H, --http` | Использовать plain-HTTP (без TLS) | |
|
||||
| `-x, --tx` | Отправлять мусорные данные 64кб на сервер | |
|
||||
| `-i, --ip <IP>` | IP-адрес сервера, на который будут идти запросы (должен отвечать >64kb на любые домены) | 5.78.7.195 |
|
||||
| `-P, --path <PATH>` | Путь к файлу на сервере | 100MB.bin |
|
||||
| `-a, --endpoint <AGENCY_ENDPOINT>` | Адрес сервера, на который будут загружены результаты сканирования | https://cheburcheck.ru/agency/report |
|
||||
| `-k, --key <KEY>` | API-ключ | |
|
||||
@@ -0,0 +1,25 @@
|
||||
use std::{env, fs, path::Path};
|
||||
use reqwest::blocking::Client;
|
||||
|
||||
fn main() {
|
||||
let out_dir = env::var("OUT_DIR").expect("OUT_DIR not set");
|
||||
let junk_path = Path::new(&out_dir).join("junk.bin");
|
||||
|
||||
let mut data = vec![0u8; 64 * 1024];
|
||||
getrandom::fill(&mut data).expect("Failed to generate random bytes");
|
||||
fs::write(&junk_path, &data).expect("Failed to write random data");
|
||||
|
||||
let client = Client::new();
|
||||
println!("cargo:rerun-if-env-changed=DIST_DOMAIN_COUNT");
|
||||
let data = client.get(
|
||||
format!("https://tranco-list.eu/download/2NPQ9/{}", option_env!("DIST_DOMAIN_COUNT")
|
||||
.map(|x| x.parse::<u32>().expect("DIST_DOMAIN_COUNT is not a number"))
|
||||
.unwrap_or(1_000_000)))
|
||||
.send().expect("Failed to send request")
|
||||
.bytes().expect("Failed to read bytes");
|
||||
|
||||
let list_csv = Path::new(&out_dir).join("list.csv");
|
||||
fs::write(&list_csv, &data).expect("Failed to write list");
|
||||
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Display;
|
||||
use std::path::PathBuf;
|
||||
use log::info;
|
||||
use reports::Evidence;
|
||||
use crate::Verbosity;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Counter {
|
||||
ok: usize,
|
||||
block: usize,
|
||||
err: usize,
|
||||
pub early: usize,
|
||||
pub results: HashMap<String, Evidence>,
|
||||
}
|
||||
|
||||
impl Counter {
|
||||
pub fn save_results(&self, output: &PathBuf) -> anyhow::Result<()> {
|
||||
let mut out = csv::WriterBuilder::new().from_path(output)?;
|
||||
out.write_record(&["target", "evidence"])?;
|
||||
for (target, evidence) in &self.results {
|
||||
out.write_record(&[target, &evidence.to_string()])?;
|
||||
}
|
||||
info!("Saved results to {:?}", output);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn print_results(&self, verbosity: &Verbosity) {
|
||||
if verbosity > &Verbosity::Silent {
|
||||
info!("Results:");
|
||||
for (target, evidence) in &self.results {
|
||||
match evidence {
|
||||
Evidence::Ok if verbosity >= &Verbosity::All => println!(" [Ok] {}", target),
|
||||
Evidence::Blocked if verbosity >= &Verbosity::Block => println!(" [Blocked] {}", target),
|
||||
Evidence::ConnectError if verbosity >= &Verbosity::Error => println!(" [ConnectError] {}", target),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
pub fn total(&self) -> usize {
|
||||
self.ok + self.block + self.err
|
||||
}
|
||||
|
||||
pub fn add(&mut self, target: &str, evidence: Evidence) {
|
||||
match evidence {
|
||||
Evidence::Ok => self.ok += 1,
|
||||
Evidence::Blocked => self.block += 1,
|
||||
Evidence::ConnectError | Evidence::Error => self.err += 1,
|
||||
}
|
||||
self.results.insert(target.to_string(), evidence);
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for Counter {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let total = self.total();
|
||||
write!(f, "OK {} ({:.2}%) | Blocked {} (early: {}) ({:.2}%) | Error {} ({:.2}%)",
|
||||
self.ok,
|
||||
self.ok as f32 / total as f32 * 100.0,
|
||||
self.block, self.early,
|
||||
self.block as f32 / total as f32 * 100.0,
|
||||
self.err,
|
||||
self.err as f32 / total as f32 * 100.0)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
mod resolver;
|
||||
mod counter;
|
||||
|
||||
use crate::resolver::Resolver;
|
||||
use anyhow::Result;
|
||||
use clap::{Parser, ValueEnum};
|
||||
use futures::stream::FuturesUnordered;
|
||||
use futures::StreamExt;
|
||||
use indicatif::{ProgressIterator, ProgressStyle};
|
||||
use log::{error, info, warn, LevelFilter};
|
||||
use reports::{AgencyReport, Evidence, ReporterConfig};
|
||||
use reqwest::redirect::Policy;
|
||||
use reqwest::Client;
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Display;
|
||||
use std::net::IpAddr;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::time::Instant;
|
||||
use counter::Counter;
|
||||
|
||||
const JUNK: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/junk.bin"));
|
||||
|
||||
#[derive(Serialize, Debug, Ord, PartialOrd, Eq, PartialEq, Clone, ValueEnum)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
enum Verbosity {
|
||||
Silent,
|
||||
Error,
|
||||
Block,
|
||||
All,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug, Clone)]
|
||||
#[command(author, version, about = "DPI probe: checks blockage of domains by SNI")]
|
||||
struct Args {
|
||||
/// Output results file
|
||||
#[arg(required = false)]
|
||||
output: Option<PathBuf>,
|
||||
|
||||
/// Fake target to establish probing settings.
|
||||
/// Use with caution - it might trigger TLS block
|
||||
#[arg(short, long, required = false)]
|
||||
fake: Option<String>,
|
||||
|
||||
/// Take first N targets
|
||||
#[arg(short, long, default_value_t = 100_000)]
|
||||
count: usize,
|
||||
|
||||
/// Read timeout in seconds
|
||||
#[arg(short, long, default_value_t = 5)]
|
||||
timeout_secs: u64,
|
||||
|
||||
/// Maximum concurrent probes. Make sure that it doesn't exceed 'ulimit -n'
|
||||
#[arg(short, long = "probes", default_value_t = 1000)]
|
||||
probe_count: usize,
|
||||
|
||||
/// Display probing results in console
|
||||
#[arg(short, long, default_value_t = Verbosity::Silent, value_enum)]
|
||||
verbosity: Verbosity,
|
||||
|
||||
/// Attempts to establish connection
|
||||
#[arg(short, long, default_value_t = 2)]
|
||||
retry_count: usize,
|
||||
|
||||
/// Try using plain HTTP without TLS
|
||||
#[arg(short = 'H', long, default_value_t = false)]
|
||||
http: bool,
|
||||
|
||||
/// Send 64kb junk to server
|
||||
#[arg(short = 'x', long, default_value_t = false)]
|
||||
tx: bool,
|
||||
|
||||
/// Target IP to probe with.
|
||||
/// It should be included in IP-ranges of interest.
|
||||
/// The server must respond to any SNI/Host with a response larger than 64kb.
|
||||
#[arg(short, long, default_value = "5.78.7.195", value_parser = |v: &str| v.parse::<IpAddr>())]
|
||||
ip: IpAddr,
|
||||
|
||||
/// File name on the server to test
|
||||
#[arg(short = 'P', long, default_value = "100MB.bin")]
|
||||
path: String,
|
||||
|
||||
/// Custom agency endpoint address
|
||||
#[arg(short, long = "endpoint", default_value_t = option_env!("AGENCY_ENDPOINT")
|
||||
.unwrap_or("https://cheburcheck.ru/agency/upload")
|
||||
.to_string())]
|
||||
agency_endpoint: String,
|
||||
|
||||
/// Agency endpoint API key
|
||||
#[arg(short, long, env = "AGENCY_KEY")]
|
||||
key: Option<String>,
|
||||
|
||||
}
|
||||
|
||||
impl Args {
|
||||
fn to_reporter_config(&self) -> ReporterConfig {
|
||||
ReporterConfig {
|
||||
http: self.http,
|
||||
tx_junk: self.tx,
|
||||
ip: self.ip.clone(),
|
||||
path: self.path.clone(),
|
||||
retry_count: self.retry_count,
|
||||
timeout_secs: self.timeout_secs,
|
||||
probe_count: self.probe_count,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_client(args: &Args, attempt: usize) -> reqwest::Result<Client> {
|
||||
let client = Client::builder()
|
||||
.danger_accept_invalid_certs(true)
|
||||
.redirect(Policy::none())
|
||||
.use_rustls_tls()
|
||||
.dns_resolver(Arc::new(Resolver::new(args.ip)))
|
||||
.read_timeout(Duration::from_secs(args.timeout_secs * attempt as u64))
|
||||
.timeout(Duration::from_secs(15));
|
||||
|
||||
Ok(client.build()?)
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let args = Args::parse();
|
||||
env_logger::builder().filter_level(LevelFilter::Info).init();
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
{
|
||||
let file_limit: Option<usize> = unsafe { libc::getdtablesize() }.try_into().ok();
|
||||
if matches!(file_limit, Some(file_limit) if file_limit <= args.probe_count + 128) {
|
||||
warn!("Open file limit is too low ({})! Consider increasing it using `ulimit -n`.", file_limit.unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
let api_client = Client::new();
|
||||
info!("Loading targets list...");
|
||||
let targets = include_str!(concat!(env!("OUT_DIR"), "/list.csv"));
|
||||
let targets: Vec<String> = targets.lines().take(args.count)
|
||||
.map(|s| s.split(",").last().unwrap().to_string()).collect();
|
||||
|
||||
info!("Probing {} domains with {} concurrent probes...", targets.len(), args.probe_count);
|
||||
let sem = Arc::new(tokio::sync::Semaphore::new(args.probe_count));
|
||||
let cancelled = wait_for_ctrlc();
|
||||
let start = Instant::now();
|
||||
let mut futs = FuturesUnordered::new();
|
||||
for target in targets.into_iter().progress()
|
||||
.with_style(ProgressStyle::default_bar()
|
||||
.template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {human_pos}/{human_len} ({eta}, {per_sec})")?
|
||||
.progress_chars("#>-")) {
|
||||
if cancelled() {
|
||||
break;
|
||||
}
|
||||
let permit = sem.clone().acquire_owned().await?;
|
||||
let args = args.clone();
|
||||
let fake_target = args.fake.clone();
|
||||
futs.push(tokio::spawn(async move {
|
||||
let res = check_target(&args, fake_target.as_ref().unwrap_or(&target)).await;
|
||||
drop(permit);
|
||||
(target, res)
|
||||
}));
|
||||
}
|
||||
info!("Collecting results...");
|
||||
|
||||
let mut counter = Counter::default();
|
||||
while let Some(res) = futs.next().await {
|
||||
match res {
|
||||
Ok((target, Ok(Verdict::Accepted))) => {
|
||||
counter.add(&target, Evidence::Ok);
|
||||
}
|
||||
Ok((target, Ok(Verdict::Blocked { early }))) => {
|
||||
counter.add(&target, Evidence::Blocked);
|
||||
if early {
|
||||
counter.early += 1;
|
||||
}
|
||||
}
|
||||
Ok((target, Err(e))) if e.is_connect() => {
|
||||
counter.add(&target, Evidence::ConnectError);
|
||||
if args.verbosity >= Verbosity::Error {
|
||||
println!("{e:?}");
|
||||
}
|
||||
}
|
||||
Ok((target, Err(_))) => {
|
||||
counter.add(&target, Evidence::Error);
|
||||
}
|
||||
Err(join_err) => {
|
||||
error!("Task join error: {}", join_err);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
counter.print_results(&args.verbosity);
|
||||
if let Some(output) = &args.output {
|
||||
counter.save_results(output)?;
|
||||
}
|
||||
|
||||
info!("Probed {} domains in {}s! \nSummary: {counter}", counter.total(), start.elapsed().as_secs());
|
||||
if let Err(e) = upload_results(&args, &api_client, counter.results).await {
|
||||
warn!("Upload failed: {}", e);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn upload_results(args: &Args, api_client: &Client, results: HashMap<String, Evidence>) -> Result<()> {
|
||||
info!("Uploading to {}", args.agency_endpoint);
|
||||
|
||||
let uploaded = api_client.post(&args.agency_endpoint)
|
||||
.header("Content-Type", "application/msgpack")
|
||||
.body(rmp_serde::to_vec(&AgencyReport {
|
||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
config: args.to_reporter_config(),
|
||||
data: results,
|
||||
})?);
|
||||
|
||||
let uploaded = if let Some(key) = &args.key {
|
||||
uploaded.header("Authorization", format!("Bearer {key}"))
|
||||
} else { uploaded };
|
||||
|
||||
let uploaded = uploaded.send().await?;
|
||||
|
||||
// let uploaded = uploaded.error_for_status()?;
|
||||
|
||||
if uploaded.status().is_success() {
|
||||
info!("Uploaded ({})!", uploaded.status().to_string());
|
||||
} else {
|
||||
warn!("Upload failed: {}", uploaded.status().to_string());
|
||||
}
|
||||
info!("Agency response: {}", uploaded.text().await?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn wait_for_ctrlc() -> impl Fn() -> bool {
|
||||
let cancelled = Arc::new(AtomicUsize::new(0));
|
||||
let cancelled_ctrlc = cancelled.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
|
||||
match cancelled_ctrlc.fetch_add(1, Ordering::SeqCst) {
|
||||
0 => warn!("Ctrl-C received. Finishing up and saving..."),
|
||||
_ => {
|
||||
warn!("Forcing exit.");
|
||||
std::process::exit(130);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
move || {
|
||||
cancelled.load(Ordering::SeqCst) != 0
|
||||
}
|
||||
}
|
||||
|
||||
enum Verdict {
|
||||
Blocked { early: bool },
|
||||
Accepted,
|
||||
}
|
||||
|
||||
async fn check_target(args: &Args, target: &str) -> Result<Verdict, reqwest::Error> {
|
||||
let url = format!("http{}://{target}/{}", if args.http {""} else {"s"}, args.path);
|
||||
let mut attempts = 0;
|
||||
|
||||
loop {
|
||||
attempts += 1;
|
||||
let client = build_client(&args, 1)?;
|
||||
let mut resp = client.get(&url)
|
||||
.header("Range", "bytes=0-65536");
|
||||
if args.tx {
|
||||
resp = resp.body(JUNK)
|
||||
}
|
||||
let resp = resp.send()
|
||||
.await;
|
||||
|
||||
let resp = match resp {
|
||||
Ok(resp) => match (resp.status(), resp.bytes().await) {
|
||||
(status, Ok(b)) => Ok((status, b)),
|
||||
(_, Err(e)) => Err((e, false)),
|
||||
},
|
||||
Err(e) => Err((e, true)),
|
||||
};
|
||||
return match resp {
|
||||
Ok((status, bytes)) => {
|
||||
if !status.is_success() {
|
||||
warn!("Domain {target} returned non-OK code: {status}");
|
||||
}
|
||||
if bytes.len() < 65535 {
|
||||
warn!("Domain {target} completed with {} bytes: \n{}", bytes.len(), String::from_utf8_lossy(bytes.as_ref()));
|
||||
}
|
||||
Ok(Verdict::Accepted)
|
||||
}
|
||||
Err((e, early)) => {
|
||||
if attempts < args.retry_count {
|
||||
continue;
|
||||
}
|
||||
if e.is_timeout() {
|
||||
Ok(Verdict::Blocked { early })
|
||||
} else {
|
||||
error!("{} -> Error: {:?}", target, e);
|
||||
Err(e)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
|
||||
pub struct Resolver {
|
||||
ip: SocketAddr,
|
||||
}
|
||||
|
||||
impl Resolver {
|
||||
pub fn new(ip: IpAddr) -> Resolver {
|
||||
Resolver {
|
||||
ip: SocketAddr::from(SocketAddr::new(ip, 0)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Resolve for Resolver {
|
||||
fn resolve(&self, _: Name) -> Resolving {
|
||||
let ip = self.ip.clone();
|
||||
Box::pin(async move {
|
||||
Ok(Addrs::from(Box::new(vec![ip].into_iter())))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
[package]
|
||||
name = "reports"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true }
|
||||
@@ -0,0 +1,42 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Display;
|
||||
use std::net::IpAddr;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct AgencyReport {
|
||||
pub version: String,
|
||||
pub config: ReporterConfig,
|
||||
pub data: HashMap<String, Evidence>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub enum Evidence {
|
||||
Ok,
|
||||
Blocked,
|
||||
ConnectError,
|
||||
Error,
|
||||
}
|
||||
|
||||
impl Display for Evidence {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let str = match self {
|
||||
Evidence::Ok => "ok",
|
||||
Evidence::Blocked => "blocked",
|
||||
Evidence::ConnectError => "connect_error",
|
||||
Evidence::Error => "unknown_error",
|
||||
};
|
||||
write!(f, "{}", str)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct ReporterConfig {
|
||||
pub http: bool,
|
||||
pub tx_junk: bool,
|
||||
pub ip: IpAddr,
|
||||
pub path: String,
|
||||
pub retry_count: usize,
|
||||
pub timeout_secs: u64,
|
||||
pub probe_count: usize,
|
||||
}
|
||||
@@ -9,7 +9,9 @@ RUN apt update && apt install -y libssl-dev pkg-config
|
||||
RUN --mount=type=cache,target=/build/target \
|
||||
--mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=secret,id=DATABASE_URL \
|
||||
set -eux; \
|
||||
export DATABASE_URL=$(cat /run/secrets/DATABASE_URL); \
|
||||
cargo build --release --package website; \
|
||||
objcopy --compress-debug-sections target/release/website ./main
|
||||
|
||||
|
||||
+8
-3
@@ -1,16 +1,21 @@
|
||||
[package]
|
||||
name = "website"
|
||||
version = "0.1.4"
|
||||
version = "0.2.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
rocket = "0.5.1"
|
||||
rocket = { version = "0.5.1", features = ["msgpack", "json"] }
|
||||
rocket_db_pools = { version = "0.2.0", features = ["sqlx_postgres"]}
|
||||
rocket_dyn_templates = { version = "0.2.0", features = ["tera"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
sqlx = { version = "0.7", features = ["runtime-tokio", "postgres", "migrate", "chrono", "uuid"] }
|
||||
rocket-client-addr = "0.5.4"
|
||||
serde = { workspace = true }
|
||||
reports = { path = "../reports" }
|
||||
querying = { path = "../querying" }
|
||||
env_logger = "0.11.8"
|
||||
rocket-cache-response = "0.6.4"
|
||||
log = { workspace = true }
|
||||
dotenvy = { version = "0.15.7" }
|
||||
|
||||
[build-dependencies]
|
||||
reqwest = { version = "0.12", features = ["blocking", "json"] }
|
||||
|
||||
+36
-18
@@ -1,10 +1,11 @@
|
||||
use std::{
|
||||
env,
|
||||
fs::{File},
|
||||
fs::File,
|
||||
io::{self, Cursor},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use flate2::read::GzDecoder;
|
||||
use reqwest::blocking::Client;
|
||||
use serde::Deserialize;
|
||||
@@ -21,13 +22,35 @@ struct Dist {
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let out_dir = env::var("OUT_DIR")?;
|
||||
let lucide_js_path = Path::new(&out_dir).join("lucide.js");
|
||||
|
||||
let client = Client::new();
|
||||
println!("cargo:rerun-if-changed=migrations");
|
||||
let out_dir = PathBuf::from(env::var("OUT_DIR")?);
|
||||
|
||||
println!("cargo:rerun-if-env-changed=LUCIDE_VERSION");
|
||||
let metadata_url = format!("https://registry.npmjs.org/lucide/{}", option_env!("LUCIDE_VERSION").unwrap_or("latest"));
|
||||
install_npm("lucide", option_env!("LUCIDE_VERSION"), HashMap::from([
|
||||
(PathBuf::from("package/dist/umd/lucide.min.js"), out_dir.join("lucide.js")),
|
||||
]))?;
|
||||
|
||||
println!("cargo:rerun-if-env-changed=CHARTJS_VERSION");
|
||||
install_npm("chart.js", option_env!("CHARTJS_VERSION"), HashMap::from([
|
||||
(PathBuf::from("package/dist/chart.umd.min.js"), out_dir.join("chart.js")),
|
||||
]))?;
|
||||
|
||||
println!("cargo:rerun-if-env-changed=CHARTDATALABELS_VERSION");
|
||||
install_npm("chartjs-plugin-datalabels", option_env!("CHARTDATALABELS_VERSION"), HashMap::from([
|
||||
(PathBuf::from("package/dist/chartjs-plugin-datalabels.min.js"), out_dir.join("chartjs-plugin-datalabels.js")),
|
||||
]))?;
|
||||
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn install_npm(package: &str, version: Option<&str>, mut entries: HashMap<PathBuf, PathBuf>) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let client = Client::new();
|
||||
let metadata_url = format!(
|
||||
"https://registry.npmjs.org/{package}/{}",
|
||||
version.unwrap_or("latest")
|
||||
);
|
||||
|
||||
println!("Fetching npm metadata from {}", metadata_url);
|
||||
let metadata: NpmMetadata = client.get(metadata_url).send()?.json()?;
|
||||
@@ -38,24 +61,19 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let tar = GzDecoder::new(Cursor::new(tarball_bytes));
|
||||
let mut archive = Archive::new(tar);
|
||||
|
||||
let mut lucide_js_found = false;
|
||||
for entry in archive.entries()? {
|
||||
let mut entry = entry?;
|
||||
let path = entry.path()?;
|
||||
if path == Path::new("package/dist/umd/lucide.min.js") {
|
||||
println!("Found lucide.min.js, extracting to {:?}", lucide_js_path);
|
||||
let mut out_file = File::create(&lucide_js_path)?;
|
||||
let path = PathBuf::from(entry.path()?);
|
||||
if let Some(dest) = entries.remove(&path) {
|
||||
println!("Found {:?}, extracting to {:?}", path.file_name(), dest);
|
||||
let mut out_file = File::create(&dest)?;
|
||||
io::copy(&mut entry, &mut out_file)?;
|
||||
lucide_js_found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if !lucide_js_found {
|
||||
panic!("Could not find dist/lucide.min.js inside the tarball");
|
||||
if !entries.is_empty() {
|
||||
panic!("Not all entries found! {:?}", entries);
|
||||
}
|
||||
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
-- Queries
|
||||
CREATE TABLE IF NOT EXISTS queries
|
||||
(
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
query VARCHAR(255) NOT NULL,
|
||||
source_ip VARCHAR(39) NOT NULL,
|
||||
source_country_code VARCHAR(5),
|
||||
source_city_geo_name_id INT,
|
||||
|
||||
target_country_code VARCHAR(5),
|
||||
target_asn VARCHAR(32),
|
||||
target_provider VARCHAR(255),
|
||||
|
||||
resolved_ips VARCHAR(39)[],
|
||||
|
||||
cdn_networks VARCHAR(43)[],
|
||||
cdn_providers VARCHAR(255)[],
|
||||
|
||||
rkn_domain VARCHAR(255),
|
||||
|
||||
date TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS queries_query_date_idx ON queries (query, date);
|
||||
|
||||
-- Human reports
|
||||
CREATE TABLE IF NOT EXISTS human_reports
|
||||
(
|
||||
id UUID PRIMARY KEY REFERENCES queries (id),
|
||||
source_ip VARCHAR(39) NOT NULL,
|
||||
date TIMESTAMP DEFAULT NOW(),
|
||||
works BOOLEAN
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS human_reports_date_idx ON human_reports (date);
|
||||
CREATE INDEX IF NOT EXISTS human_reports_works_idx ON human_reports (works);
|
||||
|
||||
-- Domain rankings list
|
||||
CREATE TABLE IF NOT EXISTS domains
|
||||
(
|
||||
domain VARCHAR(255) PRIMARY KEY,
|
||||
rank INT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS domains_domain_rank_idx ON domains (domain, rank);
|
||||
|
||||
-- Agency reporters
|
||||
CREATE TABLE IF NOT EXISTS reporters
|
||||
(
|
||||
id SERIAL PRIMARY KEY,
|
||||
token VARCHAR(255) NOT NULL,
|
||||
name VARCHAR(255) NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS reporters_token_idx ON reporters (token);
|
||||
|
||||
-- Agency reports
|
||||
CREATE TABLE IF NOT EXISTS reports
|
||||
(
|
||||
id SERIAL PRIMARY KEY,
|
||||
reporter SERIAL,
|
||||
reporter_ip VARCHAR(39) NOT NULL,
|
||||
date TIMESTAMP DEFAULT NOW(),
|
||||
|
||||
version VARCHAR(32) NOT NULL,
|
||||
http BOOLEAN,
|
||||
tx_junk BOOLEAN,
|
||||
ip VARCHAR(39),
|
||||
path VARCHAR(255),
|
||||
retry_count INT,
|
||||
timeout_secs INT,
|
||||
probe_count INT,
|
||||
|
||||
FOREIGN KEY (reporter) REFERENCES reporters (id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS reports_reporter_date_id_idx ON reports (reporter, date DESC, id);
|
||||
|
||||
DO
|
||||
$$
|
||||
BEGIN
|
||||
CREATE TYPE evidence AS ENUM ('ok', 'blocked', 'connection_error', 'unknown_error');
|
||||
EXCEPTION
|
||||
WHEN duplicate_object THEN null;
|
||||
END
|
||||
$$;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS report_row
|
||||
(
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
report_id SERIAL,
|
||||
domain VARCHAR(255),
|
||||
evidence evidence,
|
||||
FOREIGN KEY (report_id) REFERENCES reports (id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS report_row_id_domain_evidence_idx ON report_row (report_id, domain, evidence);
|
||||
|
||||
-- Whitelist view
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS
|
||||
whitelist AS
|
||||
WITH ranked_reports AS (SELECT rr.domain,
|
||||
rr.evidence,
|
||||
r.date,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY
|
||||
rr.domain
|
||||
ORDER BY
|
||||
r.date DESC
|
||||
) AS rn
|
||||
FROM report_row rr
|
||||
JOIN reports r ON rr.report_id = r.id
|
||||
WHERE r.reporter = 1)
|
||||
SELECT rr.domain,
|
||||
d.rank,
|
||||
MAX(
|
||||
CASE
|
||||
WHEN rr.evidence = 'ok' THEN rr.date
|
||||
END
|
||||
) AS last_ok
|
||||
FROM ranked_reports rr
|
||||
LEFT JOIN domains d ON d.domain = rr.domain
|
||||
WHERE rr.rn <= 5
|
||||
GROUP BY rr.domain,
|
||||
d.rank
|
||||
HAVING COUNT(*) FILTER (
|
||||
WHERE
|
||||
rr.evidence = 'ok'
|
||||
) >= COUNT(*) / 2.0
|
||||
ORDER BY d.rank;
|
||||
@@ -0,0 +1,85 @@
|
||||
use crate::Db;
|
||||
use reports::AgencyReport;
|
||||
use rocket::http::Status;
|
||||
use rocket::serde::json::serde_json::json;
|
||||
use rocket::serde::json::{Json, Value};
|
||||
use rocket::serde::msgpack::MsgPack;
|
||||
use rocket_client_addr::ClientRealAddr;
|
||||
use rocket_db_pools::Connection;
|
||||
use sqlx::Acquire;
|
||||
|
||||
pub struct Agency {
|
||||
pub id: i32,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[rocket::post("/report", format = "application/msgpack", data = "<report>")]
|
||||
pub async fn upload_report(
|
||||
report: MsgPack<AgencyReport>,
|
||||
addr: &ClientRealAddr,
|
||||
agency: Agency,
|
||||
mut db: Connection<Db>,
|
||||
) -> Result<Json<Value>, (Status, String)> {
|
||||
let mut tx = db
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
|
||||
let report = report.into_inner();
|
||||
|
||||
let report_id: i32 = sqlx::query_scalar(
|
||||
"INSERT INTO reports (
|
||||
reporter,
|
||||
reporter_ip,
|
||||
version,
|
||||
http,
|
||||
tx_junk,
|
||||
ip,
|
||||
path,
|
||||
retry_count,
|
||||
timeout_secs,
|
||||
probe_count
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id",
|
||||
)
|
||||
.bind(agency.id)
|
||||
.bind(addr.ip.to_string())
|
||||
.bind(report.version)
|
||||
.bind(report.config.http)
|
||||
.bind(report.config.tx_junk)
|
||||
.bind(report.config.ip.to_string())
|
||||
.bind(report.config.path)
|
||||
.bind(report.config.retry_count as i32)
|
||||
.bind(report.config.timeout_secs as i64)
|
||||
.bind(report.config.probe_count as i32)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
|
||||
|
||||
let mut copy_in = tx
|
||||
.copy_in_raw("COPY report_row (report_id, evidence, domain) FROM STDIN (FORMAT CSV)")
|
||||
.await
|
||||
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
|
||||
|
||||
for (domain, evidence) in report.data {
|
||||
let line = format!("{},{},{}\n", report_id, evidence, domain);
|
||||
copy_in
|
||||
.send(line.as_bytes())
|
||||
.await
|
||||
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
|
||||
}
|
||||
|
||||
copy_in
|
||||
.finish()
|
||||
.await
|
||||
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
|
||||
|
||||
sqlx::query!("REFRESH MATERIALIZED VIEW whitelist")
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
|
||||
|
||||
tx.commit()
|
||||
.await
|
||||
.map_err(|e| (Status::InternalServerError, e.to_string()))?;
|
||||
|
||||
Ok(Json(json!({ "ok": true, "id": report_id })))
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
use crate::agency::Agency;
|
||||
use crate::Db;
|
||||
use querying::target::Target;
|
||||
use querying::{Check, CheckVerdict, Checker};
|
||||
use rocket::http::Status;
|
||||
use rocket::outcome::{try_outcome, IntoOutcome};
|
||||
use rocket::request::{FromRequest, Outcome};
|
||||
use rocket::tokio::sync::RwLockReadGuard;
|
||||
use rocket::Request;
|
||||
use rocket_client_addr::ClientRealAddr;
|
||||
use rocket_db_pools::Connection;
|
||||
use serde::Serialize;
|
||||
use sqlx::types::chrono::NaiveDateTime;
|
||||
use sqlx::types::Uuid;
|
||||
|
||||
pub async fn save_query(
|
||||
db: &mut Connection<Db>,
|
||||
target: &Target,
|
||||
check: &Check,
|
||||
addr: &ClientRealAddr,
|
||||
checker: RwLockReadGuard<'_, Checker>,
|
||||
) -> Result<Uuid, sqlx::Error> {
|
||||
let (cdn_networks, cdn_providers, rkn_domain): (Vec<_>, Vec<_>, Option<_>) =
|
||||
if let CheckVerdict::Blocked {
|
||||
cdn_provider_subnets,
|
||||
rkn_domain,
|
||||
..
|
||||
} = &check.verdict
|
||||
{
|
||||
(
|
||||
cdn_provider_subnets
|
||||
.values()
|
||||
.flatten()
|
||||
.map(|n| n.cidr.to_string())
|
||||
.collect(),
|
||||
cdn_provider_subnets.keys().map(|p| p.to_string()).collect(),
|
||||
rkn_domain.clone(),
|
||||
)
|
||||
} else {
|
||||
(vec![], vec![], None)
|
||||
};
|
||||
|
||||
let id = sqlx::query_scalar(
|
||||
"INSERT INTO queries (
|
||||
query,
|
||||
source_ip,
|
||||
source_country_code,
|
||||
source_city_geo_name_id,
|
||||
target_country_code,
|
||||
target_asn,
|
||||
target_provider,
|
||||
resolved_ips,
|
||||
cdn_networks,
|
||||
cdn_providers,
|
||||
rkn_domain
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING id",
|
||||
)
|
||||
.bind(target.to_query())
|
||||
.bind(addr.ip.to_string())
|
||||
.bind(
|
||||
checker
|
||||
.geo_ip(addr.ip)
|
||||
.await
|
||||
.map(|i| i.country_code)
|
||||
.ok()
|
||||
.flatten(),
|
||||
)
|
||||
.bind(check.geo.city_geo_name_id.map(|id| id as i32))
|
||||
.bind(check.geo.country_code.clone())
|
||||
.bind(check.geo.asn.clone())
|
||||
.bind(check.geo.organisation.clone())
|
||||
.bind(
|
||||
check
|
||||
.ips
|
||||
.iter()
|
||||
.map(|i| i.to_string())
|
||||
.collect::<Vec<String>>(),
|
||||
)
|
||||
.bind(cdn_networks)
|
||||
.bind(cdn_providers)
|
||||
.bind(rkn_domain)
|
||||
.fetch_one(&mut ***db)
|
||||
.await?;
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
#[rocket::async_trait]
|
||||
impl<'r> FromRequest<'r> for Agency {
|
||||
type Error = Option<rocket_db_pools::Error<sqlx::Error>>;
|
||||
|
||||
async fn from_request(request: &'r Request<'_>) -> Outcome<Self, Self::Error> {
|
||||
let mut db = try_outcome!(Connection::<Db>::from_request(request).await);
|
||||
let token = request.headers().get_one("Authorization");
|
||||
|
||||
let token = try_outcome!(
|
||||
token
|
||||
.and_then(|t| t.split_once(" "))
|
||||
.map(|(_, tok)| tok.to_string())
|
||||
.or_forward(Status::Unauthorized)
|
||||
);
|
||||
|
||||
let agency = try_outcome!(
|
||||
sqlx::query!("SELECT id, name FROM reporters WHERE token = $1", token)
|
||||
.fetch_optional(&mut **db)
|
||||
.await
|
||||
.map_err(|e| Some(rocket_db_pools::Error::Get(e)))
|
||||
.or_forward(Status::InternalServerError)
|
||||
);
|
||||
agency
|
||||
.map(|r| Agency {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
})
|
||||
.or_forward(Status::Unauthorized)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug, sqlx::FromRow)]
|
||||
pub struct WhitelistedEntry {
|
||||
domain: Option<String>,
|
||||
rank: Option<i32>,
|
||||
last_ok: Option<NaiveDateTime>,
|
||||
}
|
||||
|
||||
pub async fn check_whitelist(
|
||||
domain: &str,
|
||||
db: &mut Connection<Db>,
|
||||
) -> Result<Option<WhitelistedEntry>, sqlx::Error> {
|
||||
sqlx::query_as!(
|
||||
WhitelistedEntry,
|
||||
"SELECT *
|
||||
FROM whitelist
|
||||
WHERE $1 = domain
|
||||
OR $1 LIKE CONCAT('%.', domain)
|
||||
ORDER BY LENGTH(domain) DESC
|
||||
LIMIT 1",
|
||||
domain
|
||||
)
|
||||
.fetch_optional(&mut ***db)
|
||||
.await
|
||||
.into()
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, sqlx::FromRow)]
|
||||
pub struct WhitelistHistogramBin {
|
||||
pub bin_id: Option<i32>,
|
||||
pub bin_min_rank: Option<i32>,
|
||||
pub bin_max_rank: Option<i32>,
|
||||
pub count: Option<i64>,
|
||||
}
|
||||
|
||||
pub async fn collect_histogram(
|
||||
db: &mut Connection<Db>,
|
||||
bins: i32,
|
||||
limit: i32,
|
||||
filter: bool,
|
||||
) -> Result<Vec<WhitelistHistogramBin>, sqlx::Error> {
|
||||
|
||||
sqlx::query_as!(
|
||||
WhitelistHistogramBin,
|
||||
"WITH bins AS (
|
||||
SELECT generate_series(0, $1 - 1) AS bin
|
||||
)
|
||||
SELECT
|
||||
b.bin as bin_id,
|
||||
b.bin * $2 + 1 AS bin_min_rank,
|
||||
(b.bin + 1) * $2 AS bin_max_rank,
|
||||
COUNT(case when not $3 or w.domain not like '%.co.uk' then 1 end) AS count
|
||||
FROM bins b
|
||||
LEFT JOIN whitelist w
|
||||
ON FLOOR(w.rank / $2) = b.bin
|
||||
GROUP BY b.bin
|
||||
ORDER BY b.bin;", bins, limit / bins, filter
|
||||
)
|
||||
.fetch_all(&mut ***db)
|
||||
.await
|
||||
.into()
|
||||
}
|
||||
+180
-54
@@ -1,21 +1,34 @@
|
||||
#[macro_use] extern crate rocket;
|
||||
#[macro_use]
|
||||
extern crate rocket;
|
||||
mod agency;
|
||||
mod db;
|
||||
mod whitelist;
|
||||
|
||||
use crate::db::{check_whitelist, save_query};
|
||||
use log::error;
|
||||
use querying::resolver::Resolver;
|
||||
use querying::target::Target;
|
||||
use querying::{Check, CheckError, CheckVerdict, Checker};
|
||||
use rocket::fairing::AdHoc;
|
||||
use rocket::fs::FileServer;
|
||||
use rocket::http::Status;
|
||||
use rocket::response::content::RawJavaScript;
|
||||
use rocket::tokio::sync::RwLock;
|
||||
use rocket::tokio::time;
|
||||
use rocket::{fairing, tokio, Build, Request, Rocket, State};
|
||||
use rocket_cache_response::CacheResponse;
|
||||
use rocket_client_addr::ClientRealAddr;
|
||||
use rocket_db_pools::{Connection, Database};
|
||||
use rocket_dyn_templates::{context, Metadata, Template};
|
||||
use serde::Serialize;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use rocket::fs::FileServer;
|
||||
use rocket::http::Status;
|
||||
use rocket::{tokio, Request, State};
|
||||
use rocket::response::content::RawJavaScript;
|
||||
use rocket_dyn_templates::{context, Metadata, Template};
|
||||
use serde::Serialize;
|
||||
use log::error;
|
||||
use rocket::tokio::sync::RwLock;
|
||||
use rocket::tokio::time;
|
||||
use rocket_cache_response::CacheResponse;
|
||||
use querying::{Check, CheckError, CheckVerdict, Checker};
|
||||
use querying::resolver::Resolver;
|
||||
use querying::target::Target;
|
||||
use sqlx::types::Uuid;
|
||||
|
||||
#[derive(rocket_db_pools::Database)]
|
||||
#[database("cheburcheck")]
|
||||
struct Db(sqlx::PgPool);
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct GlobalContext {
|
||||
@@ -24,19 +37,24 @@ struct GlobalContext {
|
||||
|
||||
impl GlobalContext {
|
||||
fn new() -> Self {
|
||||
GlobalContext { version: env!("CARGO_PKG_VERSION") }
|
||||
GlobalContext {
|
||||
version: env!("CARGO_PKG_VERSION"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[get("/")]
|
||||
async fn index(checker: &State<Arc<RwLock<Checker>>>) -> Template {
|
||||
let checker_ref = checker.read().await;
|
||||
Template::render("index", context! {
|
||||
global: GlobalContext::new(),
|
||||
domain_count: format_number(checker_ref.total_domains().await),
|
||||
v4_count: format_number(checker_ref.total_v4s().await),
|
||||
last_update: checker_ref.last_update(),
|
||||
})
|
||||
Template::render(
|
||||
"index",
|
||||
context! {
|
||||
global: GlobalContext::new(),
|
||||
domain_count: format_number(checker_ref.total_domains().await),
|
||||
v4_count: format_number(checker_ref.total_v4s().await),
|
||||
last_update: checker_ref.last_update(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[get("/kb/<page>")]
|
||||
@@ -46,9 +64,12 @@ fn page(metadata: Metadata, page: &str) -> Option<Template> {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(Template::render(page, context! {
|
||||
global: GlobalContext::new(),
|
||||
}))
|
||||
Some(Template::render(
|
||||
page,
|
||||
context! {
|
||||
global: GlobalContext::new(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
#[get("/healthcheck")]
|
||||
@@ -60,32 +81,86 @@ async fn healthcheck(checker: &State<Arc<RwLock<Checker>>>) -> (Status, String)
|
||||
}
|
||||
}
|
||||
|
||||
#[post("/feedback/<uuid>/<works>")]
|
||||
async fn feedback(uuid: &str, works: bool, mut db: Connection<Db>, addr: &ClientRealAddr) -> Result<(), Status> {
|
||||
sqlx::query!(
|
||||
"INSERT INTO human_reports (id, source_ip, works) VALUES ($1, $2, $3)",
|
||||
Uuid::try_parse(uuid).map_err(|_| Status::BadRequest)?,
|
||||
addr.ip.to_string(),
|
||||
works
|
||||
).execute(&mut **db).await.map_err(|_| Status::InternalServerError)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[get("/check?<target>")]
|
||||
async fn check(target: &str, checker: &State<Arc<RwLock<Checker>>>) -> Result<Template, Status> {
|
||||
async fn check(
|
||||
target: &str,
|
||||
checker: &State<Arc<RwLock<Checker>>>,
|
||||
addr: &ClientRealAddr,
|
||||
mut db: Connection<Db>,
|
||||
) -> Result<Template, Status> {
|
||||
let target = Target::from(target);
|
||||
match checker.read().await.check(target.clone()).await {
|
||||
Err(CheckError::NotFound) =>
|
||||
Ok(Template::render("empty", context! {
|
||||
global: GlobalContext::new(),
|
||||
target: target.to_query(),
|
||||
target_type: target.readable_type(),
|
||||
})),
|
||||
Ok(Check { verdict: CheckVerdict::Clear, geo, ips }) =>
|
||||
Ok(Template::render("result", context! {
|
||||
let check = checker.read().await.check(target.clone()).await;
|
||||
let id = if let Ok(check) = &check {
|
||||
match save_query(&mut db, &target, check, addr, checker.read().await).await {
|
||||
Ok(id) => Some(id.to_string()),
|
||||
Err(e) => {
|
||||
warn!("Failed to save check: {:?}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let whitelist = if let Target::Domain(domain) = &target {
|
||||
check_whitelist(domain, &mut db)
|
||||
.await
|
||||
.map_err(|_| Status::InternalServerError)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
match check {
|
||||
Err(CheckError::NotFound) => Ok(Template::render(
|
||||
"empty",
|
||||
context! {
|
||||
global: GlobalContext::new(),
|
||||
target: target.to_query(),
|
||||
target_type: target.readable_type(),
|
||||
},
|
||||
)),
|
||||
Ok(Check {
|
||||
verdict: CheckVerdict::Clear,
|
||||
geo,
|
||||
ips,
|
||||
}) => Ok(Template::render(
|
||||
"result",
|
||||
context! {
|
||||
id,
|
||||
global: GlobalContext::new(),
|
||||
found: false,
|
||||
target: target.to_query(),
|
||||
target_type: target.readable_type(),
|
||||
whitelist,
|
||||
ips,
|
||||
geo,
|
||||
})),
|
||||
},
|
||||
)),
|
||||
Ok(Check {
|
||||
verdict: CheckVerdict::Blocked {
|
||||
rkn_domain,
|
||||
rkn_subnets,
|
||||
cdn_provider_subnets
|
||||
}, geo, ips }) =>
|
||||
Ok(Template::render("result", context! {
|
||||
verdict:
|
||||
CheckVerdict::Blocked {
|
||||
rkn_domain,
|
||||
rkn_subnets,
|
||||
cdn_provider_subnets,
|
||||
},
|
||||
geo,
|
||||
ips,
|
||||
}) => Ok(Template::render(
|
||||
"result",
|
||||
context! {
|
||||
id,
|
||||
global: GlobalContext::new(),
|
||||
found: true,
|
||||
domain: rkn_domain,
|
||||
@@ -95,9 +170,11 @@ async fn check(target: &str, checker: &State<Arc<RwLock<Checker>>>) -> Result<Te
|
||||
.collect::<Vec<_>>(),
|
||||
target: target.to_query(),
|
||||
target_type: target.readable_type(),
|
||||
whitelist,
|
||||
ips,
|
||||
geo,
|
||||
})),
|
||||
},
|
||||
)),
|
||||
Err(e) => {
|
||||
error!("check failed {:?}", e);
|
||||
Err(Status::InternalServerError)
|
||||
@@ -107,14 +184,17 @@ async fn check(target: &str, checker: &State<Arc<RwLock<Checker>>>) -> Result<Te
|
||||
|
||||
#[catch(default)]
|
||||
fn default(status: Status, _req: &Request) -> Template {
|
||||
Template::render("error", context! {
|
||||
global: GlobalContext::new(),
|
||||
status: status.code,
|
||||
reason: status.reason_lossy(),
|
||||
})
|
||||
Template::render(
|
||||
"error",
|
||||
context! {
|
||||
global: GlobalContext::new(),
|
||||
status: status.code,
|
||||
reason: status.reason_lossy(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[rocket::get("/vendor/lucide.js")]
|
||||
#[rocket::get("/lucide.js")]
|
||||
fn lucide() -> CacheResponse<RawJavaScript<&'static [u8]>> {
|
||||
CacheResponse::Public {
|
||||
responder: RawJavaScript(include_bytes!(concat!(env!("OUT_DIR"), "/lucide.js"))),
|
||||
@@ -122,9 +202,26 @@ fn lucide() -> CacheResponse<RawJavaScript<&'static [u8]>> {
|
||||
must_revalidate: false,
|
||||
}
|
||||
}
|
||||
#[rocket::get("/chart.js")]
|
||||
fn chartjs() -> CacheResponse<RawJavaScript<&'static [u8]>> {
|
||||
CacheResponse::Public {
|
||||
responder: RawJavaScript(include_bytes!(concat!(env!("OUT_DIR"), "/chart.js"))),
|
||||
max_age: 604800,
|
||||
must_revalidate: false,
|
||||
}
|
||||
}
|
||||
#[rocket::get("/chartjs-plugin-datalabels.js")]
|
||||
fn chartjs_datalabels() -> CacheResponse<RawJavaScript<&'static [u8]>> {
|
||||
CacheResponse::Public {
|
||||
responder: RawJavaScript(include_bytes!(concat!(env!("OUT_DIR"), "/chartjs-plugin-datalabels.js"))),
|
||||
max_age: 604800,
|
||||
must_revalidate: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn format_number(number: usize) -> String {
|
||||
number.to_string()
|
||||
number
|
||||
.to_string()
|
||||
.as_bytes()
|
||||
.rchunks(3)
|
||||
.rev()
|
||||
@@ -134,12 +231,31 @@ fn format_number(number: usize) -> String {
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
async fn run_migrations(rocket: Rocket<Build>) -> fairing::Result {
|
||||
match Db::fetch(&rocket) {
|
||||
Some(db) => match sqlx::migrate!("./migrations").run(&**db).await {
|
||||
Ok(_) => Ok(rocket),
|
||||
Err(e) => {
|
||||
error!("Failed to run database migrations: {}", e);
|
||||
Err(rocket)
|
||||
}
|
||||
},
|
||||
None => Err(rocket),
|
||||
}
|
||||
}
|
||||
|
||||
#[launch]
|
||||
async fn rocket() -> _ {
|
||||
env_logger::builder().filter_level(log::LevelFilter::Info).init();
|
||||
env_logger::builder()
|
||||
.filter_level(log::LevelFilter::Info)
|
||||
.init();
|
||||
|
||||
let mut interval = time::interval(Duration::from_secs(std::env::var("DATABASE_INTERVAL_SECONDS")
|
||||
.unwrap_or("21600".to_string()).parse().unwrap()));
|
||||
let mut interval = time::interval(Duration::from_secs(
|
||||
std::env::var("DATABASE_INTERVAL_SECONDS")
|
||||
.unwrap_or("21600".to_string())
|
||||
.parse()
|
||||
.unwrap(),
|
||||
));
|
||||
|
||||
let checker = Arc::new(RwLock::new(Checker::new().await));
|
||||
|
||||
@@ -154,10 +270,20 @@ async fn rocket() -> _ {
|
||||
}
|
||||
});
|
||||
|
||||
rocket::build()
|
||||
let figment = rocket::Config::figment().merge((
|
||||
"databases.cheburcheck.url",
|
||||
dotenvy::var("DATABASE_URL").expect("DATABASE_URL must be set"),
|
||||
));
|
||||
|
||||
rocket::custom(figment)
|
||||
.manage(Resolver::new().await)
|
||||
.manage(checker)
|
||||
.mount("/", routes![index, lucide, check, healthcheck, page])
|
||||
.attach(Db::init())
|
||||
.attach(AdHoc::try_on_ignite("SQLx Migrations", run_migrations))
|
||||
.mount("/", routes![index, check, healthcheck, page, feedback])
|
||||
.mount("/vendor", routes![lucide, chartjs, chartjs_datalabels])
|
||||
.mount("/agency", routes![agency::upload_report])
|
||||
.mount("/whitelist", routes![whitelist::histogram, whitelist::export_csv])
|
||||
.register("/", catchers![default])
|
||||
.mount("/", FileServer::from(PathBuf::from("static")))
|
||||
.attach(Template::fairing())
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
use crate::Db;
|
||||
use rocket::futures::StreamExt;
|
||||
use rocket::http::{ContentType, Status};
|
||||
use rocket::request::FromParam;
|
||||
use rocket_cache_response::CacheResponse;
|
||||
use rocket_db_pools::Connection;
|
||||
use std::io;
|
||||
use rocket::serde::json::Json;
|
||||
use crate::db::{collect_histogram, WhitelistHistogramBin};
|
||||
|
||||
enum ExportType {
|
||||
Full,
|
||||
Domains,
|
||||
}
|
||||
|
||||
impl<'r> FromParam<'r> for ExportType {
|
||||
type Error = &'r str;
|
||||
|
||||
fn from_param(param: &'r str) -> Result<Self, Self::Error> {
|
||||
match param {
|
||||
"full.csv" => Ok(ExportType::Full),
|
||||
"domains.csv" => Ok(ExportType::Domains),
|
||||
_ => Err(param),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[get("/<export_type>")]
|
||||
pub async fn export_csv(
|
||||
export_type: ExportType,
|
||||
mut db: Connection<Db>,
|
||||
) -> Result<CacheResponse<(ContentType, Vec<u8>)>, io::Error> {
|
||||
let query = match export_type {
|
||||
ExportType::Full => {
|
||||
"COPY (SELECT domain, rank, last_ok FROM whitelist) TO STDOUT WITH (FORMAT CSV, HEADER, ENCODING 'UTF8')"
|
||||
}
|
||||
ExportType::Domains => {
|
||||
"COPY (SELECT domain FROM whitelist) TO STDOUT WITH (FORMAT CSV, ENCODING 'UTF8')"
|
||||
}
|
||||
};
|
||||
|
||||
let mut stream = db
|
||||
.copy_out_raw(query)
|
||||
.await
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
|
||||
|
||||
let mut data = Vec::default();
|
||||
|
||||
while let Some(bytes_result) = stream.next().await {
|
||||
let bytes = bytes_result.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
|
||||
data.extend(bytes)
|
||||
}
|
||||
|
||||
Ok(CacheResponse::Public {
|
||||
responder: (ContentType::CSV, data),
|
||||
max_age: 86400,
|
||||
must_revalidate: false,
|
||||
})
|
||||
}
|
||||
|
||||
#[get("/histogram?<filter>")]
|
||||
pub async fn histogram(mut db: Connection<Db>, filter: Option<bool>) -> Result<Json<Vec<WhitelistHistogramBin>>, Status> {
|
||||
Ok(Json(collect_histogram(&mut db, 50, 100_000, filter.is_some()).await
|
||||
.map_err(|e| Status::InternalServerError)?))
|
||||
}
|
||||
+106
-1
@@ -15,6 +15,10 @@
|
||||
--red-bg-subtle: rgba(69, 10, 10, 0.2);
|
||||
--red-border: rgba(127, 29, 29, 0.3);
|
||||
|
||||
--yellow-color: #f0b100; /* red-500 */
|
||||
--yellow-bg-subtle: rgba(69, 64, 10, 0.2);
|
||||
--yellow-border: rgba(127, 114, 29, 0.3);
|
||||
|
||||
--green-color: #22c55e; /* green-500 */
|
||||
--green-bg-subtle: rgba(5, 46, 22, 0.2);
|
||||
--green-border: rgba(20, 83, 45, 0.3);
|
||||
@@ -22,6 +26,11 @@
|
||||
--font-mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar { width: 8px; }
|
||||
::-webkit-scrollbar-track { background: #09090b; }
|
||||
::-webkit-scrollbar-thumb { background: var(--border-color); }
|
||||
::-webkit-scrollbar-thumb:hover { background: var(--input-border); }
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
margin: 0;
|
||||
@@ -262,7 +271,19 @@ h1.error-code {
|
||||
.clean-theme .alert-box {
|
||||
border: 1px solid var(--green-border);
|
||||
background: var(--green-bg-subtle);
|
||||
color: #4ade80;
|
||||
color: var(--green-color);
|
||||
}
|
||||
|
||||
.whitelist-theme {
|
||||
border-color: var(--yellow-border);
|
||||
background-color: rgba(46, 45, 5, 0.1);
|
||||
}
|
||||
.whitelist-theme h2, .whitelist-theme .status-icon, .whitelist-theme .subheading { color: var(--yellow-color); }
|
||||
.whitelist-theme .reason-header { border-bottom: 1px solid var(--yellow-border); color: var(--yellow-color); }
|
||||
.whitelist-theme .alert-box {
|
||||
border: 1px solid var(--yellow-border);
|
||||
background: var(--yellow-bg-subtle);
|
||||
color: var(--yellow-color);
|
||||
}
|
||||
|
||||
.result-header {
|
||||
@@ -335,6 +356,12 @@ h1.error-code {
|
||||
text-align: right;
|
||||
}
|
||||
.row-value.alert { color: var(--red-color); }
|
||||
.row-value.success { color: var(--green-color); }
|
||||
|
||||
.hint {
|
||||
text-decoration: underline;
|
||||
text-decoration-style: dotted;
|
||||
}
|
||||
|
||||
.alert-box {
|
||||
padding: 1rem;
|
||||
@@ -413,6 +440,84 @@ h1.error-code {
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.code {
|
||||
border-left: 2px solid var(--border-color);
|
||||
padding-left: 1em;
|
||||
font-family: inherit;
|
||||
overflow-x: scroll;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.user-feedback-section {
|
||||
margin-top: 1.5rem;
|
||||
padding-top: 1rem;
|
||||
border-top: 1px dashed var(--border-color);
|
||||
}
|
||||
|
||||
.feedback-prompt {
|
||||
color: var(--text-muted);
|
||||
font-size: 0.875rem;
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
.feedback-buttons {
|
||||
display: flex;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.feedback-btn {
|
||||
padding: 0.5rem 1rem;
|
||||
border: 1px solid transparent;
|
||||
font-weight: bold;
|
||||
font-size: 0.875rem;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
transition: all 0.2s;
|
||||
flex-grow: 1;
|
||||
justify-content: center;
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
.blocked-theme .feedback-works {
|
||||
background-color: var(--green-color);
|
||||
color: var(--bg-color);
|
||||
}
|
||||
.blocked-theme .feedback-not-works, .whitelist-theme .feedback-not-works {
|
||||
background-color: transparent;
|
||||
color: var(--red-color);
|
||||
border: 1px solid var(--red-color);
|
||||
}
|
||||
.blocked-theme .feedback-works:hover { background-color: #34d399; }
|
||||
.blocked-theme .feedback-not-works:hover { background-color: rgba(239, 68, 68, 0.1); }
|
||||
|
||||
.clean-theme .feedback-works, .whitelist-theme .feedback-works {
|
||||
background-color: transparent;
|
||||
color: var(--green-color);
|
||||
border: 1px solid var(--green-color);
|
||||
}
|
||||
.clean-theme .feedback-not-works {
|
||||
background-color: var(--red-color);
|
||||
color: var(--text-bright);
|
||||
}
|
||||
.clean-theme .feedback-works:hover { background-color: rgba(34, 197, 94, 0.1); }
|
||||
.clean-theme .feedback-not-works:hover { background-color: #f87171; }
|
||||
|
||||
.feedback-status {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
font-size: 0.875rem;
|
||||
padding: 0.5rem 0;
|
||||
color: #d4d4d4;
|
||||
}
|
||||
.blocked-theme .feedback-status { color: #f87171; }
|
||||
.clean-theme .feedback-status { color: var(--green-color); }
|
||||
|
||||
.hidden {
|
||||
display: none;
|
||||
}
|
||||
|
||||
@media (min-width: 640px) {
|
||||
body { padding: 2rem; }
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
{% macro head() %}
|
||||
<script src="/vendor/chart.js"></script>
|
||||
<script src="/vendor/chartjs-plugin-datalabels.js"></script>
|
||||
{% endmacro %}
|
||||
|
||||
{% macro histogram(id="histogram",
|
||||
endpoint="/whitelist/histogram",
|
||||
data_label="Количество ресурсов",
|
||||
x_label="Ранг домена",
|
||||
y_label="Количество ресурсов") %}
|
||||
<canvas id="{{ id }}"></canvas>
|
||||
|
||||
<style>
|
||||
#{{ id }} {
|
||||
max-height: 480px;
|
||||
}
|
||||
</style>
|
||||
|
||||
<script>
|
||||
(async function () {
|
||||
function formatNumber(num) {
|
||||
return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
||||
}
|
||||
const TEXT_LIGHT = '#d4d4d4';
|
||||
const RED_ACCENT = '#ef4444';
|
||||
const AXIS_COLOR = '#404040';
|
||||
const GRID_COLOR = 'rgba(64, 64, 64, 0.2)';
|
||||
|
||||
const ctx = document.getElementById('{{ id }}').getContext('2d');
|
||||
const response = await fetch("{{ endpoint | safe }}");
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
const data = await response.json();
|
||||
|
||||
const labels = data.map(item =>
|
||||
`${formatNumber(Math.round(item.bin_min_rank))}-${formatNumber(Math.round(item.bin_max_rank))}`
|
||||
);
|
||||
const counts = data.map(item => item.count);
|
||||
|
||||
Chart.register(ChartDataLabels);
|
||||
|
||||
new Chart(ctx, {
|
||||
type: 'bar',
|
||||
data: {
|
||||
labels: labels,
|
||||
datasets: [{
|
||||
label: '{{ data_label }}',
|
||||
data: counts,
|
||||
backgroundColor: RED_ACCENT,
|
||||
borderColor: RED_ACCENT,
|
||||
borderWidth: 1,
|
||||
borderRadius: 6,
|
||||
hoverBackgroundColor: `${RED_ACCENT}cc`,
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
datalabels: {
|
||||
anchor: 'end',
|
||||
align: 'top',
|
||||
offset: 8,
|
||||
color: TEXT_LIGHT,
|
||||
font: {
|
||||
weight: 'bold',
|
||||
size: 10
|
||||
},
|
||||
formatter: (value) => value > 0 ? formatNumber(value) : '',
|
||||
},
|
||||
legend: {
|
||||
display: false,
|
||||
},
|
||||
tooltip: {
|
||||
callbacks: {
|
||||
label: function(context) {
|
||||
let label = context.dataset.label || '';
|
||||
if (label) {
|
||||
label += ': ';
|
||||
}
|
||||
if (context.parsed.y !== null) {
|
||||
label += formatNumber(context.parsed.y);
|
||||
}
|
||||
return label;
|
||||
}
|
||||
},
|
||||
backgroundColor: 'rgba(30, 30, 30, 0.9)',
|
||||
titleColor: TEXT_LIGHT,
|
||||
bodyColor: TEXT_LIGHT,
|
||||
}
|
||||
},
|
||||
scales: {
|
||||
x: {
|
||||
title: {
|
||||
display: true,
|
||||
text: '{{ x_label }}',
|
||||
color: TEXT_LIGHT
|
||||
},
|
||||
ticks: {
|
||||
color: TEXT_LIGHT,
|
||||
maxRotation: 45,
|
||||
minRotation: 45
|
||||
},
|
||||
grid: {
|
||||
color: GRID_COLOR,
|
||||
drawBorder: false
|
||||
},
|
||||
border: {
|
||||
color: AXIS_COLOR
|
||||
}
|
||||
},
|
||||
y: {
|
||||
type: 'linear',
|
||||
title: {
|
||||
display: true,
|
||||
text: '{{ y_label }}',
|
||||
color: TEXT_LIGHT
|
||||
},
|
||||
ticks: {
|
||||
color: TEXT_LIGHT,
|
||||
callback: () => ''
|
||||
},
|
||||
grid: {
|
||||
color: GRID_COLOR,
|
||||
drawBorder: false
|
||||
},
|
||||
border: {
|
||||
color: AXIS_COLOR
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{% endmacro metadata %}
|
||||
@@ -33,6 +33,13 @@
|
||||
и <a href="https://antifilter.network/">antifilter.network</a>.
|
||||
</p>
|
||||
|
||||
{{ typography::heading(title="Что такое «белый список»?") }}
|
||||
<p>
|
||||
РКН формирует свой белый список доменов, которые доступны,
|
||||
даже если их адреса находятся в списке заблокированных подсетей.
|
||||
</p>
|
||||
<a href="/kb/whitelist">Подробнее...</a>
|
||||
|
||||
{{ typography::heading(title='Мне написало, что сайт "Заблокирован", но у меня все работает!') }}
|
||||
<p>
|
||||
Мы не можем с точностью утверждать, заблокирован сайт или нет.
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
{% extends 'page' %}
|
||||
{% import 'page' as page %}
|
||||
{% import 'histogram' as histogram %}
|
||||
|
||||
{% block head %}
|
||||
{{ super() }}
|
||||
{{ histogram::head() }}
|
||||
{% endblock head %}
|
||||
|
||||
{% block metadata %}
|
||||
{{ page::metadata(title="Белые списки", description="Анализ данных из белых списков блокировок", keywords="белые списки, whitelist", url="/kb/whitelist") }}
|
||||
{% endblock metadata %}
|
||||
|
||||
{% block page_text %}
|
||||
<h1>Белые списки доменов</h1>
|
||||
<p>
|
||||
Российские операторы связи начали применять новый тип блокировок CDN,
|
||||
при котором загрузка контента обрывается после передачи примерно 16–20 килобайт данных,
|
||||
из‑за чего большинство сайтов на Cloudflare и других зарубежных платформах становятся
|
||||
практически неработоспособными для пользователей в России.
|
||||
Одновременно сохраняются и развиваются белые списки с популярными и «социально значимыми» доменами,
|
||||
которые старательно выводятся из‑под подобных ограничений.
|
||||
</p>
|
||||
<p>
|
||||
Суть схемы в том, что соединение технически устанавливается, первые килобайты HTML,
|
||||
CSS или скриптов передаются, но затем трафик режется или соединение принудительно сбрасывается на уровне
|
||||
российских операторов, обычно после примерно 16 килобайт (10–14 пакетов в зависимости от протокола).
|
||||
Этого объёма достаточно, чтобы страница начала загружаться и создавалась иллюзия доступности,
|
||||
но ключевые части сайта (основной HTML, JS‑бандлы, стили, API‑запросы) не проходят, и сервис фактически
|
||||
перестаёт работать.
|
||||
</p>
|
||||
<pre class="code">$ curl -k https://cheburcheck.ru/100MB.bin -o/dev/null -r 0-65536 --resolve cheburcheck.ru:443:5.78.7.195 --max-time 5
|
||||
% Total % Received % Xferd Average Speed Time Time Time Current
|
||||
Dload Upload Total Spent Left Speed
|
||||
24 65537 24 16101 0 0 3220 0 0:00:20 0:00:04 0:00:16 0
|
||||
curl: (28) Operation timed out after 5000 milliseconds with 16101 out of 65537 bytes received</pre>
|
||||
<p>
|
||||
Определение домена и выбор, «разрешить или задушить» запрос, в текущей схеме делается на связке DPI‑фильтров.
|
||||
По умолчанию, все HTTP(s) запросы на адреса из подсетей хостеров блокируются. Однако, запросы
|
||||
к некоторым доменам из тех же подсетей проходят нормально — они обнаруживаются по открытому расширению
|
||||
TLS SNI, либо по заголовку Host (в случае с plain http).
|
||||
</p>
|
||||
<p>
|
||||
Проверить нахождение домена в белом списке можно с помощью простого HTTPS-запроса с подменой SNI на адрес из
|
||||
заблокированных диапазонов:
|
||||
</p>
|
||||
<pre class="code">$ curl -k https://ok.ru/100MB.bin -o/dev/null -r 0-65536 --resolve ok.ru:443:5.78.7.195
|
||||
% Total % Received % Xferd Average Speed Time Time Time Current
|
||||
Dload Upload Total Spent Left Speed
|
||||
100 65537 100 65537 0 0 55226 0 0:00:01 0:00:01 --:--:-- 55258</pre>
|
||||
<p>
|
||||
Также, стоит отметить, что домены белого списка добавляются с маской по поддоменам:
|
||||
</p>
|
||||
<pre class="code">$ curl -k https://cheburcheck.ok.ru/100MB.bin -o/dev/null -r 0-65536 --resolve cheburcheck.ok.ru:443:
|
||||
5.78.7.195
|
||||
% Total % Received % Xferd Average Speed Time Time Time Current
|
||||
Dload Upload Total Spent Left Speed
|
||||
100 65537 100 65537 0 0 53192 0 0:00:01 0:00:01 --:--:-- 53195</pre>
|
||||
|
||||
{{ typography::heading(title="Автоматическое сканирование") }}
|
||||
<p>
|
||||
Используя методы, указанные выше, мы разработали инструмент для автоматического сканирования и анализа
|
||||
блокировок
|
||||
<b><a href="https://github.com/LowderPlay/cheburcheck/tree/master/reporter">Cheburcheck Reporter</a></b>.
|
||||
Он позволяет сканировать большое количество доменов
|
||||
за короткий срок за счет параллелизации запросов.
|
||||
</p>
|
||||
<p>
|
||||
Исходя из анализа 1,000,000 доменов из рейтинга
|
||||
<a href="https://tranco-list.eu/list/2NPQ9">Tranco list от 26 ноября 2025</a>,
|
||||
в белом списке содержится около 1000 доменов. Все* домены из списка входят в топ-100,000 рейтинга.
|
||||
</p>
|
||||
<i>
|
||||
* — Мы не включаем в это число домены из зоны .co.uk, так как по какой-то причине, они все находятся в белом списке.
|
||||
</i>
|
||||
|
||||
<i class="caption">Гистограмма количества доменов относительно их положения в рейтинге</i>
|
||||
{{ histogram::histogram(endpoint="/whitelist/histogram") }}
|
||||
<br>
|
||||
<i class="caption">Гистограмма без учета доменов .co.uk</i>
|
||||
{{ histogram::histogram(id="filtered", endpoint="/whitelist/histogram?filter") }}
|
||||
|
||||
{{ typography::heading(title="Скачать списки") }}
|
||||
<p>
|
||||
Мы публикуем результаты наших сканирований в виде CSV-файлов:
|
||||
</p>
|
||||
<div>
|
||||
<i data-lucide="file-symlink" width="16" height="16"></i> <a href="/whitelist/full.csv">Полный список (CSV)</a>
|
||||
</div>
|
||||
<div>
|
||||
<i data-lucide="file-symlink" width="16" height="16"></i> <a href="/whitelist/domains.csv">Только домены (CSV)</a>
|
||||
</div>
|
||||
{% endblock page_text %}
|
||||
@@ -3,16 +3,27 @@
|
||||
{% block content %}
|
||||
{% include 'search-form' %}
|
||||
|
||||
<div class="result-panel {% if found %}blocked-theme{% else %}clean-theme{% endif %}">
|
||||
{% set warning = whitelist and not domain %}
|
||||
|
||||
<div class="result-panel {% if warning %}whitelist-theme{% elif found %}blocked-theme{% else %}clean-theme{% endif %}">
|
||||
<div class="result-header">
|
||||
<div class="icon-box status-icon">
|
||||
{% if found %}
|
||||
{% if whitelist %}
|
||||
<i data-lucide="shield-alert" width="32" height="32"></i>
|
||||
{% elif blocked %}
|
||||
<i data-lucide="shield-x" width="32" height="32"></i>
|
||||
{% else %}
|
||||
<i data-lucide="shield-check" width="32" height="32"></i>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if found %}
|
||||
{% if warning %}
|
||||
<div>
|
||||
<h2>Белый список</h2>
|
||||
<p class="subheading text-sm">
|
||||
Ресурс находится в белом списке
|
||||
</p>
|
||||
</div>
|
||||
{% elif found %}
|
||||
<div>
|
||||
<h2>Заблокирован</h2>
|
||||
<p class="subheading text-sm">Ресурс был найден в списках блокировок</p>
|
||||
@@ -44,7 +55,7 @@
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="row-label">Хостинг / ISP</span>
|
||||
<span class="row-value">{{ geo.organisation }}</span>
|
||||
<span class="row-value">{% if geo.organisation %}{{ geo.organisation }}{% else %}-{% endif %}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="row-label">Локация</span>
|
||||
@@ -52,7 +63,7 @@
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="row-label">ASN</span>
|
||||
<span class="row-value">{{ geo.asn }}</span>
|
||||
<span class="row-value">{% if geo.asn %}{{ geo.asn }}{% else %}-{% endif %}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -78,6 +89,21 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% if whitelist %}
|
||||
<div class="detail-row">
|
||||
<a href="/kb/whitelist" class="row-label">Белый список (?)</a>
|
||||
<span class="row-value success">
|
||||
НАЙДЕН - <span class="hint"
|
||||
title="Дата последнего сканирования, когда данный домен был найден в белом списке">
|
||||
<script>
|
||||
document.write(new Date("{{ whitelist.last_ok }}").toLocaleDateString());
|
||||
</script>
|
||||
</span>
|
||||
</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="detail-row">
|
||||
<span class="row-label">Реестр РКН</span>
|
||||
|
||||
@@ -106,7 +132,36 @@
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="user-feedback-section">
|
||||
<p class="feedback-prompt">У вас работает этот ресурс?</p>
|
||||
<div class="feedback-buttons">
|
||||
<button class="feedback-btn feedback-works" onclick="sendFeedback(true)">
|
||||
<i data-lucide="thumbs-up" width="16" height="16"></i>
|
||||
Работает
|
||||
</button>
|
||||
<button class="feedback-btn feedback-not-works" onclick="sendFeedback(false)">
|
||||
<i data-lucide="thumbs-down" width="16" height="16"></i>
|
||||
Не работает
|
||||
</button>
|
||||
</div>
|
||||
<div class="feedback-status hidden">
|
||||
<i data-lucide="thumbs-up" width="16" height="16"></i>
|
||||
<span>Спасибо за ваш отзыв!</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function sendFeedback(works) {
|
||||
document.querySelector('.feedback-buttons').classList.add('hidden');
|
||||
document.querySelector('.feedback-prompt').classList.add('hidden');
|
||||
document.querySelector('.feedback-status').classList.remove('hidden');
|
||||
|
||||
fetch(`/feedback/{{ id }}/${works}`, {
|
||||
method: 'POST',
|
||||
});
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
{% endblock content %}
|
||||
|
||||
Reference in New Issue
Block a user