diff --git a/.dockerignore b/.dockerignore index 2f7896d..2d2ecd6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1 @@ -target/ +.git/ diff --git a/.gitignore b/.gitignore index d85258a..c403c34 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ /target -/databases +.idea/ diff --git a/Cargo.lock b/Cargo.lock index 1dd2cf2..b4e1feb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,29 +169,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "blocklist-check" -version = "0.1.3" -dependencies = [ - "chrono", - "csv", - "env_logger", - "flate2", - "hickory-resolver", - "ipnet", - "ipnet-trie", - "log", - "maxminddb", - "reqwest", - "rocket", - "rocket-cache-response", - "rocket_dyn_templates", - "serde", - "tar", - "trie-rs", - "url", -] - [[package]] name = "bstr" version = "1.12.1" @@ -222,9 +199,9 @@ checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] name = "cc" -version = "1.2.46" +version = "1.2.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97463e1064cb1b1c1384ad0a0b9c8abd0988e2a91f52606c80ef14aadb63e36" +checksum = "cd405d82c84ff7f35739f175f67d8b9fb7687a0e84ccdc78bd3568839827cf07" dependencies = [ "find-msvc-tools", "shlex", @@ -284,6 +261,19 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "console" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.61.2", +] + [[package]] name = "cookie" version = "0.18.1" @@ -371,9 +361,9 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "typenum", @@ -481,6 +471,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -824,7 +820,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.3.1", + "http 1.4.0", "indexmap", "slab", "tokio", @@ -834,9 +830,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" [[package]] name = "heck" @@ -865,7 +861,7 @@ dependencies = [ "futures-io", "futures-util", "h2 0.4.12", - "http 1.3.1", + "http 1.4.0", "idna", "ipnet", "once_cell", @@ -918,12 +914,11 @@ dependencies = [ [[package]] name = "http" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", - "fnv", "itoa", ] @@ -945,7 +940,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.3.1", + "http 1.4.0", ] [[package]] @@ -956,7 +951,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "pin-project-lite", ] @@ -1017,7 +1012,7 @@ dependencies = [ "futures-channel", "futures-core", "h2 0.4.12", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "httparse", "itoa", @@ -1034,7 +1029,7 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http 1.3.1", + "http 1.4.0", "hyper 1.8.1", "hyper-util", "rustls", @@ -1042,7 +1037,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.2", + "webpki-roots 1.0.4", ] [[package]] @@ -1072,7 +1067,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "hyper 1.8.1", "ipnet", @@ -1231,9 +1226,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", "hashbrown", @@ -1241,6 +1236,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indicatif" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9375e112e4b463ec1b1c6c011953545c65a30164fbab5b581df32b3abf0dcb88" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + [[package]] name = "inlinable_string" version = "0.1.15" @@ -1360,9 +1368,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.81" +version = "0.3.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" dependencies = [ "once_cell", "wasm-bindgen", @@ -1402,9 +1410,9 @@ checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libm" -version = "0.2.13" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9627da5196e5d8ed0b0495e61e518847578da83483c37288316d9b2e03a7f72" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" @@ -1568,7 +1576,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http 1.3.1", + "http 1.4.0", "httparse", "memchr", "mime", @@ -1780,20 +1788,19 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.7.15" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" +checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22" dependencies = [ "memchr", - "thiserror", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.15" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e" +checksum = "51f72981ade67b1ca6adc26ec221be9f463f2b5839c7508998daa17c23d94d7f" dependencies = [ "pest", "pest_generator", @@ -1801,9 +1808,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.15" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b" +checksum = "dee9efd8cdb50d719a80088b76f81aec7c41ed6d522ee750178f83883d271625" dependencies = [ "pest", "pest_meta", @@ -1814,11 +1821,10 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.15" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea" +checksum = "bf1d70880e76bdc13ba52eafa6239ce793d85c8e43896507e43dd8984ff05b82" dependencies = [ - "once_cell", "pest", "sha2", ] @@ -1950,6 +1956,28 @@ dependencies = [ "yansi", ] +[[package]] +name = "querying" +version = "0.1.0" +dependencies = [ + "async-trait", + "chrono", + "csv", + "futures-util", + "hickory-resolver", + "indicatif", + "ipnet", + "ipnet-trie", + "log", + "maxminddb", + "reqwest", + "serde", + "thiserror", + "tokio", + "trie-rs", + "url", +] + [[package]] name = "quinn" version = "0.11.9" @@ -1963,7 +1991,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.1", "thiserror", "tokio", "tracing", @@ -2000,9 +2028,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.1", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -2170,7 +2198,7 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.12", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "hyper 1.8.1", @@ -2202,7 +2230,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.2", + "webpki-roots 1.0.4", ] [[package]] @@ -2555,9 +2583,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.6" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" dependencies = [ "libc", ] @@ -2654,9 +2682,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.110" +version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", @@ -2685,20 +2713,20 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.10.0", "core-foundation", "system-configuration-sys", ] [[package]] name = "system-configuration-sys" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ "core-foundation-sys", "libc", @@ -2850,6 +2878,7 @@ dependencies = [ "bytes", "libc", "mio 1.1.0", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2 0.6.1", @@ -2970,14 +2999,14 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +checksum = "9cf146f99d442e8e68e585f5d798ccd3cad9a7835b917e09728880a862706456" dependencies = [ "bitflags 2.10.0", "bytes", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "iri-string", "pin-project-lite", @@ -3117,12 +3146,24 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + [[package]] name = "untrusted" version = "0.9.0" @@ -3218,9 +3259,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.104" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" dependencies = [ "cfg-if", "once_cell", @@ -3229,25 +3270,11 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - [[package]] name = "wasm-bindgen-futures" -version = "0.4.54" +version = "0.4.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e038d41e478cc73bae0ff9b36c60cff1c98b8f38f8d7e8061e79ee63608ac5c" +checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" dependencies = [ "cfg-if", "js-sys", @@ -3258,9 +3285,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.104" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3268,22 +3295,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.104" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" dependencies = [ + "bumpalo", "proc-macro2", "quote", "syn", - "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.104" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" dependencies = [ "unicode-ident", ] @@ -3303,9 +3330,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.81" +version = "0.3.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" +checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" dependencies = [ "js-sys", "wasm-bindgen", @@ -3327,18 +3354,34 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.2", + "webpki-roots 1.0.4", ] [[package]] name = "webpki-roots" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" dependencies = [ "rustls-pki-types", ] +[[package]] +name = "website" +version = "0.1.3" +dependencies = [ + "env_logger", + "flate2", + "log", + "querying", + "reqwest", + "rocket", + "rocket-cache-response", + "rocket_dyn_templates", + "serde", + "tar", +] + [[package]] name = "widestring" version = "1.2.1" @@ -3730,18 +3773,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.27" +version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +checksum = "4ea879c944afe8a2b25fef16bb4ba234f47c694565e97383b36f3a878219065c" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.27" +version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +checksum = "cf955aa904d6040f70dc8e9384444cb1030aed272ba3cb09bbc4ab9e7c1f34f5" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 01df8fb..963f006 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,27 +1,10 @@ -[package] -name = "blocklist-check" -version = "0.1.3" -edition = "2024" +[workspace] +members = [ "querying", + "website", +] +resolver = "3" -[dependencies] -hickory-resolver = { version = "0.26.0-alpha.1", features = ["tokio", "webpki-roots", "https-ring"] } -rocket = "0.5.1" -rocket_dyn_templates = { version = "0.2.0", features = ["tera"] } -serde = { version = "1", features = ["derive"] } -log = "0.4.28" -maxminddb = "0.26" -ipnet-trie = "0.3.0" -ipnet = "2.11.0" -csv = "1.4.0" -trie-rs = "0.4.2" -env_logger = "0.11.8" +[workspace.dependencies] +tokio = { version = "1.48.0", features = ["full"] } reqwest = { version = "0.12.24", features = ["stream", "rustls-tls-webpki-roots"] } -chrono = { version = "0.4.42", features = ["serde"] } -rocket-cache-response = "0.6.4" -url = "2.5.7" - -[build-dependencies] -reqwest = { version = "0.12", features = ["blocking", "json"] } -serde = { version = "1", features = ["derive"] } -tar = "0.4" -flate2 = "1.0" +log = "0.4.28" diff --git a/README.md b/README.md index c9a08ac..55340bc 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ - **Rust** — язык программирования, на котором написан весь бекенд. - **Rocket.rs** — веб-фреймворк для построения HTTP API и серверной логики. -- **DoH резолвер Cloudflare** — для разрешения DNS-запросов используется DNS-over-HTTPS через Cloudflare, что позволяет обойти локальные ограничения и получать актуальные данные. +- **DoH резолвер Quad9** — для разрешения DNS-запросов используется DNS-over-HTTPS, что позволяет обойти локальные ограничения и получать актуальные данные. - **Tera** — шаблонизатор для генерации HTML страниц, используется для фронтенда. - **lucide** — библиотека иконок, применяемая для визуального оформления интерфейса. diff --git a/Rocket.toml b/Rocket.toml new file mode 100644 index 0000000..c770e1e --- /dev/null +++ b/Rocket.toml @@ -0,0 +1,2 @@ +[default] +template_dir = "website/templates/" diff --git a/querying/Cargo.toml b/querying/Cargo.toml new file mode 100644 index 0000000..e787d03 --- /dev/null +++ b/querying/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "querying" +version = "0.1.0" +edition = "2024" + +[dependencies] +ipnet-trie = "0.3.0" +ipnet = "2.11.0" +serde = { version = "1", features = ["derive"] } +async-trait = "0.1.89" +trie-rs = "0.4.2" +tokio = { workspace = true } +reqwest = { workspace = true } +csv = "1.4.0" +maxminddb = "0.26.0" +log = { workspace = true } +hickory-resolver = { version = "0.26.0-alpha.1", features = ["tokio", "webpki-roots", "https-ring"] } +thiserror = "2.0.17" +url = "2.5.7" +indicatif = "0.18.3" +chrono = { version = "0.4.42", features = ["serde"] } +futures-util = "0.3.31" diff --git a/dist-domains.txt b/querying/dist-domains.txt similarity index 100% rename from dist-domains.txt rename to querying/dist-domains.txt diff --git a/src/geoip.rs b/querying/src/geoip.rs similarity index 83% rename from src/geoip.rs rename to querying/src/geoip.rs index b027cb0..9c38f21 100644 --- a/src/geoip.rs +++ b/querying/src/geoip.rs @@ -1,9 +1,10 @@ use std::{io}; use std::io::Error; use std::net::IpAddr; +use async_trait::async_trait; use maxminddb::{geoip2, MaxMindDbError}; use maxminddb::geoip2::{city, country, City, Country}; -use rocket::serde::Serialize; +use serde::Serialize; use crate::updater::{fetch_db, Updatable}; pub struct GeoIp { @@ -84,14 +85,14 @@ impl GeoIp { impl Updatable for GeoIp { type Base = (Vec, Vec, Vec); - async fn download() -> Result { - Ok((fetch_db(Self::get_url("GEO_ASN", "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-ASN.mmdb")).await?, - fetch_db(Self::get_url("GEO_COUNTRY", "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-Country.mmdb")).await?, - fetch_db(Self::get_url("GEO_CITY", "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb")).await?)) + async fn download() -> Result { + Ok((fetch_db(Self::get_url("GEO_ASN", "https://git.io/GeoLite2-ASN.mmdb")).await?, + fetch_db(Self::get_url("GEO_COUNTRY", "https://git.io/GeoLite2-Country.mmdb")).await?, + fetch_db(Self::get_url("GEO_CITY", "https://git.io/GeoLite2-City.mmdb")).await?)) } async fn install(&mut self, (asn, country, city): Self::Base) -> Result<(), Error> { self.update(asn, country, city) - .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) + .map_err(|e| Error::new(io::ErrorKind::Other, e)) } } diff --git a/querying/src/lib.rs b/querying/src/lib.rs new file mode 100644 index 0000000..658a300 --- /dev/null +++ b/querying/src/lib.rs @@ -0,0 +1,173 @@ +use std::collections::{HashMap, HashSet}; +use std::net::IpAddr; +use std::sync::Arc; +use chrono::{DateTime, Utc}; +use ipnet::IpNet; +use log::error; +use thiserror::Error; +use tokio::sync::{watch, RwLock}; +use crate::geoip::{GeoIp, IpInfo}; +use crate::lists::{CdnList, NetworkRecord, RuBlacklist}; +use crate::resolver::{ResolveError, Resolver}; +use crate::target::Target; +use crate::updater::Updatable; + +pub mod geoip; +pub mod lists; +pub mod resolver; +pub mod updater; +pub mod target; + +pub struct Checker { + rx: watch::Receiver>>, + tx: watch::Sender>>, + cdn_list: Arc>, + ru_blacklist: Arc>, + geo_ip: Arc>, + resolver: Resolver, +} + +pub struct Check { + pub verdict: CheckVerdict, + pub geo: IpInfo, + pub ips: Vec, +} + +pub enum CheckVerdict { + Clear, + Blocked { + rkn_domain: Option, + rkn_subnets: HashSet, + cdn_provider_subnets: HashMap>, + }, +} + +#[derive(Debug, Error)] +pub enum CheckError { + #[error("resolve error")] + ResolveError(#[from] ResolveError), + #[error("geoip error")] + GeoIpError, + #[error("domain not found")] + NotFound, +} + +impl Checker { + pub async fn new() -> Checker { + let (tx, rx) = watch::channel(None); + + Checker { + rx, + tx, + cdn_list: Arc::new(RwLock::new(CdnList::new())), + ru_blacklist: Arc::new(RwLock::new(RuBlacklist::new())), + geo_ip: Arc::new(RwLock::new(GeoIp::new())), + resolver: Resolver::new().await, + } + } + + pub async fn check(&self, target: Target) -> Result { + let ips = match target.resolve(&self.resolver).await { + Ok(ips) => ips, + Err(ResolveError::NxDomain) => { + return Err(CheckError::NotFound); + } + Err(e) => { + error!("{}", e); + return Err(CheckError::ResolveError(e)); + }, + }; + let geo_ip = self.geo_ip.read().await; + let geo = match ips.get(0).map(|ip| geo_ip.lookup(ip.clone())) { + None => IpInfo::default(), + Some(Ok(ip)) => ip, + Some(Err(e)) => { + error!("{}", e); + return Err(CheckError::GeoIpError); + }, + }; + let mut cdn_provider_subnets: HashMap> = HashMap::new(); + + let cdn_list = self.cdn_list.read().await; + ips.iter() + .filter_map(|ip| cdn_list.contains(ip)) + .map(|ip| (match &ip.region { + None => ip.provider.clone(), + Some(region) => format!("{} ({})", ip.provider, region), + }, ip.clone())) + .for_each(|(k, v)| { + cdn_provider_subnets.entry(k).or_default().insert(v); + }); + + let ru_blacklist = self.ru_blacklist.read().await; + let domain = match &target { + Target::Domain(domain) => ru_blacklist.contains_domain(domain), + _ => None + }; + + let rkn_subnets: HashSet = ips.iter() + .filter_map(|ip| ru_blacklist.contains_ip(ip)) + .collect(); + + Ok(Check { + verdict: match (domain, cdn_provider_subnets.is_empty(), rkn_subnets.is_empty()) { + (None, true, true) => CheckVerdict::Clear, + (domain, _, _) => CheckVerdict::Blocked { + rkn_domain: domain, + rkn_subnets, + cdn_provider_subnets, + } + }, + geo, + ips + }) + } + + pub fn last_update(&self) -> Option> { + self.rx.borrow().clone() + } + + pub async fn update_all(&self) { + match GeoIp::download().await { + Ok(base) => { + if let Err(e) = self.geo_ip.write().await.install(base).await { + error!("Failed to update GeoIP: {}", e); + } + } + Err(e) => { + error!("Failed to download GeoIP: {}", e); + } + } + match RuBlacklist::download().await { + Ok(base) => { + if let Err(e) = self.ru_blacklist.write().await.install(base).await { + error!("Failed to update RKN: {}", e); + } + } + Err(e) => { + error!("Failed to download RKN: {}", e); + } + } + + match CdnList::download().await { + Ok(base) => { + if let Err(e) = self.cdn_list.write().await.install(base).await { + error!("Failed to update CDN: {}", e); + } + } + Err(e) => { + error!("Failed to download CDN: {}", e); + } + } + self.tx.send(Some(Utc::now())).unwrap(); + } + + pub async fn total_domains(&self) -> usize { + self.ru_blacklist.read().await.domain_count + } + + pub async fn total_v4s(&self) -> usize { + (self.cdn_list.read().await.v4_count() + self.ru_blacklist.read().await.v4_count()) as usize + } + +} diff --git a/src/lists.rs b/querying/src/lists.rs similarity index 99% rename from src/lists.rs rename to querying/src/lists.rs index a3c5e30..29c401a 100644 --- a/src/lists.rs +++ b/querying/src/lists.rs @@ -3,8 +3,10 @@ use std::io; use std::io::{BufRead, Error, Read}; use std::net::IpAddr; use std::str::FromStr; +use async_trait::async_trait; use ipnet::IpNet; use ipnet_trie::IpnetTrie; +use log::info; use serde::{de, Deserialize, Deserializer, Serializer}; use trie_rs::map::{Trie, TrieBuilder}; use crate::updater::{fetch_db, Updatable}; diff --git a/src/resolver.rs b/querying/src/resolver.rs similarity index 61% rename from src/resolver.rs rename to querying/src/resolver.rs index fb9b7ab..383d9eb 100644 --- a/src/resolver.rs +++ b/querying/src/resolver.rs @@ -1,12 +1,21 @@ +use std::io::{Error, ErrorKind}; use std::net::IpAddr; use hickory_resolver::config::{LookupIpStrategy, ResolverConfig, ResolverOpts}; use hickory_resolver::name_server::{TokioConnectionProvider}; -use hickory_resolver::proto::ProtoError; +use thiserror::Error; pub struct Resolver { resolver: hickory_resolver::Resolver, } +#[derive(Error, Debug)] +pub enum ResolveError { + #[error("domain not found")] + NxDomain, + #[error("resolver error")] + Other(#[from] Error), +} + impl Resolver { pub async fn new() -> Resolver { let config = ResolverConfig::quad9_https(); @@ -18,8 +27,13 @@ impl Resolver { Resolver { resolver } } - pub async fn lookup_ips(&self, domain: &str) -> Result, ProtoError> { - Ok(self.resolver.lookup_ip(domain).await? + pub async fn lookup_ips(&self, domain: &str) -> Result, ResolveError> { + Ok(self.resolver.lookup_ip(domain).await + .map_err(|e| if e.kind.is_no_records_found() { + ResolveError::NxDomain + } else { + ResolveError::Other(Error::new(ErrorKind::Other, e)) + })? .into_iter().collect()) } } diff --git a/src/target.rs b/querying/src/target.rs similarity index 93% rename from src/target.rs rename to querying/src/target.rs index b2d27d5..d0acbcb 100644 --- a/src/target.rs +++ b/querying/src/target.rs @@ -1,8 +1,8 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -use hickory_resolver::proto::ProtoError; use url::Url; -use crate::resolver::Resolver; +use crate::resolver::{ResolveError, Resolver}; +#[derive(Debug, Clone)] pub enum Target { Domain(String), Ipv4(Ipv4Addr), @@ -37,7 +37,7 @@ impl Target { } } - pub async fn resolve(&self, resolver: &Resolver) -> Result, ProtoError> { + pub async fn resolve(&self, resolver: &Resolver) -> Result, ResolveError> { Ok(match self { Target::Domain(domain) => resolver.lookup_ips(domain).await?, Target::Ipv4(ipv4) => vec![IpAddr::V4(ipv4.clone())], diff --git a/querying/src/updater.rs b/querying/src/updater.rs new file mode 100644 index 0000000..37a38b1 --- /dev/null +++ b/querying/src/updater.rs @@ -0,0 +1,47 @@ +use std::fmt::Display; +use std::io; +use std::io::Error; +use async_trait::async_trait; +use indicatif::{ProgressBar, ProgressStyle}; +use futures_util::StreamExt; +use log::info; +use reqwest::IntoUrl; + +pub async fn fetch_db(url: T) -> Result, Error> { + info!("Fetching {}", url); + let response = reqwest::get(url).await + .map_err(|e| Error::new(io::ErrorKind::Other, e))? + .error_for_status() + .map_err(|e| Error::new(io::ErrorKind::Other, e))?; + + let total_size = response.content_length().unwrap_or(0); + let pb = ProgressBar::new(total_size); + pb.set_style(ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})") + .map_err(|e| Error::new(io::ErrorKind::Other, e))? + .progress_chars("#>-")); + + let mut bytes = Vec::new(); + bytes.reserve(total_size as usize); + let mut stream = response.bytes_stream(); + + while let Some(chunk_result) = stream.next().await { + let chunk = chunk_result.map_err(|e| Error::new(io::ErrorKind::Other, e))?; + bytes.extend(&chunk); + pb.inc(chunk.len() as u64); + } + + pb.finish_with_message("Download complete!"); + + Ok(bytes) +} + +#[async_trait] +pub trait Updatable { + type Base; + async fn download() -> Result; + async fn install(&mut self, base: Self::Base) -> Result<(), Error>; + fn get_url(key: &'static str, default: &'static str) -> String { + std::env::var(key).ok().unwrap_or(default.to_string()) + } +} diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index 3253aae..0000000 --- a/src/main.rs +++ /dev/null @@ -1,215 +0,0 @@ -mod target; -mod resolver; -mod geoip; -mod lists; -mod updater; - -#[macro_use] extern crate rocket; -extern crate maxminddb; - -use std::collections::{HashMap, HashSet}; -use std::path::PathBuf; -use std::sync::Arc; -use std::time::Duration; -use chrono::{DateTime, Utc}; -use rocket::fs::FileServer; -use rocket::http::Status; -use rocket::{tokio, Request, State}; -use rocket::response::content::RawJavaScript; -use rocket_dyn_templates::{context, Metadata, Template}; -use serde::Serialize; -use log::error; -use rocket::tokio::sync::{watch, RwLock}; -use rocket::tokio::time; -use rocket_cache_response::CacheResponse; -use crate::geoip::{GeoIp, IpInfo}; -use crate::lists::{CdnList, NetworkRecord, RuBlacklist}; -use crate::resolver::Resolver; -use crate::target::Target; -use crate::updater::update_all; - -#[derive(Serialize)] -struct GlobalContext { - version: &'static str, -} - -impl GlobalContext { - fn new() -> Self { - GlobalContext { version: env!("CARGO_PKG_VERSION") } - } -} - -#[get("/")] -fn index(info: &State>) -> Template { - Template::render("index", context! { - global: GlobalContext::new(), - info: &*info.borrow(), - }) -} - -#[get("/kb/")] -fn page(metadata: Metadata, page: &str) -> Option