diff --git a/.dockerignore b/.dockerignore index 2d2ecd6..a96e552 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ .git/ +target/ diff --git a/.gitignore b/.gitignore index c403c34..ddbac2f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target .idea/ +.env diff --git a/Cargo.lock b/Cargo.lock index 7001b5f..4b4099c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,19 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -17,6 +30,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -76,6 +95,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + [[package]] name = "async-stream" version = "0.3.6" @@ -95,7 +120,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -106,7 +131,16 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] [[package]] @@ -136,12 +170,24 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a" + [[package]] name = "binascii" version = "0.1.4" @@ -159,6 +205,9 @@ name = "bitflags" version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +dependencies = [ + "serde_core", +] [[package]] name = "block-buffer" @@ -191,6 +240,12 @@ version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.0" @@ -255,6 +310,46 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "clap" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "clap_lex" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" + [[package]] name = "colorchoice" version = "1.0.4" @@ -274,6 +369,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "cookie" version = "0.18.1" @@ -310,6 +411,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.5.0" @@ -353,6 +469,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -396,6 +521,17 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.5" @@ -441,7 +577,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -451,7 +587,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", ] [[package]] @@ -462,14 +600,23 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] [[package]] name = "encode_unicode" @@ -492,10 +639,10 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -537,6 +684,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "fastrand" version = "2.3.0" @@ -594,6 +758,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -641,6 +816,7 @@ checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -663,6 +839,28 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -677,7 +875,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -828,12 +1026,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + [[package]] name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.5.0" @@ -846,6 +1072,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hickory-proto" version = "0.26.0-alpha.1" @@ -867,8 +1099,8 @@ dependencies = [ "once_cell", "rand 0.9.2", "ring", - "rustls", - "thiserror", + "rustls 0.23.35", + "thiserror 2.0.17", "tinyvec", "tokio", "tokio-rustls", @@ -892,15 +1124,42 @@ dependencies = [ "parking_lot", "rand 0.9.2", "resolv-conf", - "rustls", + "rustls 0.23.35", "smallvec", - "thiserror", + "thiserror 2.0.17", "tokio", "tokio-rustls", "tracing", "webpki-roots 0.26.11", ] +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "http" version = "0.2.12" @@ -1032,7 +1291,7 @@ dependencies = [ "http 1.4.0", "hyper 1.8.1", "hyper-util", - "rustls", + "rustls 0.23.35", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1062,7 +1321,7 @@ version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -1231,7 +1490,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.16.1", "serde", "serde_core", ] @@ -1363,7 +1622,7 @@ checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1401,6 +1660,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "libc" @@ -1425,6 +1687,17 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -1501,7 +1774,17 @@ dependencies = [ "log", "memchr", "serde", - "thiserror", + "thiserror 2.0.17", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", ] [[package]] @@ -1516,6 +1799,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1603,6 +1892,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "normpath" version = "1.5.0" @@ -1640,12 +1939,48 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1653,6 +1988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1704,7 +2040,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1757,6 +2093,12 @@ dependencies = [ "regex", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pear" version = "0.2.9" @@ -1777,7 +2119,16 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.111", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", ] [[package]] @@ -1816,7 +2167,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1879,6 +2230,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -1951,7 +2323,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", "version_check", "yansi", ] @@ -1972,7 +2344,7 @@ dependencies = [ "maxminddb", "reqwest", "serde", - "thiserror", + "thiserror 2.0.17", "tokio", "trie-rs", "url", @@ -1990,9 +2362,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", + "rustls 0.23.35", "socket2 0.6.1", - "thiserror", + "thiserror 2.0.17", "tokio", "tracing", "web-time", @@ -2010,10 +2382,10 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash", - "rustls", + "rustls 0.23.35", "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.17", "tinyvec", "tracing", "web-time", @@ -2153,7 +2525,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2185,13 +2557,40 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "reporter" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "csv", + "env_logger", + "futures", + "getrandom 0.3.4", + "indicatif", + "libc", + "log", + "reports", + "reqwest", + "rmp-serde", + "serde", + "tokio", +] + +[[package]] +name = "reports" +version = "0.1.0" +dependencies = [ + "serde", +] + [[package]] name = "reqwest" version = "0.12.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-channel", @@ -2212,7 +2611,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.35", "rustls-pki-types", "serde", "serde_json", @@ -2253,6 +2652,28 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmp" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + [[package]] name = "rocket" version = "0.5.1" @@ -2276,9 +2697,11 @@ dependencies = [ "pin-project-lite", "rand 0.8.5", "ref-cast", + "rmp-serde", "rocket_codegen", "rocket_http", "serde", + "serde_json", "state", "tempfile", "time", @@ -2299,6 +2722,15 @@ dependencies = [ "rocket", ] +[[package]] +name = "rocket-client-addr" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ca3b4757605fd3fbdffd2265091d5dad0086403bc9f31d5719c8352b6fa432" +dependencies = [ + "rocket", +] + [[package]] name = "rocket_codegen" version = "0.5.1" @@ -2311,11 +2743,33 @@ dependencies = [ "proc-macro2", "quote", "rocket_http", - "syn", + "syn 2.0.111", "unicode-xid", "version_check", ] +[[package]] +name = "rocket_db_pools" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6578b2740ceee3e78bff63fe9299d964b7e68318446cdcb9af3b9cab46e1e9d" +dependencies = [ + "rocket", + "rocket_db_pools_codegen", + "sqlx", + "version_check", +] + +[[package]] +name = "rocket_db_pools_codegen" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "842e859f2e87a23efc0f81e25756c0fb43f18726e62daf99da7ea19fbc56cebd" +dependencies = [ + "devise", + "quote", +] + [[package]] name = "rocket_dyn_templates" version = "0.2.0" @@ -2356,6 +2810,26 @@ dependencies = [ "uncased", ] +[[package]] +name = "rsa" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rustc-hash" version = "2.1.1" @@ -2384,6 +2858,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.35" @@ -2394,11 +2879,20 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.8", "subtle", "zeroize", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pki-types" version = "1.13.0" @@ -2409,6 +2903,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.8" @@ -2462,6 +2966,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -2518,7 +3032,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2555,6 +3069,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -2590,6 +3115,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.7" @@ -2649,6 +3184,234 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790" +dependencies = [ + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" +dependencies = [ + "ahash", + "atoi", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rustls 0.21.12", + "rustls-pemfile", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots 0.25.4", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" +dependencies = [ + "dotenvy", + "either", + "heck 0.4.1", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.10.0", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 1.0.69", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.10.0", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 1.0.69", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", + "uuid", +] [[package]] name = "stable-pattern" @@ -2674,12 +3437,40 @@ dependencies = [ "loom", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.111" @@ -2708,7 +3499,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2784,13 +3575,33 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.17", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", ] [[package]] @@ -2801,7 +3612,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2894,7 +3705,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2913,7 +3724,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls", + "rustls 0.23.35", "tokio", ] @@ -3033,6 +3844,7 @@ version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3046,7 +3858,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3134,12 +3946,33 @@ dependencies = [ "version_check", ] +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -3158,6 +3991,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "unit-prefix" version = "0.5.2" @@ -3182,6 +4021,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3257,6 +4102,12 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.105" @@ -3302,7 +4153,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.111", "wasm-bindgen-shared", ] @@ -3348,6 +4199,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + [[package]] name = "webpki-roots" version = "0.26.11" @@ -3368,20 +4225,35 @@ dependencies = [ [[package]] name = "website" -version = "0.1.4" +version = "0.2.0" dependencies = [ + "dotenvy", "env_logger", "flate2", "log", "querying", + "reports", "reqwest", "rocket", "rocket-cache-response", + "rocket-client-addr", + "rocket_db_pools", "rocket_dyn_templates", "serde", + "sqlx", "tar", ] +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + [[package]] name = "widestring" version = "1.2.1" @@ -3427,7 +4299,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3438,7 +4310,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3494,6 +4366,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.60.2" @@ -3767,7 +4648,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", "synstructure", ] @@ -3788,7 +4669,7 @@ checksum = "cf955aa904d6040f70dc8e9384444cb1030aed272ba3cb09bbc4ab9e7c1f34f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -3808,7 +4689,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", "synstructure", ] @@ -3848,5 +4729,5 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] diff --git a/Cargo.toml b/Cargo.toml index 963f006..fb49698 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,8 @@ [workspace] -members = [ "querying", +members = [ + "querying", + "reporter", + "reports", "website", ] resolver = "3" @@ -8,3 +11,4 @@ resolver = "3" tokio = { version = "1.48.0", features = ["full"] } reqwest = { version = "0.12.24", features = ["stream", "rustls-tls-webpki-roots"] } log = "0.4.28" +serde = { version = "1", features = ["derive"] } diff --git a/README.md b/README.md index a4df966..3e9b17c 100644 --- a/README.md +++ b/README.md @@ -33,14 +33,16 @@ Для проверки используются списки [123jjck/cdn-ip-ranges](https://github.com/123jjck/cdn-ip-ranges), [antifilter.download](https://antifilter.download/) и [antifilter.network](https://antifilter.network). +Мы собираем собственные белые списки с помощью [Cheburcheck Reporter](reporter/README.md). + --- ## Структура проекта -* `src/` — исходный код приложения -* `templates/` — Tera-шаблоны для фронтенда -* `static/` — статические файлы -* `Cargo.toml` — зависимости и конфигурация Rust-проекта +* `querying` — модуль проверки сайтов по базам данных +* `reporter` — [Cheburcheck Reporter](reporter/README.md) +* `reports` — общий протокол для отправки отчетов +* `website` — исходный код веб-сайта --- diff --git a/Rocket.toml b/Rocket.toml index c770e1e..9aa669c 100644 --- a/Rocket.toml +++ b/Rocket.toml @@ -1,2 +1,5 @@ [default] template_dir = "website/templates/" + +[global.limits] +msgpack = 5242880 diff --git a/querying/src/geoip.rs b/querying/src/geoip.rs index 9c38f21..4b4c397 100644 --- a/querying/src/geoip.rs +++ b/querying/src/geoip.rs @@ -1,11 +1,11 @@ -use std::{io}; +use crate::updater::{fetch_db, Updatable}; +use async_trait::async_trait; +use maxminddb::geoip2::{city, country, City, Country}; +use maxminddb::{geoip2, MaxMindDbError}; +use serde::Serialize; use std::io::Error; use std::net::IpAddr; -use async_trait::async_trait; -use maxminddb::{geoip2, MaxMindDbError}; -use maxminddb::geoip2::{city, country, City, Country}; -use serde::Serialize; -use crate::updater::{fetch_db, Updatable}; +use std::io; pub struct GeoIp { asn: Option>>, @@ -15,16 +15,20 @@ pub struct GeoIp { #[derive(Serialize, Debug)] pub struct IpInfo { - asn: String, - organisation: String, + pub asn: Option, + pub country_code: Option, + pub organisation: Option, + pub city_geo_name_id: Option, location: String, } impl Default for IpInfo { fn default() -> IpInfo { IpInfo { - asn: "-".to_string(), - organisation: "-".to_string(), + asn: None, + country_code: None, + organisation: None, + city_geo_name_id: None, location: "-".to_string(), } } @@ -57,6 +61,18 @@ impl GeoIp { db.lookup::(ip)? } else { None }; + let country_code = country.as_ref().map(|c| c.country.as_ref() + .map(|c| c.iso_code + .map(|c| c.to_string())) + .flatten()) + .flatten(); + + let city_geo_name_id = city.as_ref() + .map(|c| c.city.as_ref() + .map(|c| c.geoname_id) + .flatten()) + .flatten(); + let location = match (city, country) { (Some(City { city: Some(city::City { names: Some(city), .. }), country: Some(country::Country { names: Some(country), .. }), .. }), _) => { @@ -72,11 +88,12 @@ impl GeoIp { Ok(IpInfo { location, + country_code, + city_geo_name_id, asn: asn.clone().and_then(|asn| asn.autonomous_system_number) - .map(|asn| format!("AS{}", asn)) - .unwrap_or("-".to_string()), + .map(|asn| format!("AS{}", asn)), organisation: asn.and_then(|asn| asn.autonomous_system_organization) - .unwrap_or("-").to_string(), + .map(|org| org.to_string()), }) } } diff --git a/querying/src/lib.rs b/querying/src/lib.rs index 658a300..81b0932 100644 --- a/querying/src/lib.rs +++ b/querying/src/lib.rs @@ -1,16 +1,17 @@ -use std::collections::{HashMap, HashSet}; -use std::net::IpAddr; -use std::sync::Arc; -use chrono::{DateTime, Utc}; -use ipnet::IpNet; -use log::error; -use thiserror::Error; -use tokio::sync::{watch, RwLock}; use crate::geoip::{GeoIp, IpInfo}; use crate::lists::{CdnList, NetworkRecord, RuBlacklist}; use crate::resolver::{ResolveError, Resolver}; use crate::target::Target; use crate::updater::Updatable; +use chrono::{DateTime, Utc}; +use ipnet::IpNet; +use log::error; +use std::collections::{HashMap, HashSet}; +use std::net::IpAddr; +use std::sync::Arc; +use maxminddb::MaxMindDbError; +use thiserror::Error; +use tokio::sync::{watch, RwLock}; pub mod geoip; pub mod lists; @@ -66,6 +67,10 @@ impl Checker { } } + pub async fn geo_ip(&self, ip: IpAddr) -> Result { + self.geo_ip.read().await.lookup(ip) + } + pub async fn check(&self, target: Target) -> Result { let ips = match target.resolve(&self.resolver).await { Ok(ips) => ips, diff --git a/querying/src/lists.rs b/querying/src/lists.rs index 29c401a..299d92c 100644 --- a/querying/src/lists.rs +++ b/querying/src/lists.rs @@ -1,15 +1,15 @@ -use std::collections::VecDeque; -use std::io; -use std::io::{BufRead, Error, Read}; -use std::net::IpAddr; -use std::str::FromStr; +use crate::updater::{fetch_db, Updatable}; use async_trait::async_trait; use ipnet::IpNet; use ipnet_trie::IpnetTrie; use log::info; use serde::{de, Deserialize, Deserializer, Serializer}; +use std::collections::VecDeque; +use std::io; +use std::io::{BufRead, Error, Read}; +use std::net::IpAddr; +use std::str::FromStr; use trie_rs::map::{Trie, TrieBuilder}; -use crate::updater::{fetch_db, Updatable}; pub struct CdnList { trie: IpnetTrie, diff --git a/querying/src/resolver.rs b/querying/src/resolver.rs index 383d9eb..fe3adcc 100644 --- a/querying/src/resolver.rs +++ b/querying/src/resolver.rs @@ -1,7 +1,7 @@ +use hickory_resolver::config::{LookupIpStrategy, ResolverConfig, ResolverOpts}; +use hickory_resolver::name_server::TokioConnectionProvider; use std::io::{Error, ErrorKind}; use std::net::IpAddr; -use hickory_resolver::config::{LookupIpStrategy, ResolverConfig, ResolverOpts}; -use hickory_resolver::name_server::{TokioConnectionProvider}; use thiserror::Error; pub struct Resolver { diff --git a/querying/src/target.rs b/querying/src/target.rs index d0acbcb..ee77f28 100644 --- a/querying/src/target.rs +++ b/querying/src/target.rs @@ -1,6 +1,6 @@ +use crate::resolver::{ResolveError, Resolver}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use url::Url; -use crate::resolver::{ResolveError, Resolver}; #[derive(Debug, Clone)] pub enum Target { diff --git a/querying/src/updater.rs b/querying/src/updater.rs index 37a38b1..c4d0a2d 100644 --- a/querying/src/updater.rs +++ b/querying/src/updater.rs @@ -1,11 +1,11 @@ +use async_trait::async_trait; +use futures_util::StreamExt; +use indicatif::{ProgressBar, ProgressStyle}; +use log::info; +use reqwest::IntoUrl; use std::fmt::Display; use std::io; use std::io::Error; -use async_trait::async_trait; -use indicatif::{ProgressBar, ProgressStyle}; -use futures_util::StreamExt; -use log::info; -use reqwest::IntoUrl; pub async fn fetch_db(url: T) -> Result, Error> { info!("Fetching {}", url); diff --git a/reporter/Cargo.toml b/reporter/Cargo.toml new file mode 100644 index 0000000..89ffa5e --- /dev/null +++ b/reporter/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "reporter" +version = "0.1.0" +edition = "2024" + +[dependencies] +tokio = { workspace = true } +reqwest = { workspace = true } +log = { workspace = true } +reports = { path = "../reports" } +anyhow = "1.0" +env_logger = "0.11.8" +futures = "0.3" +clap = { version = "4.5.53", features = ["derive", "env"] } +csv = "1.4.0" +indicatif = "0.18.3" +rmp-serde = "1.3.0" +serde = { version = "1.0.228", features = ["derive"] } + +[target.'cfg(unix)'.dependencies] +libc = "0.2.177" + +[build-dependencies] +reqwest = { workspace = true } +getrandom = "0.3.4" diff --git a/reporter/README.md b/reporter/README.md new file mode 100644 index 0000000..1682a33 --- /dev/null +++ b/reporter/README.md @@ -0,0 +1,43 @@ +# Cheburcheck Reporter + +Инструмент для автоматического сканирования и анализа блокировок. +Позволяет сканировать большое количество доменов за короткий срок за счет параллелизации запросов. + +## Использование + +В программе заранее установлены настройки по-умолчанию для сканирования топ-100,000 доменов из [Tranco list от 26 ноября 2025](https://tranco-list.eu/list/2NPQ9). +Для этого достаточно запустить его: + +1. Сохранить результаты в CSV + +```shell +reporter output.csv +``` + +2. Отправить результаты в **Cheburcheck Agency** + +```shell +reporter -k +``` + +> Для того чтобы получить API-ключ, +> обратитесь по адресу [agency@cheburcheck.ru](mailto:agency@cheburcheck.ru). +> +> В письме укажите ваш интернет-провайдер и регион. + +## Конфигурация + +| Параметр | Описание | Значение по-умолчанию | +|-------------------------------------|-----------------------------------------------------------------------------------------|--------------------------------------| +| `-f, --fake ` | Использовать один домен для всех запросов | | +| `-c, --count ` | Количество доменов, которые будут проверены (берутся с начала списка) | 100,000 | +| `-t, --timeout-secs ` | Максимальное время ожидания ответа от сервера | 5 секунд | +| `-p, --probes ` | Максимальное количество одновременных запросов | 1000 | +| `-v, --verbosity ` | Отображение результатов сканирования в консоли [silent, error, block, all] | silent | +| `-r, --retry-count ` | Количество попыток запросов на один домен | 2 | +| `-H, --http` | Использовать plain-HTTP (без TLS) | | +| `-x, --tx` | Отправлять мусорные данные 64кб на сервер | | +| `-i, --ip ` | IP-адрес сервера, на который будут идти запросы (должен отвечать >64kb на любые домены) | 5.78.7.195 | +| `-P, --path ` | Путь к файлу на сервере | 100MB.bin | +| `-a, --endpoint ` | Адрес сервера, на который будут загружены результаты сканирования | https://cheburcheck.ru/agency/report | +| `-k, --key ` | API-ключ | | diff --git a/reporter/build.rs b/reporter/build.rs new file mode 100644 index 0000000..0728b45 --- /dev/null +++ b/reporter/build.rs @@ -0,0 +1,25 @@ +use std::{env, fs, path::Path}; +use reqwest::blocking::Client; + +fn main() { + let out_dir = env::var("OUT_DIR").expect("OUT_DIR not set"); + let junk_path = Path::new(&out_dir).join("junk.bin"); + + let mut data = vec![0u8; 64 * 1024]; + getrandom::fill(&mut data).expect("Failed to generate random bytes"); + fs::write(&junk_path, &data).expect("Failed to write random data"); + + let client = Client::new(); + println!("cargo:rerun-if-env-changed=DIST_DOMAIN_COUNT"); + let data = client.get( + format!("https://tranco-list.eu/download/2NPQ9/{}", option_env!("DIST_DOMAIN_COUNT") + .map(|x| x.parse::().expect("DIST_DOMAIN_COUNT is not a number")) + .unwrap_or(1_000_000))) + .send().expect("Failed to send request") + .bytes().expect("Failed to read bytes"); + + let list_csv = Path::new(&out_dir).join("list.csv"); + fs::write(&list_csv, &data).expect("Failed to write list"); + + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/reporter/src/counter.rs b/reporter/src/counter.rs new file mode 100644 index 0000000..4802878 --- /dev/null +++ b/reporter/src/counter.rs @@ -0,0 +1,66 @@ +use std::collections::HashMap; +use std::fmt::Display; +use std::path::PathBuf; +use log::info; +use reports::Evidence; +use crate::Verbosity; + +#[derive(Default)] +pub struct Counter { + ok: usize, + block: usize, + err: usize, + pub early: usize, + pub results: HashMap, +} + +impl Counter { + pub fn save_results(&self, output: &PathBuf) -> anyhow::Result<()> { + let mut out = csv::WriterBuilder::new().from_path(output)?; + out.write_record(&["target", "evidence"])?; + for (target, evidence) in &self.results { + out.write_record(&[target, &evidence.to_string()])?; + } + info!("Saved results to {:?}", output); + Ok(()) + } + + pub fn print_results(&self, verbosity: &Verbosity) { + if verbosity > &Verbosity::Silent { + info!("Results:"); + for (target, evidence) in &self.results { + match evidence { + Evidence::Ok if verbosity >= &Verbosity::All => println!(" [Ok] {}", target), + Evidence::Blocked if verbosity >= &Verbosity::Block => println!(" [Blocked] {}", target), + Evidence::ConnectError if verbosity >= &Verbosity::Error => println!(" [ConnectError] {}", target), + _ => {} + } + } + } + } + pub fn total(&self) -> usize { + self.ok + self.block + self.err + } + + pub fn add(&mut self, target: &str, evidence: Evidence) { + match evidence { + Evidence::Ok => self.ok += 1, + Evidence::Blocked => self.block += 1, + Evidence::ConnectError | Evidence::Error => self.err += 1, + } + self.results.insert(target.to_string(), evidence); + } +} + +impl Display for Counter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let total = self.total(); + write!(f, "OK {} ({:.2}%) | Blocked {} (early: {}) ({:.2}%) | Error {} ({:.2}%)", + self.ok, + self.ok as f32 / total as f32 * 100.0, + self.block, self.early, + self.block as f32 / total as f32 * 100.0, + self.err, + self.err as f32 / total as f32 * 100.0) + } +} diff --git a/reporter/src/main.rs b/reporter/src/main.rs new file mode 100644 index 0000000..3378b09 --- /dev/null +++ b/reporter/src/main.rs @@ -0,0 +1,307 @@ +mod resolver; +mod counter; + +use crate::resolver::Resolver; +use anyhow::Result; +use clap::{Parser, ValueEnum}; +use futures::stream::FuturesUnordered; +use futures::StreamExt; +use indicatif::{ProgressIterator, ProgressStyle}; +use log::{error, info, warn, LevelFilter}; +use reports::{AgencyReport, Evidence, ReporterConfig}; +use reqwest::redirect::Policy; +use reqwest::Client; +use serde::Serialize; +use std::collections::HashMap; +use std::fmt::Display; +use std::net::IpAddr; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use tokio::time::Instant; +use counter::Counter; + +const JUNK: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/junk.bin")); + +#[derive(Serialize, Debug, Ord, PartialOrd, Eq, PartialEq, Clone, ValueEnum)] +#[serde(rename_all = "kebab-case")] +enum Verbosity { + Silent, + Error, + Block, + All, +} + +#[derive(Parser, Debug, Clone)] +#[command(author, version, about = "DPI probe: checks blockage of domains by SNI")] +struct Args { + /// Output results file + #[arg(required = false)] + output: Option, + + /// Fake target to establish probing settings. + /// Use with caution - it might trigger TLS block + #[arg(short, long, required = false)] + fake: Option, + + /// Take first N targets + #[arg(short, long, default_value_t = 100_000)] + count: usize, + + /// Read timeout in seconds + #[arg(short, long, default_value_t = 5)] + timeout_secs: u64, + + /// Maximum concurrent probes. Make sure that it doesn't exceed 'ulimit -n' + #[arg(short, long = "probes", default_value_t = 1000)] + probe_count: usize, + + /// Display probing results in console + #[arg(short, long, default_value_t = Verbosity::Silent, value_enum)] + verbosity: Verbosity, + + /// Attempts to establish connection + #[arg(short, long, default_value_t = 2)] + retry_count: usize, + + /// Try using plain HTTP without TLS + #[arg(short = 'H', long, default_value_t = false)] + http: bool, + + /// Send 64kb junk to server + #[arg(short = 'x', long, default_value_t = false)] + tx: bool, + + /// Target IP to probe with. + /// It should be included in IP-ranges of interest. + /// The server must respond to any SNI/Host with a response larger than 64kb. + #[arg(short, long, default_value = "5.78.7.195", value_parser = |v: &str| v.parse::())] + ip: IpAddr, + + /// File name on the server to test + #[arg(short = 'P', long, default_value = "100MB.bin")] + path: String, + + /// Custom agency endpoint address + #[arg(short, long = "endpoint", default_value_t = option_env!("AGENCY_ENDPOINT") + .unwrap_or("https://cheburcheck.ru/agency/upload") + .to_string())] + agency_endpoint: String, + + /// Agency endpoint API key + #[arg(short, long, env = "AGENCY_KEY")] + key: Option, + +} + +impl Args { + fn to_reporter_config(&self) -> ReporterConfig { + ReporterConfig { + http: self.http, + tx_junk: self.tx, + ip: self.ip.clone(), + path: self.path.clone(), + retry_count: self.retry_count, + timeout_secs: self.timeout_secs, + probe_count: self.probe_count, + } + } +} + +fn build_client(args: &Args, attempt: usize) -> reqwest::Result { + let client = Client::builder() + .danger_accept_invalid_certs(true) + .redirect(Policy::none()) + .use_rustls_tls() + .dns_resolver(Arc::new(Resolver::new(args.ip))) + .read_timeout(Duration::from_secs(args.timeout_secs * attempt as u64)) + .timeout(Duration::from_secs(15)); + + Ok(client.build()?) +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + env_logger::builder().filter_level(LevelFilter::Info).init(); + + #[cfg(target_family = "unix")] + { + let file_limit: Option = unsafe { libc::getdtablesize() }.try_into().ok(); + if matches!(file_limit, Some(file_limit) if file_limit <= args.probe_count + 128) { + warn!("Open file limit is too low ({})! Consider increasing it using `ulimit -n`.", file_limit.unwrap()); + } + } + + let api_client = Client::new(); + info!("Loading targets list..."); + let targets = include_str!(concat!(env!("OUT_DIR"), "/list.csv")); + let targets: Vec = targets.lines().take(args.count) + .map(|s| s.split(",").last().unwrap().to_string()).collect(); + + info!("Probing {} domains with {} concurrent probes...", targets.len(), args.probe_count); + let sem = Arc::new(tokio::sync::Semaphore::new(args.probe_count)); + let cancelled = wait_for_ctrlc(); + let start = Instant::now(); + let mut futs = FuturesUnordered::new(); + for target in targets.into_iter().progress() + .with_style(ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {human_pos}/{human_len} ({eta}, {per_sec})")? + .progress_chars("#>-")) { + if cancelled() { + break; + } + let permit = sem.clone().acquire_owned().await?; + let args = args.clone(); + let fake_target = args.fake.clone(); + futs.push(tokio::spawn(async move { + let res = check_target(&args, fake_target.as_ref().unwrap_or(&target)).await; + drop(permit); + (target, res) + })); + } + info!("Collecting results..."); + + let mut counter = Counter::default(); + while let Some(res) = futs.next().await { + match res { + Ok((target, Ok(Verdict::Accepted))) => { + counter.add(&target, Evidence::Ok); + } + Ok((target, Ok(Verdict::Blocked { early }))) => { + counter.add(&target, Evidence::Blocked); + if early { + counter.early += 1; + } + } + Ok((target, Err(e))) if e.is_connect() => { + counter.add(&target, Evidence::ConnectError); + if args.verbosity >= Verbosity::Error { + println!("{e:?}"); + } + } + Ok((target, Err(_))) => { + counter.add(&target, Evidence::Error); + } + Err(join_err) => { + error!("Task join error: {}", join_err); + } + }; + } + + counter.print_results(&args.verbosity); + if let Some(output) = &args.output { + counter.save_results(output)?; + } + + info!("Probed {} domains in {}s! \nSummary: {counter}", counter.total(), start.elapsed().as_secs()); + if let Err(e) = upload_results(&args, &api_client, counter.results).await { + warn!("Upload failed: {}", e); + } + + Ok(()) +} + +async fn upload_results(args: &Args, api_client: &Client, results: HashMap) -> Result<()> { + info!("Uploading to {}", args.agency_endpoint); + + let uploaded = api_client.post(&args.agency_endpoint) + .header("Content-Type", "application/msgpack") + .body(rmp_serde::to_vec(&AgencyReport { + version: env!("CARGO_PKG_VERSION").to_string(), + config: args.to_reporter_config(), + data: results, + })?); + + let uploaded = if let Some(key) = &args.key { + uploaded.header("Authorization", format!("Bearer {key}")) + } else { uploaded }; + + let uploaded = uploaded.send().await?; + + // let uploaded = uploaded.error_for_status()?; + + if uploaded.status().is_success() { + info!("Uploaded ({})!", uploaded.status().to_string()); + } else { + warn!("Upload failed: {}", uploaded.status().to_string()); + } + info!("Agency response: {}", uploaded.text().await?); + Ok(()) +} + +fn wait_for_ctrlc() -> impl Fn() -> bool { + let cancelled = Arc::new(AtomicUsize::new(0)); + let cancelled_ctrlc = cancelled.clone(); + + tokio::spawn(async move { + loop { + let _ = tokio::signal::ctrl_c().await; + + match cancelled_ctrlc.fetch_add(1, Ordering::SeqCst) { + 0 => warn!("Ctrl-C received. Finishing up and saving..."), + _ => { + warn!("Forcing exit."); + std::process::exit(130); + } + } + } + }); + + move || { + cancelled.load(Ordering::SeqCst) != 0 + } +} + +enum Verdict { + Blocked { early: bool }, + Accepted, +} + +async fn check_target(args: &Args, target: &str) -> Result { + let url = format!("http{}://{target}/{}", if args.http {""} else {"s"}, args.path); + let mut attempts = 0; + + loop { + attempts += 1; + let client = build_client(&args, 1)?; + let mut resp = client.get(&url) + .header("Range", "bytes=0-65536"); + if args.tx { + resp = resp.body(JUNK) + } + let resp = resp.send() + .await; + + let resp = match resp { + Ok(resp) => match (resp.status(), resp.bytes().await) { + (status, Ok(b)) => Ok((status, b)), + (_, Err(e)) => Err((e, false)), + }, + Err(e) => Err((e, true)), + }; + return match resp { + Ok((status, bytes)) => { + if !status.is_success() { + warn!("Domain {target} returned non-OK code: {status}"); + } + if bytes.len() < 65535 { + warn!("Domain {target} completed with {} bytes: \n{}", bytes.len(), String::from_utf8_lossy(bytes.as_ref())); + } + Ok(Verdict::Accepted) + } + Err((e, early)) => { + if attempts < args.retry_count { + continue; + } + if e.is_timeout() { + Ok(Verdict::Blocked { early }) + } else { + error!("{} -> Error: {:?}", target, e); + Err(e) + } + }, + } + } +} diff --git a/reporter/src/resolver.rs b/reporter/src/resolver.rs new file mode 100644 index 0000000..bfd517c --- /dev/null +++ b/reporter/src/resolver.rs @@ -0,0 +1,23 @@ +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; +use std::net::{IpAddr, SocketAddr}; + +pub struct Resolver { + ip: SocketAddr, +} + +impl Resolver { + pub fn new(ip: IpAddr) -> Resolver { + Resolver { + ip: SocketAddr::from(SocketAddr::new(ip, 0)), + } + } +} + +impl Resolve for Resolver { + fn resolve(&self, _: Name) -> Resolving { + let ip = self.ip.clone(); + Box::pin(async move { + Ok(Addrs::from(Box::new(vec![ip].into_iter()))) + }) + } +} diff --git a/reports/Cargo.toml b/reports/Cargo.toml new file mode 100644 index 0000000..fc89352 --- /dev/null +++ b/reports/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "reports" +version = "0.1.0" +edition = "2024" + +[dependencies] +serde = { workspace = true } diff --git a/reports/src/lib.rs b/reports/src/lib.rs new file mode 100644 index 0000000..c40ea7c --- /dev/null +++ b/reports/src/lib.rs @@ -0,0 +1,42 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fmt::Display; +use std::net::IpAddr; + +#[derive(Debug, Serialize, Deserialize)] +pub struct AgencyReport { + pub version: String, + pub config: ReporterConfig, + pub data: HashMap, +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum Evidence { + Ok, + Blocked, + ConnectError, + Error, +} + +impl Display for Evidence { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let str = match self { + Evidence::Ok => "ok", + Evidence::Blocked => "blocked", + Evidence::ConnectError => "connect_error", + Evidence::Error => "unknown_error", + }; + write!(f, "{}", str) + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ReporterConfig { + pub http: bool, + pub tx_junk: bool, + pub ip: IpAddr, + pub path: String, + pub retry_count: usize, + pub timeout_secs: u64, + pub probe_count: usize, +} diff --git a/website.Dockerfile b/website.Dockerfile index 0894f1a..9005951 100644 --- a/website.Dockerfile +++ b/website.Dockerfile @@ -9,7 +9,9 @@ RUN apt update && apt install -y libssl-dev pkg-config RUN --mount=type=cache,target=/build/target \ --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=secret,id=DATABASE_URL \ set -eux; \ + export DATABASE_URL=$(cat /run/secrets/DATABASE_URL); \ cargo build --release --package website; \ objcopy --compress-debug-sections target/release/website ./main diff --git a/website/Cargo.toml b/website/Cargo.toml index 308e7a0..046f428 100644 --- a/website/Cargo.toml +++ b/website/Cargo.toml @@ -1,16 +1,21 @@ [package] name = "website" -version = "0.1.4" +version = "0.2.0" edition = "2024" [dependencies] -rocket = "0.5.1" +rocket = { version = "0.5.1", features = ["msgpack", "json"] } +rocket_db_pools = { version = "0.2.0", features = ["sqlx_postgres"]} rocket_dyn_templates = { version = "0.2.0", features = ["tera"] } -serde = { version = "1", features = ["derive"] } +sqlx = { version = "0.7", features = ["runtime-tokio", "postgres", "migrate", "chrono", "uuid"] } +rocket-client-addr = "0.5.4" +serde = { workspace = true } +reports = { path = "../reports" } querying = { path = "../querying" } env_logger = "0.11.8" rocket-cache-response = "0.6.4" log = { workspace = true } +dotenvy = { version = "0.15.7" } [build-dependencies] reqwest = { version = "0.12", features = ["blocking", "json"] } diff --git a/website/build.rs b/website/build.rs index 7a952fa..13716d3 100644 --- a/website/build.rs +++ b/website/build.rs @@ -1,10 +1,11 @@ use std::{ env, - fs::{File}, + fs::File, io::{self, Cursor}, path::Path, }; - +use std::collections::HashMap; +use std::path::PathBuf; use flate2::read::GzDecoder; use reqwest::blocking::Client; use serde::Deserialize; @@ -21,13 +22,35 @@ struct Dist { } fn main() -> Result<(), Box> { - let out_dir = env::var("OUT_DIR")?; - let lucide_js_path = Path::new(&out_dir).join("lucide.js"); - - let client = Client::new(); + println!("cargo:rerun-if-changed=migrations"); + let out_dir = PathBuf::from(env::var("OUT_DIR")?); println!("cargo:rerun-if-env-changed=LUCIDE_VERSION"); - let metadata_url = format!("https://registry.npmjs.org/lucide/{}", option_env!("LUCIDE_VERSION").unwrap_or("latest")); + install_npm("lucide", option_env!("LUCIDE_VERSION"), HashMap::from([ + (PathBuf::from("package/dist/umd/lucide.min.js"), out_dir.join("lucide.js")), + ]))?; + + println!("cargo:rerun-if-env-changed=CHARTJS_VERSION"); + install_npm("chart.js", option_env!("CHARTJS_VERSION"), HashMap::from([ + (PathBuf::from("package/dist/chart.umd.min.js"), out_dir.join("chart.js")), + ]))?; + + println!("cargo:rerun-if-env-changed=CHARTDATALABELS_VERSION"); + install_npm("chartjs-plugin-datalabels", option_env!("CHARTDATALABELS_VERSION"), HashMap::from([ + (PathBuf::from("package/dist/chartjs-plugin-datalabels.min.js"), out_dir.join("chartjs-plugin-datalabels.js")), + ]))?; + + println!("cargo:rerun-if-changed=build.rs"); + + Ok(()) +} + +fn install_npm(package: &str, version: Option<&str>, mut entries: HashMap) -> Result<(), Box> { + let client = Client::new(); + let metadata_url = format!( + "https://registry.npmjs.org/{package}/{}", + version.unwrap_or("latest") + ); println!("Fetching npm metadata from {}", metadata_url); let metadata: NpmMetadata = client.get(metadata_url).send()?.json()?; @@ -38,24 +61,19 @@ fn main() -> Result<(), Box> { let tar = GzDecoder::new(Cursor::new(tarball_bytes)); let mut archive = Archive::new(tar); - let mut lucide_js_found = false; for entry in archive.entries()? { let mut entry = entry?; - let path = entry.path()?; - if path == Path::new("package/dist/umd/lucide.min.js") { - println!("Found lucide.min.js, extracting to {:?}", lucide_js_path); - let mut out_file = File::create(&lucide_js_path)?; + let path = PathBuf::from(entry.path()?); + if let Some(dest) = entries.remove(&path) { + println!("Found {:?}, extracting to {:?}", path.file_name(), dest); + let mut out_file = File::create(&dest)?; io::copy(&mut entry, &mut out_file)?; - lucide_js_found = true; - break; } } - if !lucide_js_found { - panic!("Could not find dist/lucide.min.js inside the tarball"); + if !entries.is_empty() { + panic!("Not all entries found! {:?}", entries); } - println!("cargo:rerun-if-changed=build.rs"); - Ok(()) } diff --git a/website/migrations/20251207232812_initial_migration.sql b/website/migrations/20251207232812_initial_migration.sql new file mode 100644 index 0000000..e5e3734 --- /dev/null +++ b/website/migrations/20251207232812_initial_migration.sql @@ -0,0 +1,130 @@ +-- Queries +CREATE TABLE IF NOT EXISTS queries +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + query VARCHAR(255) NOT NULL, + source_ip VARCHAR(39) NOT NULL, + source_country_code VARCHAR(5), + source_city_geo_name_id INT, + + target_country_code VARCHAR(5), + target_asn VARCHAR(32), + target_provider VARCHAR(255), + + resolved_ips VARCHAR(39)[], + + cdn_networks VARCHAR(43)[], + cdn_providers VARCHAR(255)[], + + rkn_domain VARCHAR(255), + + date TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS queries_query_date_idx ON queries (query, date); + +-- Human reports +CREATE TABLE IF NOT EXISTS human_reports +( + id UUID PRIMARY KEY REFERENCES queries (id), + source_ip VARCHAR(39) NOT NULL, + date TIMESTAMP DEFAULT NOW(), + works BOOLEAN +); + +CREATE INDEX IF NOT EXISTS human_reports_date_idx ON human_reports (date); +CREATE INDEX IF NOT EXISTS human_reports_works_idx ON human_reports (works); + +-- Domain rankings list +CREATE TABLE IF NOT EXISTS domains +( + domain VARCHAR(255) PRIMARY KEY, + rank INT +); + +CREATE INDEX IF NOT EXISTS domains_domain_rank_idx ON domains (domain, rank); + +-- Agency reporters +CREATE TABLE IF NOT EXISTS reporters +( + id SERIAL PRIMARY KEY, + token VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL +); + +CREATE INDEX IF NOT EXISTS reporters_token_idx ON reporters (token); + +-- Agency reports +CREATE TABLE IF NOT EXISTS reports +( + id SERIAL PRIMARY KEY, + reporter SERIAL, + reporter_ip VARCHAR(39) NOT NULL, + date TIMESTAMP DEFAULT NOW(), + + version VARCHAR(32) NOT NULL, + http BOOLEAN, + tx_junk BOOLEAN, + ip VARCHAR(39), + path VARCHAR(255), + retry_count INT, + timeout_secs INT, + probe_count INT, + + FOREIGN KEY (reporter) REFERENCES reporters (id) +); + +CREATE INDEX IF NOT EXISTS reports_reporter_date_id_idx ON reports (reporter, date DESC, id); + +DO +$$ + BEGIN + CREATE TYPE evidence AS ENUM ('ok', 'blocked', 'connection_error', 'unknown_error'); + EXCEPTION + WHEN duplicate_object THEN null; + END +$$; + +CREATE TABLE IF NOT EXISTS report_row +( + id BIGSERIAL PRIMARY KEY, + report_id SERIAL, + domain VARCHAR(255), + evidence evidence, + FOREIGN KEY (report_id) REFERENCES reports (id) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX IF NOT EXISTS report_row_id_domain_evidence_idx ON report_row (report_id, domain, evidence); + +-- Whitelist view +CREATE MATERIALIZED VIEW IF NOT EXISTS + whitelist AS +WITH ranked_reports AS (SELECT rr.domain, + rr.evidence, + r.date, + ROW_NUMBER() OVER ( + PARTITION BY + rr.domain + ORDER BY + r.date DESC + ) AS rn + FROM report_row rr + JOIN reports r ON rr.report_id = r.id + WHERE r.reporter = 1) +SELECT rr.domain, + d.rank, + MAX( + CASE + WHEN rr.evidence = 'ok' THEN rr.date + END + ) AS last_ok +FROM ranked_reports rr + LEFT JOIN domains d ON d.domain = rr.domain +WHERE rr.rn <= 5 +GROUP BY rr.domain, + d.rank +HAVING COUNT(*) FILTER ( + WHERE + rr.evidence = 'ok' + ) >= COUNT(*) / 2.0 +ORDER BY d.rank; diff --git a/website/src/agency.rs b/website/src/agency.rs new file mode 100644 index 0000000..6406379 --- /dev/null +++ b/website/src/agency.rs @@ -0,0 +1,85 @@ +use crate::Db; +use reports::AgencyReport; +use rocket::http::Status; +use rocket::serde::json::serde_json::json; +use rocket::serde::json::{Json, Value}; +use rocket::serde::msgpack::MsgPack; +use rocket_client_addr::ClientRealAddr; +use rocket_db_pools::Connection; +use sqlx::Acquire; + +pub struct Agency { + pub id: i32, + pub name: String, +} + +#[rocket::post("/report", format = "application/msgpack", data = "")] +pub async fn upload_report( + report: MsgPack, + addr: &ClientRealAddr, + agency: Agency, + mut db: Connection, +) -> Result, (Status, String)> { + let mut tx = db + .begin() + .await + .map_err(|e| (Status::InternalServerError, e.to_string()))?; + let report = report.into_inner(); + + let report_id: i32 = sqlx::query_scalar( + "INSERT INTO reports ( + reporter, + reporter_ip, + version, + http, + tx_junk, + ip, + path, + retry_count, + timeout_secs, + probe_count + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id", + ) + .bind(agency.id) + .bind(addr.ip.to_string()) + .bind(report.version) + .bind(report.config.http) + .bind(report.config.tx_junk) + .bind(report.config.ip.to_string()) + .bind(report.config.path) + .bind(report.config.retry_count as i32) + .bind(report.config.timeout_secs as i64) + .bind(report.config.probe_count as i32) + .fetch_one(&mut *tx) + .await + .map_err(|e| (Status::InternalServerError, e.to_string()))?; + + let mut copy_in = tx + .copy_in_raw("COPY report_row (report_id, evidence, domain) FROM STDIN (FORMAT CSV)") + .await + .map_err(|e| (Status::InternalServerError, e.to_string()))?; + + for (domain, evidence) in report.data { + let line = format!("{},{},{}\n", report_id, evidence, domain); + copy_in + .send(line.as_bytes()) + .await + .map_err(|e| (Status::InternalServerError, e.to_string()))?; + } + + copy_in + .finish() + .await + .map_err(|e| (Status::InternalServerError, e.to_string()))?; + + sqlx::query!("REFRESH MATERIALIZED VIEW whitelist") + .execute(&mut *tx) + .await + .map_err(|e| (Status::InternalServerError, e.to_string()))?; + + tx.commit() + .await + .map_err(|e| (Status::InternalServerError, e.to_string()))?; + + Ok(Json(json!({ "ok": true, "id": report_id }))) +} diff --git a/website/src/db.rs b/website/src/db.rs new file mode 100644 index 0000000..436ab1a --- /dev/null +++ b/website/src/db.rs @@ -0,0 +1,179 @@ +use crate::agency::Agency; +use crate::Db; +use querying::target::Target; +use querying::{Check, CheckVerdict, Checker}; +use rocket::http::Status; +use rocket::outcome::{try_outcome, IntoOutcome}; +use rocket::request::{FromRequest, Outcome}; +use rocket::tokio::sync::RwLockReadGuard; +use rocket::Request; +use rocket_client_addr::ClientRealAddr; +use rocket_db_pools::Connection; +use serde::Serialize; +use sqlx::types::chrono::NaiveDateTime; +use sqlx::types::Uuid; + +pub async fn save_query( + db: &mut Connection, + target: &Target, + check: &Check, + addr: &ClientRealAddr, + checker: RwLockReadGuard<'_, Checker>, +) -> Result { + let (cdn_networks, cdn_providers, rkn_domain): (Vec<_>, Vec<_>, Option<_>) = + if let CheckVerdict::Blocked { + cdn_provider_subnets, + rkn_domain, + .. + } = &check.verdict + { + ( + cdn_provider_subnets + .values() + .flatten() + .map(|n| n.cidr.to_string()) + .collect(), + cdn_provider_subnets.keys().map(|p| p.to_string()).collect(), + rkn_domain.clone(), + ) + } else { + (vec![], vec![], None) + }; + + let id = sqlx::query_scalar( + "INSERT INTO queries ( + query, + source_ip, + source_country_code, + source_city_geo_name_id, + target_country_code, + target_asn, + target_provider, + resolved_ips, + cdn_networks, + cdn_providers, + rkn_domain + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING id", + ) + .bind(target.to_query()) + .bind(addr.ip.to_string()) + .bind( + checker + .geo_ip(addr.ip) + .await + .map(|i| i.country_code) + .ok() + .flatten(), + ) + .bind(check.geo.city_geo_name_id.map(|id| id as i32)) + .bind(check.geo.country_code.clone()) + .bind(check.geo.asn.clone()) + .bind(check.geo.organisation.clone()) + .bind( + check + .ips + .iter() + .map(|i| i.to_string()) + .collect::>(), + ) + .bind(cdn_networks) + .bind(cdn_providers) + .bind(rkn_domain) + .fetch_one(&mut ***db) + .await?; + + Ok(id) +} + +#[rocket::async_trait] +impl<'r> FromRequest<'r> for Agency { + type Error = Option>; + + async fn from_request(request: &'r Request<'_>) -> Outcome { + let mut db = try_outcome!(Connection::::from_request(request).await); + let token = request.headers().get_one("Authorization"); + + let token = try_outcome!( + token + .and_then(|t| t.split_once(" ")) + .map(|(_, tok)| tok.to_string()) + .or_forward(Status::Unauthorized) + ); + + let agency = try_outcome!( + sqlx::query!("SELECT id, name FROM reporters WHERE token = $1", token) + .fetch_optional(&mut **db) + .await + .map_err(|e| Some(rocket_db_pools::Error::Get(e))) + .or_forward(Status::InternalServerError) + ); + agency + .map(|r| Agency { + id: r.id, + name: r.name, + }) + .or_forward(Status::Unauthorized) + } +} + +#[derive(Serialize, Debug, sqlx::FromRow)] +pub struct WhitelistedEntry { + domain: Option, + rank: Option, + last_ok: Option, +} + +pub async fn check_whitelist( + domain: &str, + db: &mut Connection, +) -> Result, sqlx::Error> { + sqlx::query_as!( + WhitelistedEntry, + "SELECT * + FROM whitelist + WHERE $1 = domain + OR $1 LIKE CONCAT('%.', domain) + ORDER BY LENGTH(domain) DESC + LIMIT 1", + domain + ) + .fetch_optional(&mut ***db) + .await + .into() +} + +#[derive(Debug, Serialize, sqlx::FromRow)] +pub struct WhitelistHistogramBin { + pub bin_id: Option, + pub bin_min_rank: Option, + pub bin_max_rank: Option, + pub count: Option, +} + +pub async fn collect_histogram( + db: &mut Connection, + bins: i32, + limit: i32, + filter: bool, +) -> Result, sqlx::Error> { + + sqlx::query_as!( + WhitelistHistogramBin, + "WITH bins AS ( + SELECT generate_series(0, $1 - 1) AS bin +) +SELECT + b.bin as bin_id, + b.bin * $2 + 1 AS bin_min_rank, + (b.bin + 1) * $2 AS bin_max_rank, + COUNT(case when not $3 or w.domain not like '%.co.uk' then 1 end) AS count +FROM bins b +LEFT JOIN whitelist w + ON FLOOR(w.rank / $2) = b.bin +GROUP BY b.bin +ORDER BY b.bin;", bins, limit / bins, filter + ) + .fetch_all(&mut ***db) + .await + .into() +} diff --git a/website/src/main.rs b/website/src/main.rs index 0b7d8ae..1c59ef2 100644 --- a/website/src/main.rs +++ b/website/src/main.rs @@ -1,21 +1,34 @@ -#[macro_use] extern crate rocket; +#[macro_use] +extern crate rocket; +mod agency; +mod db; +mod whitelist; +use crate::db::{check_whitelist, save_query}; +use log::error; +use querying::resolver::Resolver; +use querying::target::Target; +use querying::{Check, CheckError, CheckVerdict, Checker}; +use rocket::fairing::AdHoc; +use rocket::fs::FileServer; +use rocket::http::Status; +use rocket::response::content::RawJavaScript; +use rocket::tokio::sync::RwLock; +use rocket::tokio::time; +use rocket::{fairing, tokio, Build, Request, Rocket, State}; +use rocket_cache_response::CacheResponse; +use rocket_client_addr::ClientRealAddr; +use rocket_db_pools::{Connection, Database}; +use rocket_dyn_templates::{context, Metadata, Template}; +use serde::Serialize; use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; -use rocket::fs::FileServer; -use rocket::http::Status; -use rocket::{tokio, Request, State}; -use rocket::response::content::RawJavaScript; -use rocket_dyn_templates::{context, Metadata, Template}; -use serde::Serialize; -use log::error; -use rocket::tokio::sync::RwLock; -use rocket::tokio::time; -use rocket_cache_response::CacheResponse; -use querying::{Check, CheckError, CheckVerdict, Checker}; -use querying::resolver::Resolver; -use querying::target::Target; +use sqlx::types::Uuid; + +#[derive(rocket_db_pools::Database)] +#[database("cheburcheck")] +struct Db(sqlx::PgPool); #[derive(Serialize)] struct GlobalContext { @@ -24,19 +37,24 @@ struct GlobalContext { impl GlobalContext { fn new() -> Self { - GlobalContext { version: env!("CARGO_PKG_VERSION") } + GlobalContext { + version: env!("CARGO_PKG_VERSION"), + } } } #[get("/")] async fn index(checker: &State>>) -> Template { let checker_ref = checker.read().await; - Template::render("index", context! { - global: GlobalContext::new(), - domain_count: format_number(checker_ref.total_domains().await), - v4_count: format_number(checker_ref.total_v4s().await), - last_update: checker_ref.last_update(), - }) + Template::render( + "index", + context! { + global: GlobalContext::new(), + domain_count: format_number(checker_ref.total_domains().await), + v4_count: format_number(checker_ref.total_v4s().await), + last_update: checker_ref.last_update(), + }, + ) } #[get("/kb/")] @@ -46,9 +64,12 @@ fn page(metadata: Metadata, page: &str) -> Option