diff --git a/Cargo.lock b/Cargo.lock index ce648f79..3f452c4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -288,7 +288,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix", + "rustix 1.1.4", "slab", "windows-sys 0.61.2", ] @@ -328,7 +328,7 @@ dependencies = [ "cfg-if", "event-listener 5.4.1", "futures-lite", - "rustix", + "rustix 1.1.4", ] [[package]] @@ -354,7 +354,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix", + "rustix 1.1.4", "signal-hook-registry", "slab", "windows-sys 0.61.2", @@ -3450,6 +3450,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "hostname" version = "0.3.1" @@ -4576,6 +4585,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -5466,6 +5481,12 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -5682,6 +5703,106 @@ dependencies = [ "digest", ] +[[package]] +name = "pdk-classy" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fa3e632c61a7f8ad1a77c4f52d9a85a89c577881f44e89b33e28163af38e515" +dependencies = [ + "bincode", + "futures", + "getrandom 0.2.17", + "http 0.2.12", + "log", + "pdk-proxy-wasm-stub", + "protobuf", + "proxy-wasm", + "serde", + "serde_derive", + "thiserror 1.0.69", +] + +[[package]] +name = "pdk-core" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60f2b1d1c8876b54d03d35a18fba1e5172ed8d7f1c379001c11b62c909fdf654" +dependencies = [ + "anyhow", + "log", + "pdk-classy", + "pdk-macros", + "pdk-script", + "protobuf", + "protobuf-codegen", + "rmp-serde", + "serde", + "serde_derive", + "serde_json", + "sha2", + "url", +] + +[[package]] +name = "pdk-ip-filter-lib" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dab00d1dfe7b232fcb5424c3af23721993b439a51960f1f3152760228f492ec" +dependencies = [ + "anyhow", + "ipnet", + "pdk-core", + "thiserror 1.0.69", +] + +[[package]] +name = "pdk-macros" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da346bb3e02aad6b6bf31e6e38798c0f3cdf8bd0319fe97d44addc5a0ea5de92" +dependencies = [ + "proc-macro2", + "quote 1.0.45", + "syn 1.0.109", +] + +[[package]] +name = "pdk-pel" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c60996708c43b91581ea9e6fa324af7eae5a9b8146dd96cacf2868e9260b654" +dependencies = [ + "base64 0.22.1", + "getrandom 0.2.17", + "serde_json", + "thiserror 1.0.69", + "uuid", +] + +[[package]] +name = "pdk-proxy-wasm-stub" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469571b12631b71dce33890917bec59f58c53f9d3b05b748d5aa873c950e1702" + +[[package]] +name = "pdk-script" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8842ea2908eb9b94ed7daa522cc75fcd5fe3738a20d7ecde67cb7829dbf129e" +dependencies = [ + "log", + "num-traits", + "oorandom", + "pdk-classy", + "pdk-pel", + "roxmltree", + "serde", + "serde_json", + "thiserror 1.0.69", + "url", +] + [[package]] name = "pear" version = "0.2.9" @@ -6033,7 +6154,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.5.2", "pin-project-lite 0.2.17", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -6278,6 +6399,67 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf-codegen" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3976825c0014bbd2f3b34f0001876604fe87e0c86cd8fa54251530f1544ace" +dependencies = [ + "anyhow", + "once_cell", + "protobuf", + "protobuf-parse", + "regex", + "tempfile", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf-parse" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973" +dependencies = [ + "anyhow", + "indexmap 2.13.1", + "log", + "protobuf", + "protobuf-support", + "tempfile", + "thiserror 1.0.69", + "which", +] + +[[package]] +name = "protobuf-support" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" +dependencies = [ + "thiserror 1.0.69", +] + +[[package]] +name = "proxy-wasm" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8d35d9e2bc5104e2e954b149aa1d5f9fa3bb27f73b45b2706020fed101db685" +dependencies = [ + "hashbrown 0.16.1", + "log", +] + [[package]] name = "pxfm" version = "0.1.28" @@ -7055,6 +7237,7 @@ dependencies = [ "lazy_static", "mime", "moka", + "pdk-ip-filter-lib", "regex", "reqwest 0.13.2", "revolt-config", @@ -7068,6 +7251,7 @@ dependencies = [ "tokio 1.51.0", "tracing", "tracing-subscriber", + "url", "utoipa", "utoipa-scalar", ] @@ -7608,6 +7792,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -7617,7 +7814,7 @@ dependencies = [ "bitflags 2.11.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -8781,7 +8978,7 @@ dependencies = [ "fastrand 2.4.0", "getrandom 0.4.2", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -9979,6 +10176,18 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "widestring" version = "1.2.1" diff --git a/crates/core/config/Revolt.toml b/crates/core/config/Revolt.toml index 30a4622e..aee3a841 100644 --- a/crates/core/config/Revolt.toml +++ b/crates/core/config/Revolt.toml @@ -132,6 +132,8 @@ pkcs8 = "" key_id = "" team_id = "" +[january] +blocked_domains = [] [files] # Encryption key for stored files diff --git a/crates/core/config/src/lib.rs b/crates/core/config/src/lib.rs index a49b1cdf..b0cfa455 100644 --- a/crates/core/config/src/lib.rs +++ b/crates/core/config/src/lib.rs @@ -302,6 +302,11 @@ impl Pushd { } } +#[derive(Deserialize, Debug, Clone)] +pub struct January { + pub blocked_domains: Vec, +} + #[derive(Deserialize, Debug, Clone)] pub struct FilesLimit { pub min_file_size: usize, @@ -421,6 +426,7 @@ pub struct Settings { pub hosts: Hosts, pub api: Api, pub pushd: Pushd, + pub january: January, pub files: Files, pub features: Features, pub sentry: Sentry, diff --git a/crates/services/january/Cargo.toml b/crates/services/january/Cargo.toml index 78ec07ad..7059cca0 100644 --- a/crates/services/january/Cargo.toml +++ b/crates/services/january/Cargo.toml @@ -27,6 +27,8 @@ tokio = { workspace = true, features = [] } # Web requests reqwest = { workspace = true, features = ["json"] } +pdk-ip-filter-lib = "1.8.0" +url = { workspace = true } # Logging tracing = { workspace = true } diff --git a/crates/services/january/src/requests.rs b/crates/services/january/src/requests.rs index cf681171..b2b75430 100644 --- a/crates/services/january/src/requests.rs +++ b/crates/services/january/src/requests.rs @@ -1,12 +1,13 @@ use encoding_rs::{Encoding, UTF_8_INIT}; use lazy_static::lazy_static; use mime::Mime; +use pdk_ip_filter_lib::IpFilter; use regex::Regex; use reqwest::{ header::{self, CONTENT_TYPE}, redirect, Client, Response, }; -use revolt_config::report_internal_error; +use revolt_config::{config, report_internal_error}; use revolt_files::{create_thumbnail, decode_image, image_size_vec, is_valid_image, video_size}; use revolt_models::v0::{Embed, Image, ImageSize, Video}; use revolt_result::{create_error, Error, Result}; @@ -14,6 +15,7 @@ use std::{ io::{Cursor, Write}, time::Duration, }; +use url::{Host, Url}; lazy_static! { /// Request client @@ -23,7 +25,7 @@ lazy_static! { .redirect(redirect::Policy::custom(|attempt| { if attempt.previous().len() > 5 { // TODO config attempt.error("too many redirects") - } else if attempt.url().host_str() == Some("jan.revolt.chat") { // TODO config + } else if attempt.url().host_str() == Some("proxy.stoatusercontent.com") { // TODO config attempt.stop() } else { attempt.follow() @@ -33,10 +35,10 @@ lazy_static! { .expect("reqwest Client"); /// Spoof User Agent as Discord - static ref RE_USER_AGENT_SPOOFING_AS_DISCORD: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:vx|fx)?twitter|(?:fixv|fixup)?x|(?:old\\.|new\\.|www\\.)reddit).com").expect("valid regex"); + static ref RE_USER_AGENT_SPOOFING_AS_DISCORD: Regex = Regex::new("^(?:(?:vx|fx)?twitter|(?:fixv|fixup)?x|(?:old\\.|new\\.|www\\.)reddit).com").expect("valid regex"); /// Regex for matching new Reddit URLs - static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex"); + static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex"); /// Cache for proxy results static ref PROXY_CACHE: moka::future::Cache)>> = moka::future::Cache::builder() @@ -59,6 +61,28 @@ lazy_static! { .max_capacity(10_000) // Cache up to 10k embeds .time_to_live(Duration::from_secs(60)) // For up to 1 minute .build(); + + static ref IP_BLOCKLIST: IpFilter = IpFilter::block(&[ + "10.0.0.0/8", // something something modern problem require modern solutions + "192.168.0.0/16", + "172.16.0.0/16", + "172.17.0.0/16", + "172.18.0.0/16", + "172.19.0.0/16", + "172.20.0.0/16", + "172.21.0.0/16", + "172.22.0.0/16", + "172.23.0.0/16", + "172.24.0.0/16", + "172.25.0.0/16", + "172.26.0.0/16", + "172.27.0.0/16", + "172.28.0.0/16", + "172.29.0.0/16", + "172.30.0.0/16", + "172.31.0.0/16", + "172.32.0.0/16"] + ).unwrap(); } /// Information about a successful request @@ -73,7 +97,7 @@ impl Request { if let Some(hit) = PROXY_CACHE.get(url).await { hit } else { - let Request { response, mime } = Request::new(url).await?; + let Request { response, mime } = Request::new_from_str(url).await?; if matches!(mime.type_(), mime::IMAGE | mime::VIDEO) { let bytes = report_internal_error!(response.bytes().await); @@ -135,7 +159,7 @@ impl Request { let request = if let Some(request) = request { request } else { - let request = Request::new(url).await?; + let request = Request::new_from_str(url).await?; if matches!(request.mime.type_(), mime::IMAGE) { request } else { @@ -173,7 +197,7 @@ impl Request { let response = if let Some(Request { response, .. }) = request { response } else { - let Request { response, mime } = Request::new(url).await?; + let Request { response, mime } = Request::new_from_str(url).await?; if matches!(mime.type_(), mime::VIDEO) { response } else { @@ -212,7 +236,7 @@ impl Request { if let Some(hit) = EMBED_CACHE.get(&url).await { Ok(hit) } else { - let request = Request::new(&url).await?; + let request = Request::new_from_str(&url).await?; let embed = match (request.mime.type_(), request.mime.subtype()) { (_, mime::HTML) => { let content_type = request @@ -255,15 +279,19 @@ impl Request { } /// Send a new request to a service - pub async fn new(url: &str) -> Result { + pub async fn new(url: Url) -> Result { + let url_host_str = url.host_str().ok_or(create_error!(ProxyError))?.to_string(); + + Request::url_is_blacklisted(&url).await?; + let response = CLIENT .get(url) .header( "User-Agent", - if RE_USER_AGENT_SPOOFING_AS_DISCORD.is_match(url) { + if RE_USER_AGENT_SPOOFING_AS_DISCORD.is_match(&url_host_str) { "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)" } else { - "Mozilla/5.0 (compatible; January/2.0; +https://github.com/revoltchat/backend)" + "Mozilla/5.0 (compatible; January/2.0; +https://github.com/stoatchat/stoatchat)" }, ) .header("Accept-Language", "en-US,en;q=0.5") @@ -290,12 +318,44 @@ impl Request { Ok(Request { response, mime }) } + pub async fn new_from_str(url: &str) -> Result { + let proper_url = Url::parse(url).map_err(|_| create_error!(ProxyError))?; + Request::new(proper_url).await + } + /// Check if something exists - pub async fn exists(url: &str) -> bool { + pub async fn exists(url: Url) -> bool { if let Ok(response) = CLIENT.head(url).send().await { response.status().is_success() } else { false } } + + pub async fn exists_from_str(url: &str) -> Result { + let proper_url = Url::parse(url).map_err(|_| create_error!(ProxyError))?; + Ok(Request::exists(proper_url).await) + } + + pub async fn url_is_blacklisted(url: &Url) -> Result<()> { + if let Some(host) = url.host() { + match host { + Host::Ipv4(ipv4) => { + let url_str = ipv4.to_string(); + if !IP_BLOCKLIST.is_allowed(&url_str) { + return Err(create_error!(InvalidOperation)); + } + } + Host::Domain(domain) => { + let config = config().await; + if config.january.blocked_domains.iter().any(|x| x == domain) { + return Err(create_error!(InvalidOperation)); + } + } + _ => (), + } + }; + + Ok(()) + } } diff --git a/crates/services/january/src/website_embed.rs b/crates/services/january/src/website_embed.rs index 25de1537..6ab1af58 100644 --- a/crates/services/january/src/website_embed.rs +++ b/crates/services/january/src/website_embed.rs @@ -236,11 +236,12 @@ pub async fn populate_special(original_url: String, metadata: &mut WebsiteMetada metadata.site_name.take(); // Verify the video exists - if !crate::requests::Request::exists(&format!( + if !crate::requests::Request::exists_from_str(&format!( "http://img.youtube.com/vi/{}/sddefault.jpg", id )) .await + .unwrap_or(false) { return; }