HMC/handmade-revolt-backend
6
0
Fork 0
forked from jmug/stoatchat
Code Pull Requests Actions Packages Releases 2 Activity

feat: blacklist private ip ranges and add january domain blocklist (#731)

Browse Source 
Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>
This commit is contained in:
Tom
2026-05-06 16:25:31 -07:00
committed by GitHub
parent 5378cd22b4
commit 6b41db984b
6 changed files with 299 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

221
Cargo.lock generated
View File

@@ -288,7 +288,7 @@ dependencies = [
"futures-lite", "futures-lite",
"parking", "parking",
"polling", "polling",
"rustix", "rustix 1.1.4",
"slab", "slab",
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
@@ -328,7 +328,7 @@ dependencies = [
"cfg-if", "cfg-if",
"event-listener 5.4.1", "event-listener 5.4.1",
"futures-lite", "futures-lite",
"rustix", "rustix 1.1.4",
] ]
[[package]] [[package]]
@@ -354,7 +354,7 @@ dependencies = [
"cfg-if", "cfg-if",
"futures-core", "futures-core",
"futures-io", "futures-io",
"rustix", "rustix 1.1.4",
"signal-hook-registry", "signal-hook-registry",
"slab", "slab",
"windows-sys 0.61.2", "windows-sys 0.61.2",
@@ -3450,6 +3450,15 @@ dependencies = [
"digest", "digest",
] ]
[[package]]
name = "home"
version = "0.5.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d"
dependencies = [
"windows-sys 0.61.2",
]
[[package]] [[package]]
name = "hostname" name = "hostname"
version = "0.3.1" version = "0.3.1"
@@ -4576,6 +4585,12 @@ dependencies = [
"memchr", "memchr",
] ]
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]] [[package]]
name = "linux-raw-sys" name = "linux-raw-sys"
version = "0.12.1" version = "0.12.1"
@@ -5466,6 +5481,12 @@ dependencies = [
"portable-atomic", "portable-atomic",
] ]
[[package]]
name = "oorandom"
version = "11.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]] [[package]]
name = "opaque-debug" name = "opaque-debug"
version = "0.3.1" version = "0.3.1"
@@ -5682,6 +5703,106 @@ dependencies = [
"digest", "digest",
] ]
[[package]]
name = "pdk-classy"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fa3e632c61a7f8ad1a77c4f52d9a85a89c577881f44e89b33e28163af38e515"
dependencies = [
"bincode",
"futures",
"getrandom 0.2.17",
"http 0.2.12",
"log",
"pdk-proxy-wasm-stub",
"protobuf",
"proxy-wasm",
"serde",
"serde_derive",
"thiserror 1.0.69",
]
[[package]]
name = "pdk-core"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60f2b1d1c8876b54d03d35a18fba1e5172ed8d7f1c379001c11b62c909fdf654"
dependencies = [
"anyhow",
"log",
"pdk-classy",
"pdk-macros",
"pdk-script",
"protobuf",
"protobuf-codegen",
"rmp-serde",
"serde",
"serde_derive",
"serde_json",
"sha2",
"url",
]
[[package]]
name = "pdk-ip-filter-lib"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dab00d1dfe7b232fcb5424c3af23721993b439a51960f1f3152760228f492ec"
dependencies = [
"anyhow",
"ipnet",
"pdk-core",
"thiserror 1.0.69",
]
[[package]]
name = "pdk-macros"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da346bb3e02aad6b6bf31e6e38798c0f3cdf8bd0319fe97d44addc5a0ea5de92"
dependencies = [
"proc-macro2",
"quote 1.0.45",
"syn 1.0.109",
]
[[package]]
name = "pdk-pel"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c60996708c43b91581ea9e6fa324af7eae5a9b8146dd96cacf2868e9260b654"
dependencies = [
"base64 0.22.1",
"getrandom 0.2.17",
"serde_json",
"thiserror 1.0.69",
"uuid",
]
[[package]]
name = "pdk-proxy-wasm-stub"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "469571b12631b71dce33890917bec59f58c53f9d3b05b748d5aa873c950e1702"
[[package]]
name = "pdk-script"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8842ea2908eb9b94ed7daa522cc75fcd5fe3738a20d7ecde67cb7829dbf129e"
dependencies = [
"log",
"num-traits",
"oorandom",
"pdk-classy",
"pdk-pel",
"roxmltree",
"serde",
"serde_json",
"thiserror 1.0.69",
"url",
]
[[package]] [[package]]
name = "pear" name = "pear"
version = "0.2.9" version = "0.2.9"
@@ -6033,7 +6154,7 @@ dependencies = [
"concurrent-queue", "concurrent-queue",
"hermit-abi 0.5.2", "hermit-abi 0.5.2",
"pin-project-lite 0.2.17", "pin-project-lite 0.2.17",
"rustix", "rustix 1.1.4",
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
@@ -6278,6 +6399,67 @@ dependencies = [
"prost", "prost",
] ]
[[package]]
name = "protobuf"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
dependencies = [
"once_cell",
"protobuf-support",
"thiserror 1.0.69",
]
[[package]]
name = "protobuf-codegen"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d3976825c0014bbd2f3b34f0001876604fe87e0c86cd8fa54251530f1544ace"
dependencies = [
"anyhow",
"once_cell",
"protobuf",
"protobuf-parse",
"regex",
"tempfile",
"thiserror 1.0.69",
]
[[package]]
name = "protobuf-parse"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973"
dependencies = [
"anyhow",
"indexmap 2.13.1",
"log",
"protobuf",
"protobuf-support",
"tempfile",
"thiserror 1.0.69",
"which",
]
[[package]]
name = "protobuf-support"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "proxy-wasm"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8d35d9e2bc5104e2e954b149aa1d5f9fa3bb27f73b45b2706020fed101db685"
dependencies = [
"hashbrown 0.16.1",
"log",
]
[[package]] [[package]]
name = "pxfm" name = "pxfm"
version = "0.1.28" version = "0.1.28"
@@ -7055,6 +7237,7 @@ dependencies = [
"lazy_static", "lazy_static",
"mime", "mime",
"moka", "moka",
"pdk-ip-filter-lib",
"regex", "regex",
"reqwest 0.13.2", "reqwest 0.13.2",
"revolt-config", "revolt-config",
@@ -7068,6 +7251,7 @@ dependencies = [
"tokio 1.51.0", "tokio 1.51.0",
"tracing", "tracing",
"tracing-subscriber", "tracing-subscriber",
"url",
"utoipa", "utoipa",
"utoipa-scalar", "utoipa-scalar",
] ]
@@ -7608,6 +7792,19 @@ dependencies = [
"semver", "semver",
] ]
[[package]]
name = "rustix"
version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
"bitflags 2.11.0",
"errno",
"libc",
"linux-raw-sys 0.4.15",
"windows-sys 0.59.0",
]
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "1.1.4" version = "1.1.4"
@@ -7617,7 +7814,7 @@ dependencies = [
"bitflags 2.11.0", "bitflags 2.11.0",
"errno", "errno",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys 0.12.1",
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
@@ -8781,7 +8978,7 @@ dependencies = [
"fastrand 2.4.0", "fastrand 2.4.0",
"getrandom 0.4.2", "getrandom 0.4.2",
"once_cell", "once_cell",
"rustix", "rustix 1.1.4",
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
@@ -9979,6 +10176,18 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
[[package]]
name = "which"
version = "4.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
dependencies = [
"either",
"home",
"once_cell",
"rustix 0.38.44",
]
[[package]] [[package]]
name = "widestring" name = "widestring"
version = "1.2.1" version = "1.2.1"

2
crates/core/config/Revolt.toml
View File

@@ -132,6 +132,8 @@ pkcs8 = ""
key_id = "" key_id = ""
team_id = "" team_id = ""
[january]
blocked_domains = []
[files] [files]
# Encryption key for stored files # Encryption key for stored files

6
crates/core/config/src/lib.rs
View File

@@ -302,6 +302,11 @@ impl Pushd {
} }
} }
#[derive(Deserialize, Debug, Clone)]
pub struct January {
pub blocked_domains: Vec<String>,
}
#[derive(Deserialize, Debug, Clone)] #[derive(Deserialize, Debug, Clone)]
pub struct FilesLimit { pub struct FilesLimit {
pub min_file_size: usize, pub min_file_size: usize,
@@ -421,6 +426,7 @@ pub struct Settings {
pub hosts: Hosts, pub hosts: Hosts,
pub api: Api, pub api: Api,
pub pushd: Pushd, pub pushd: Pushd,
pub january: January,
pub files: Files, pub files: Files,
pub features: Features, pub features: Features,
pub sentry: Sentry, pub sentry: Sentry,

2
crates/services/january/Cargo.toml
View File

@@ -27,6 +27,8 @@ tokio = { workspace = true, features = [] }
# Web requests # Web requests
reqwest = { workspace = true, features = ["json"] } reqwest = { workspace = true, features = ["json"] }
pdk-ip-filter-lib = "1.8.0"
url = { workspace = true }
# Logging # Logging
tracing = { workspace = true } tracing = { workspace = true }

84
crates/services/january/src/requests.rs
View File

@@ -1,12 +1,13 @@
use encoding_rs::{Encoding, UTF_8_INIT}; use encoding_rs::{Encoding, UTF_8_INIT};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use mime::Mime; use mime::Mime;
use pdk_ip_filter_lib::IpFilter;
use regex::Regex; use regex::Regex;
use reqwest::{ use reqwest::{
header::{self, CONTENT_TYPE}, header::{self, CONTENT_TYPE},
redirect, Client, Response, redirect, Client, Response,
}; };
use revolt_config::report_internal_error; use revolt_config::{config, report_internal_error};
use revolt_files::{create_thumbnail, decode_image, image_size_vec, is_valid_image, video_size}; use revolt_files::{create_thumbnail, decode_image, image_size_vec, is_valid_image, video_size};
use revolt_models::v0::{Embed, Image, ImageSize, Video}; use revolt_models::v0::{Embed, Image, ImageSize, Video};
use revolt_result::{create_error, Error, Result}; use revolt_result::{create_error, Error, Result};
@@ -14,6 +15,7 @@ use std::{
io::{Cursor, Write}, io::{Cursor, Write},
time::Duration, time::Duration,
}; };
use url::{Host, Url};
lazy_static! { lazy_static! {
/// Request client /// Request client
@@ -23,7 +25,7 @@ lazy_static! {
.redirect(redirect::Policy::custom(|attempt| { .redirect(redirect::Policy::custom(|attempt| {
if attempt.previous().len() > 5 { // TODO config if attempt.previous().len() > 5 { // TODO config
attempt.error("too many redirects") attempt.error("too many redirects")
} else if attempt.url().host_str() == Some("jan.revolt.chat") { // TODO config } else if attempt.url().host_str() == Some("proxy.stoatusercontent.com") { // TODO config
attempt.stop() attempt.stop()
} else { } else {
attempt.follow() attempt.follow()
@@ -33,10 +35,10 @@ lazy_static! {
.expect("reqwest Client"); .expect("reqwest Client");
/// Spoof User Agent as Discord /// Spoof User Agent as Discord
static ref RE_USER_AGENT_SPOOFING_AS_DISCORD: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:vx|fx)?twitter|(?:fixv|fixup)?x|(?:old\\.|new\\.|www\\.)reddit).com").expect("valid regex"); static ref RE_USER_AGENT_SPOOFING_AS_DISCORD: Regex = Regex::new("^(?:(?:vx|fx)?twitter|(?:fixv|fixup)?x|(?:old\\.|new\\.|www\\.)reddit).com").expect("valid regex");
/// Regex for matching new Reddit URLs /// Regex for matching new Reddit URLs
static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex"); static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex");
/// Cache for proxy results /// Cache for proxy results
static ref PROXY_CACHE: moka::future::Cache<String, Result<(String, Vec<u8>)>> = moka::future::Cache::builder() static ref PROXY_CACHE: moka::future::Cache<String, Result<(String, Vec<u8>)>> = moka::future::Cache::builder()
@@ -59,6 +61,28 @@ lazy_static! {
.max_capacity(10_000) // Cache up to 10k embeds .max_capacity(10_000) // Cache up to 10k embeds
.time_to_live(Duration::from_secs(60)) // For up to 1 minute .time_to_live(Duration::from_secs(60)) // For up to 1 minute
.build(); .build();
static ref IP_BLOCKLIST: IpFilter = IpFilter::block(&[
"10.0.0.0/8", // something something modern problem require modern solutions
"192.168.0.0/16",
"172.16.0.0/16",
"172.17.0.0/16",
"172.18.0.0/16",
"172.19.0.0/16",
"172.20.0.0/16",
"172.21.0.0/16",
"172.22.0.0/16",
"172.23.0.0/16",
"172.24.0.0/16",
"172.25.0.0/16",
"172.26.0.0/16",
"172.27.0.0/16",
"172.28.0.0/16",
"172.29.0.0/16",
"172.30.0.0/16",
"172.31.0.0/16",
"172.32.0.0/16"]
).unwrap();
} }
/// Information about a successful request /// Information about a successful request
@@ -73,7 +97,7 @@ impl Request {
if let Some(hit) = PROXY_CACHE.get(url).await { if let Some(hit) = PROXY_CACHE.get(url).await {
hit hit
} else { } else {
let Request { response, mime } = Request::new(url).await?; let Request { response, mime } = Request::new_from_str(url).await?;
if matches!(mime.type_(), mime::IMAGE | mime::VIDEO) { if matches!(mime.type_(), mime::IMAGE | mime::VIDEO) {
let bytes = report_internal_error!(response.bytes().await); let bytes = report_internal_error!(response.bytes().await);
@@ -135,7 +159,7 @@ impl Request {
let request = if let Some(request) = request { let request = if let Some(request) = request {
request request
} else { } else {
let request = Request::new(url).await?; let request = Request::new_from_str(url).await?;
if matches!(request.mime.type_(), mime::IMAGE) { if matches!(request.mime.type_(), mime::IMAGE) {
request request
} else { } else {
@@ -173,7 +197,7 @@ impl Request {
let response = if let Some(Request { response, .. }) = request { let response = if let Some(Request { response, .. }) = request {
response response
} else { } else {
let Request { response, mime } = Request::new(url).await?; let Request { response, mime } = Request::new_from_str(url).await?;
if matches!(mime.type_(), mime::VIDEO) { if matches!(mime.type_(), mime::VIDEO) {
response response
} else { } else {
@@ -212,7 +236,7 @@ impl Request {
if let Some(hit) = EMBED_CACHE.get(&url).await { if let Some(hit) = EMBED_CACHE.get(&url).await {
Ok(hit) Ok(hit)
} else { } else {
let request = Request::new(&url).await?; let request = Request::new_from_str(&url).await?;
let embed = match (request.mime.type_(), request.mime.subtype()) { let embed = match (request.mime.type_(), request.mime.subtype()) {
(_, mime::HTML) => { (_, mime::HTML) => {
let content_type = request let content_type = request
@@ -255,15 +279,19 @@ impl Request {
} }
/// Send a new request to a service /// Send a new request to a service
pub async fn new(url: &str) -> Result<Request> { pub async fn new(url: Url) -> Result<Request> {
let url_host_str = url.host_str().ok_or(create_error!(ProxyError))?.to_string();
Request::url_is_blacklisted(&url).await?;
let response = CLIENT let response = CLIENT
.get(url) .get(url)
.header( .header(
"User-Agent", "User-Agent",
if RE_USER_AGENT_SPOOFING_AS_DISCORD.is_match(url) { if RE_USER_AGENT_SPOOFING_AS_DISCORD.is_match(&url_host_str) {
"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)" "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"
} else { } else {
"Mozilla/5.0 (compatible; January/2.0; +https://github.com/revoltchat/backend)" "Mozilla/5.0 (compatible; January/2.0; +https://github.com/stoatchat/stoatchat)"
}, },
) )
.header("Accept-Language", "en-US,en;q=0.5") .header("Accept-Language", "en-US,en;q=0.5")
@@ -290,12 +318,44 @@ impl Request {
Ok(Request { response, mime }) Ok(Request { response, mime })
} }
pub async fn new_from_str(url: &str) -> Result<Request> {
let proper_url = Url::parse(url).map_err(|_| create_error!(ProxyError))?;
Request::new(proper_url).await
}
/// Check if something exists /// Check if something exists
pub async fn exists(url: &str) -> bool { pub async fn exists(url: Url) -> bool {
if let Ok(response) = CLIENT.head(url).send().await { if let Ok(response) = CLIENT.head(url).send().await {
response.status().is_success() response.status().is_success()
} else { } else {
false false
} }
} }
pub async fn exists_from_str(url: &str) -> Result<bool> {
let proper_url = Url::parse(url).map_err(|_| create_error!(ProxyError))?;
Ok(Request::exists(proper_url).await)
}
pub async fn url_is_blacklisted(url: &Url) -> Result<()> {
if let Some(host) = url.host() {
match host {
Host::Ipv4(ipv4) => {
let url_str = ipv4.to_string();
if !IP_BLOCKLIST.is_allowed(&url_str) {
return Err(create_error!(InvalidOperation));
}
}
Host::Domain(domain) => {
let config = config().await;
if config.january.blocked_domains.iter().any(|x| x == domain) {
return Err(create_error!(InvalidOperation));
}
}
_ => (),
}
};
Ok(())
}
} }

3
crates/services/january/src/website_embed.rs
View File

@@ -236,11 +236,12 @@ pub async fn populate_special(original_url: String, metadata: &mut WebsiteMetada
metadata.site_name.take(); metadata.site_name.take();
// Verify the video exists // Verify the video exists
if !crate::requests::Request::exists(&format!( if !crate::requests::Request::exists_from_str(&format!(
"http://img.youtube.com/vi/{}/sddefault.jpg", "http://img.youtube.com/vi/{}/sddefault.jpg",
id id
)) ))
.await .await
.unwrap_or(false)
{ {
return; return;
} }