forked from jmug/stoatchat
feat: blacklist private ip ranges and add january domain blocklist (#731)
Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>
This commit is contained in:
Tom
GitHub
221
Cargo.lock
generated
221
Cargo.lock
generated
@@ -288,7 +288,7 @@ dependencies = [
|
||||
"futures-lite",
|
||||
"parking",
|
||||
"polling",
|
||||
"rustix",
|
||||
"rustix 1.1.4",
|
||||
"slab",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
@@ -328,7 +328,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"event-listener 5.4.1",
|
||||
"futures-lite",
|
||||
"rustix",
|
||||
"rustix 1.1.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -354,7 +354,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"rustix",
|
||||
"rustix 1.1.4",
|
||||
"signal-hook-registry",
|
||||
"slab",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -3450,6 +3450,15 @@ dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "home"
|
||||
version = "0.5.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hostname"
|
||||
version = "0.3.1"
|
||||
@@ -4576,6 +4585,12 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.4.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
@@ -5466,6 +5481,12 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oorandom"
|
||||
version = "11.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
version = "0.3.1"
|
||||
@@ -5682,6 +5703,106 @@ dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdk-classy"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fa3e632c61a7f8ad1a77c4f52d9a85a89c577881f44e89b33e28163af38e515"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"futures",
|
||||
"getrandom 0.2.17",
|
||||
"http 0.2.12",
|
||||
"log",
|
||||
"pdk-proxy-wasm-stub",
|
||||
"protobuf",
|
||||
"proxy-wasm",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdk-core"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60f2b1d1c8876b54d03d35a18fba1e5172ed8d7f1c379001c11b62c909fdf654"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"log",
|
||||
"pdk-classy",
|
||||
"pdk-macros",
|
||||
"pdk-script",
|
||||
"protobuf",
|
||||
"protobuf-codegen",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdk-ip-filter-lib"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dab00d1dfe7b232fcb5424c3af23721993b439a51960f1f3152760228f492ec"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ipnet",
|
||||
"pdk-core",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdk-macros"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da346bb3e02aad6b6bf31e6e38798c0f3cdf8bd0319fe97d44addc5a0ea5de92"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote 1.0.45",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdk-pel"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c60996708c43b91581ea9e6fa324af7eae5a9b8146dd96cacf2868e9260b654"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"getrandom 0.2.17",
|
||||
"serde_json",
|
||||
"thiserror 1.0.69",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdk-proxy-wasm-stub"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "469571b12631b71dce33890917bec59f58c53f9d3b05b748d5aa873c950e1702"
|
||||
|
||||
[[package]]
|
||||
name = "pdk-script"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8842ea2908eb9b94ed7daa522cc75fcd5fe3738a20d7ecde67cb7829dbf129e"
|
||||
dependencies = [
|
||||
"log",
|
||||
"num-traits",
|
||||
"oorandom",
|
||||
"pdk-classy",
|
||||
"pdk-pel",
|
||||
"roxmltree",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 1.0.69",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pear"
|
||||
version = "0.2.9"
|
||||
@@ -6033,7 +6154,7 @@ dependencies = [
|
||||
"concurrent-queue",
|
||||
"hermit-abi 0.5.2",
|
||||
"pin-project-lite 0.2.17",
|
||||
"rustix",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
@@ -6278,6 +6399,67 @@ dependencies = [
|
||||
"prost",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf"
|
||||
version = "3.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"protobuf-support",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf-codegen"
|
||||
version = "3.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d3976825c0014bbd2f3b34f0001876604fe87e0c86cd8fa54251530f1544ace"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"once_cell",
|
||||
"protobuf",
|
||||
"protobuf-parse",
|
||||
"regex",
|
||||
"tempfile",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf-parse"
|
||||
version = "3.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"indexmap 2.13.1",
|
||||
"log",
|
||||
"protobuf",
|
||||
"protobuf-support",
|
||||
"tempfile",
|
||||
"thiserror 1.0.69",
|
||||
"which",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf-support"
|
||||
version = "3.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
|
||||
dependencies = [
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proxy-wasm"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8d35d9e2bc5104e2e954b149aa1d5f9fa3bb27f73b45b2706020fed101db685"
|
||||
dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pxfm"
|
||||
version = "0.1.28"
|
||||
@@ -7055,6 +7237,7 @@ dependencies = [
|
||||
"lazy_static",
|
||||
"mime",
|
||||
"moka",
|
||||
"pdk-ip-filter-lib",
|
||||
"regex",
|
||||
"reqwest 0.13.2",
|
||||
"revolt-config",
|
||||
@@ -7068,6 +7251,7 @@ dependencies = [
|
||||
"tokio 1.51.0",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"utoipa",
|
||||
"utoipa-scalar",
|
||||
]
|
||||
@@ -7608,6 +7792,19 @@ dependencies = [
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.4.15",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.4"
|
||||
@@ -7617,7 +7814,7 @@ dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
@@ -8781,7 +8978,7 @@ dependencies = [
|
||||
"fastrand 2.4.0",
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
@@ -9979,6 +10176,18 @@ version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "4.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
|
||||
dependencies = [
|
||||
"either",
|
||||
"home",
|
||||
"once_cell",
|
||||
"rustix 0.38.44",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "widestring"
|
||||
version = "1.2.1"
|
||||
|
||||
@@ -132,6 +132,8 @@ pkcs8 = ""
|
||||
key_id = ""
|
||||
team_id = ""
|
||||
|
||||
[january]
|
||||
blocked_domains = []
|
||||
|
||||
[files]
|
||||
# Encryption key for stored files
|
||||
|
||||
@@ -302,6 +302,11 @@ impl Pushd {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct January {
|
||||
pub blocked_domains: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct FilesLimit {
|
||||
pub min_file_size: usize,
|
||||
@@ -421,6 +426,7 @@ pub struct Settings {
|
||||
pub hosts: Hosts,
|
||||
pub api: Api,
|
||||
pub pushd: Pushd,
|
||||
pub january: January,
|
||||
pub files: Files,
|
||||
pub features: Features,
|
||||
pub sentry: Sentry,
|
||||
|
||||
@@ -27,6 +27,8 @@ tokio = { workspace = true, features = [] }
|
||||
|
||||
# Web requests
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
pdk-ip-filter-lib = "1.8.0"
|
||||
url = { workspace = true }
|
||||
|
||||
# Logging
|
||||
tracing = { workspace = true }
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
use encoding_rs::{Encoding, UTF_8_INIT};
|
||||
use lazy_static::lazy_static;
|
||||
use mime::Mime;
|
||||
use pdk_ip_filter_lib::IpFilter;
|
||||
use regex::Regex;
|
||||
use reqwest::{
|
||||
header::{self, CONTENT_TYPE},
|
||||
redirect, Client, Response,
|
||||
};
|
||||
use revolt_config::report_internal_error;
|
||||
use revolt_config::{config, report_internal_error};
|
||||
use revolt_files::{create_thumbnail, decode_image, image_size_vec, is_valid_image, video_size};
|
||||
use revolt_models::v0::{Embed, Image, ImageSize, Video};
|
||||
use revolt_result::{create_error, Error, Result};
|
||||
@@ -14,6 +15,7 @@ use std::{
|
||||
io::{Cursor, Write},
|
||||
time::Duration,
|
||||
};
|
||||
use url::{Host, Url};
|
||||
|
||||
lazy_static! {
|
||||
/// Request client
|
||||
@@ -23,7 +25,7 @@ lazy_static! {
|
||||
.redirect(redirect::Policy::custom(|attempt| {
|
||||
if attempt.previous().len() > 5 { // TODO config
|
||||
attempt.error("too many redirects")
|
||||
} else if attempt.url().host_str() == Some("jan.revolt.chat") { // TODO config
|
||||
} else if attempt.url().host_str() == Some("proxy.stoatusercontent.com") { // TODO config
|
||||
attempt.stop()
|
||||
} else {
|
||||
attempt.follow()
|
||||
@@ -33,10 +35,10 @@ lazy_static! {
|
||||
.expect("reqwest Client");
|
||||
|
||||
/// Spoof User Agent as Discord
|
||||
static ref RE_USER_AGENT_SPOOFING_AS_DISCORD: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:vx|fx)?twitter|(?:fixv|fixup)?x|(?:old\\.|new\\.|www\\.)reddit).com").expect("valid regex");
|
||||
static ref RE_USER_AGENT_SPOOFING_AS_DISCORD: Regex = Regex::new("^(?:(?:vx|fx)?twitter|(?:fixv|fixup)?x|(?:old\\.|new\\.|www\\.)reddit).com").expect("valid regex");
|
||||
|
||||
/// Regex for matching new Reddit URLs
|
||||
static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex");
|
||||
static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex");
|
||||
|
||||
/// Cache for proxy results
|
||||
static ref PROXY_CACHE: moka::future::Cache<String, Result<(String, Vec<u8>)>> = moka::future::Cache::builder()
|
||||
@@ -59,6 +61,28 @@ lazy_static! {
|
||||
.max_capacity(10_000) // Cache up to 10k embeds
|
||||
.time_to_live(Duration::from_secs(60)) // For up to 1 minute
|
||||
.build();
|
||||
|
||||
static ref IP_BLOCKLIST: IpFilter = IpFilter::block(&[
|
||||
"10.0.0.0/8", // something something modern problem require modern solutions
|
||||
"192.168.0.0/16",
|
||||
"172.16.0.0/16",
|
||||
"172.17.0.0/16",
|
||||
"172.18.0.0/16",
|
||||
"172.19.0.0/16",
|
||||
"172.20.0.0/16",
|
||||
"172.21.0.0/16",
|
||||
"172.22.0.0/16",
|
||||
"172.23.0.0/16",
|
||||
"172.24.0.0/16",
|
||||
"172.25.0.0/16",
|
||||
"172.26.0.0/16",
|
||||
"172.27.0.0/16",
|
||||
"172.28.0.0/16",
|
||||
"172.29.0.0/16",
|
||||
"172.30.0.0/16",
|
||||
"172.31.0.0/16",
|
||||
"172.32.0.0/16"]
|
||||
).unwrap();
|
||||
}
|
||||
|
||||
/// Information about a successful request
|
||||
@@ -73,7 +97,7 @@ impl Request {
|
||||
if let Some(hit) = PROXY_CACHE.get(url).await {
|
||||
hit
|
||||
} else {
|
||||
let Request { response, mime } = Request::new(url).await?;
|
||||
let Request { response, mime } = Request::new_from_str(url).await?;
|
||||
|
||||
if matches!(mime.type_(), mime::IMAGE | mime::VIDEO) {
|
||||
let bytes = report_internal_error!(response.bytes().await);
|
||||
@@ -135,7 +159,7 @@ impl Request {
|
||||
let request = if let Some(request) = request {
|
||||
request
|
||||
} else {
|
||||
let request = Request::new(url).await?;
|
||||
let request = Request::new_from_str(url).await?;
|
||||
if matches!(request.mime.type_(), mime::IMAGE) {
|
||||
request
|
||||
} else {
|
||||
@@ -173,7 +197,7 @@ impl Request {
|
||||
let response = if let Some(Request { response, .. }) = request {
|
||||
response
|
||||
} else {
|
||||
let Request { response, mime } = Request::new(url).await?;
|
||||
let Request { response, mime } = Request::new_from_str(url).await?;
|
||||
if matches!(mime.type_(), mime::VIDEO) {
|
||||
response
|
||||
} else {
|
||||
@@ -212,7 +236,7 @@ impl Request {
|
||||
if let Some(hit) = EMBED_CACHE.get(&url).await {
|
||||
Ok(hit)
|
||||
} else {
|
||||
let request = Request::new(&url).await?;
|
||||
let request = Request::new_from_str(&url).await?;
|
||||
let embed = match (request.mime.type_(), request.mime.subtype()) {
|
||||
(_, mime::HTML) => {
|
||||
let content_type = request
|
||||
@@ -255,15 +279,19 @@ impl Request {
|
||||
}
|
||||
|
||||
/// Send a new request to a service
|
||||
pub async fn new(url: &str) -> Result<Request> {
|
||||
pub async fn new(url: Url) -> Result<Request> {
|
||||
let url_host_str = url.host_str().ok_or(create_error!(ProxyError))?.to_string();
|
||||
|
||||
Request::url_is_blacklisted(&url).await?;
|
||||
|
||||
let response = CLIENT
|
||||
.get(url)
|
||||
.header(
|
||||
"User-Agent",
|
||||
if RE_USER_AGENT_SPOOFING_AS_DISCORD.is_match(url) {
|
||||
if RE_USER_AGENT_SPOOFING_AS_DISCORD.is_match(&url_host_str) {
|
||||
"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"
|
||||
} else {
|
||||
"Mozilla/5.0 (compatible; January/2.0; +https://github.com/revoltchat/backend)"
|
||||
"Mozilla/5.0 (compatible; January/2.0; +https://github.com/stoatchat/stoatchat)"
|
||||
},
|
||||
)
|
||||
.header("Accept-Language", "en-US,en;q=0.5")
|
||||
@@ -290,12 +318,44 @@ impl Request {
|
||||
Ok(Request { response, mime })
|
||||
}
|
||||
|
||||
pub async fn new_from_str(url: &str) -> Result<Request> {
|
||||
let proper_url = Url::parse(url).map_err(|_| create_error!(ProxyError))?;
|
||||
Request::new(proper_url).await
|
||||
}
|
||||
|
||||
/// Check if something exists
|
||||
pub async fn exists(url: &str) -> bool {
|
||||
pub async fn exists(url: Url) -> bool {
|
||||
if let Ok(response) = CLIENT.head(url).send().await {
|
||||
response.status().is_success()
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn exists_from_str(url: &str) -> Result<bool> {
|
||||
let proper_url = Url::parse(url).map_err(|_| create_error!(ProxyError))?;
|
||||
Ok(Request::exists(proper_url).await)
|
||||
}
|
||||
|
||||
pub async fn url_is_blacklisted(url: &Url) -> Result<()> {
|
||||
if let Some(host) = url.host() {
|
||||
match host {
|
||||
Host::Ipv4(ipv4) => {
|
||||
let url_str = ipv4.to_string();
|
||||
if !IP_BLOCKLIST.is_allowed(&url_str) {
|
||||
return Err(create_error!(InvalidOperation));
|
||||
}
|
||||
}
|
||||
Host::Domain(domain) => {
|
||||
let config = config().await;
|
||||
if config.january.blocked_domains.iter().any(|x| x == domain) {
|
||||
return Err(create_error!(InvalidOperation));
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
};
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -236,11 +236,12 @@ pub async fn populate_special(original_url: String, metadata: &mut WebsiteMetada
|
||||
metadata.site_name.take();
|
||||
|
||||
// Verify the video exists
|
||||
if !crate::requests::Request::exists(&format!(
|
||||
if !crate::requests::Request::exists_from_str(&format!(
|
||||
"http://img.youtube.com/vi/{}/sddefault.jpg",
|
||||
id
|
||||
))
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user