From d46c7f7f3c04524c0639c3e0a122626f8e0b3bf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0spik?= Date: Sat, 9 May 2026 01:23:30 +0300 Subject: [PATCH] feat: add embed support for YouTube Shorts (#734) * feat: add embed support for YouTube Shorts Signed-off-by: ispik * feat: blacklist private ip ranges and add january domain blocklist Signed-off-by: IAmTomahawkx * fix: remove duplicates Signed-off-by: ispik --------- Signed-off-by: ispik Signed-off-by: IAmTomahawkx Co-authored-by: IAmTomahawkx --- crates/services/january/src/requests.rs | 10 ++++++++++ crates/services/january/src/website_embed.rs | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/services/january/src/requests.rs b/crates/services/january/src/requests.rs index bec5bc1d..b63f203e 100644 --- a/crates/services/january/src/requests.rs +++ b/crates/services/january/src/requests.rs @@ -33,6 +33,9 @@ lazy_static! { /// Regex for matching new Reddit URLs static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex"); + /// Regex for matching YouTube Shorts URLs + static ref RE_URL_YOUTUBE_SHORTS: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www\\.)?youtube\\.com)/shorts/([a-zA-Z0-9_-]+)").expect("valid regex"); + /// Cache for proxy results static ref PROXY_CACHE: moka::future::Cache)>> = moka::future::Cache::builder() .weigher(|_key, value: &Result<(String, Vec)>| -> u32 { @@ -214,6 +217,13 @@ impl Request { .to_string(); } + // Re-map Youtube Shorts to regular Youtube links + if let Some(captures) = RE_URL_YOUTUBE_SHORTS.captures(&url) { + if let Some(video_id) = captures.get(1) { + url = format!("https://youtube.com/watch?v={}", video_id.as_str()); + } + } + // Generate the actual embed if let Some(hit) = EMBED_CACHE.get(&url).await { Ok(hit) diff --git a/crates/services/january/src/website_embed.rs b/crates/services/january/src/website_embed.rs index 6ab1af58..0e025b73 100644 --- a/crates/services/january/src/website_embed.rs +++ b/crates/services/january/src/website_embed.rs @@ -190,7 +190,7 @@ pub async fn create_website_embed(original_url: &str, document: &str) -> Option< pub async fn populate_special(original_url: String, metadata: &mut WebsiteMetadata) { lazy_static! { - static ref RE_YOUTUBE: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www|m)\\.)?(?:(?:youtube\\.com|youtu.be))(?:/(?:[\\w\\-]+\\?v=|embed/|v/)?)([\\w\\-]+)(?:\\S+)?$").unwrap(); + static ref RE_YOUTUBE: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www|m)\\.)?(?:(?:youtube\\.com|youtu.be))(?:/(?:[\\w\\-]+\\?v=|embed/|v/|shorts/)?)([\\w\\-]+)(?:\\S+)?$").unwrap(); static ref RE_LIGHTSPEED: Regex = Regex::new("^(?:https?://)?(?:[\\w]+\\.)?lightspeed\\.tv/([a-z0-9_]{4,25})").unwrap();