jmug/stoatchat
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

fix(services/january): put html parsing into block to prevent issues with futures

Browse Source
This commit is contained in:
Paul Makles
2024-10-02 14:09:41 +01:00
parent bb6bcda8bd
commit bb202079e0
1 changed files with 60 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
crates/services/january/src/website_embed.rs
View File

@@ -9,34 +9,38 @@ use scraper::{Html, Selector};
/// Create website metadata from URL and document /// Create website metadata from URL and document
pub async fn create_website_embed(original_url: &str, document: &str) -> Option<WebsiteMetadata> { pub async fn create_website_embed(original_url: &str, document: &str) -> Option<WebsiteMetadata> {
let document = Html::parse_document(document); let (mut meta, mut link) = {
let document = Html::parse_document(document);
// create selectors // create selectors
let meta_selector = Selector::parse("meta").ok()?; let meta_selector = Selector::parse("meta").ok()?;
let link_selector = Selector::parse("link").ok()?; let link_selector = Selector::parse("link").ok()?;
// extract meta tags // extract meta tags
let mut meta = HashMap::new(); let mut meta = HashMap::new();
for el in document.select(&meta_selector) { for el in document.select(&meta_selector) {
let node = el.value(); let node = el.value();
if let (Some(property), Some(content)) = ( if let (Some(property), Some(content)) = (
node.attr("property").or_else(|| node.attr("name")), node.attr("property").or_else(|| node.attr("name")),
node.attr("content"), node.attr("content"),
) { ) {
meta.insert(property.to_string(), content.to_string()); meta.insert(property.to_string(), content.to_string());
}
} }
}
// extract rel links // extract rel links
let mut link = HashMap::new(); let mut link = HashMap::new();
for el in document.select(&link_selector) { for el in document.select(&link_selector) {
let node = el.value(); let node = el.value();
if let (Some(property), Some(content)) = (node.attr("rel"), node.attr("href")) { if let (Some(property), Some(content)) = (node.attr("rel"), node.attr("href")) {
link.insert(property.to_string(), content.to_string()); link.insert(property.to_string(), content.to_string());
}
} }
}
(meta, link)
};
// build metadata // build metadata
let mut metadata = WebsiteMetadata { let mut metadata = WebsiteMetadata {
@@ -137,32 +141,36 @@ pub async fn create_website_embed(original_url: &str, document: &str) -> Option<
}; };
// populate extra metadata for popular websites // populate extra metadata for popular websites
populate_special(original_url.to_owned(), &mut metadata); populate_special(original_url.to_owned(), &mut metadata).await;
// TODO: these do not work because compiler is tripping:
// fetch video size if missing // fetch video size if missing
// if let Some(Video { width, height, url }) = &metadata.video { if metadata.special.is_none() {
// if width == &0 || height == &0 { if let Some(Video { width, height, url }) = &metadata.video {
// metadata.video = match crate::requests::Request::fetch_video_metadata(url, None).await { if width == &0 || height == &0 {
// Ok(Some(video)) => Some(video), metadata.video =
// _ => None, match crate::requests::Request::fetch_video_metadata(url, None).await {
// } Ok(Some(video)) => Some(video),
// } _ => None,
// } }
}
}
}
// fetch image size if missing // fetch image size if missing
// if let Some(Image { if metadata.special.is_none() && metadata.image.is_none() {
// width, height, url, .. if let Some(Image {
// }) = &metadata.image width, height, url, ..
// { }) = &metadata.image
// if width == &0 || height == &0 { {
// metadata.image = match crate::requests::Request::fetch_image_metadata(url, None).await { if width == &0 || height == &0 {
// Ok(Some(image)) => Some(image), metadata.image =
// _ => None, match crate::requests::Request::fetch_image_metadata(url, None).await {
// } Ok(Some(image)) => Some(image),
// } _ => None,
// } }
}
}
}
// truncate data // truncate data
metadata.truncate(); metadata.truncate();
@@ -175,7 +183,7 @@ pub async fn create_website_embed(original_url: &str, document: &str) -> Option<
} }
} }
pub fn populate_special(original_url: String, metadata: &mut WebsiteMetadata) { pub async fn populate_special(original_url: String, metadata: &mut WebsiteMetadata) {
lazy_static! { lazy_static! {
static ref RE_YOUTUBE: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www|m)\\.)?(?:(?:youtube\\.com|youtu.be))(?:/(?:[\\w\\-]+\\?v=|embed/|v/)?)([\\w\\-]+)(?:\\S+)?$").unwrap(); static ref RE_YOUTUBE: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www|m)\\.)?(?:(?:youtube\\.com|youtu.be))(?:/(?:[\\w\\-]+\\?v=|embed/|v/)?)([\\w\\-]+)(?:\\S+)?$").unwrap();
@@ -223,15 +231,14 @@ pub fn populate_special(original_url: String, metadata: &mut WebsiteMetadata) {
metadata.site_name.take(); metadata.site_name.take();
// Verify the video exists // Verify the video exists
// TODO: breaks axum :( if !crate::requests::Request::exists(&format!(
// if !crate::requests::Request::exists(&format!( "http://img.youtube.com/vi/{}/sddefault.jpg",
// "http://img.youtube.com/vi/{}/sddefault.jpg", id
// id ))
// )) .await
// .await {
// { return;
// return; }
// }
} }
if let Some(timestamp_captures) = RE_TIMESTAMP.captures_iter(url).next() { if let Some(timestamp_captures) = RE_TIMESTAMP.captures_iter(url).next() {