From 99a6e19cfcc5ea44a919b2b13cff8b9e3fadc910 Mon Sep 17 00:00:00 2001 From: goat <16760685+goaaats@users.noreply.github.com> Date: Wed, 8 Sep 2021 16:56:17 +0200 Subject: [PATCH 1/3] feat(january): ignore links between angle brackets --- src/database/entities/microservice/january.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/database/entities/microservice/january.rs b/src/database/entities/microservice/january.rs index 39df9ef0..97ffaf62 100644 --- a/src/database/entities/microservice/january.rs +++ b/src/database/entities/microservice/january.rs @@ -100,11 +100,15 @@ impl Embed { pub async fn generate(content: String) -> Result> { lazy_static! { static ref RE_CODE: Regex = Regex::new("```(?:.|\n)+?```|`(?:.|\n)+?`").unwrap(); + static ref RE_IGNORED: Regex = Regex::new("()").unwrap(); } // Ignore code blocks. let content = RE_CODE.replace_all(&content, ""); + // Ignore all content between angle brackets starting with http. + let content = RE_IGNORED.replace_all(&content, ""); + let content = content // Ignore quoted lines. .split("\n") @@ -121,7 +125,6 @@ impl Embed { .join("\n"); // ! FIXME: allow multiple links - // ! FIXME: prevent generation if link is surrounded with < > let mut finder = LinkFinder::new(); finder.kinds(&[LinkKind::Url]); let links: Vec<_> = finder.links(&content).collect(); From 8443cfab00ef91e2cc3f0a4553b3891c0c08a3e2 Mon Sep 17 00:00:00 2001 From: goat <16760685+goaaats@users.noreply.github.com> Date: Wed, 8 Sep 2021 17:59:29 +0200 Subject: [PATCH 2/3] feat(january): fetch embeds for multiple links --- src/database/entities/microservice/january.rs | 58 +++++++++++++------ 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/src/database/entities/microservice/january.rs b/src/database/entities/microservice/january.rs index 97ffaf62..4f92f636 100644 --- a/src/database/entities/microservice/january.rs +++ b/src/database/entities/microservice/january.rs @@ -124,34 +124,56 @@ impl Embed { .collect::>() .join("\n"); - // ! FIXME: allow multiple links let mut finder = LinkFinder::new(); finder.kinds(&[LinkKind::Url]); - let links: Vec<_> = finder.links(&content).collect(); + let links: Vec<_> = finder.links(&content).take(5).collect(); if links.len() == 0 { return Err(Error::LabelMe); } - let link = &links[0]; + let mut embeds: Vec = Vec::new(); - let client = reqwest::Client::new(); - let result = client - .get(&format!("{}/embed", *JANUARY_URL)) - .query(&[("url", link.as_str())]) - .send() - .await; + // ! FIXME: allow configuration of number of embeds + // ! FIXME: batch request to january? + let mut link_index = 0; - match result { - Err(_) => return Err(Error::LabelMe), - Ok(result) => match result.status() { - reqwest::StatusCode::OK => { - let res: Embed = result.json().await.map_err(|_| Error::InvalidOperation)?; + while link_index < links.len() { + let link = &links[link_index]; - Ok(vec![res]) - } - _ => return Err(Error::LabelMe), - }, + // Check if we did the same link already in for this message. + if link_index != 0 && links.iter().take(link_index).any(|x| x.as_str() == link.as_str()) { + link_index = link_index + 1; + continue; + } + + let client = reqwest::Client::new(); + let result = client + .get(&format!("{}/embed", *JANUARY_URL)) + .query(&[("url", link.as_str())]) + .send() + .await; + + if result.is_err() { + link_index = link_index + 1; + continue; + } + + let response = result.unwrap(); + if response.status().is_success() { + let res: Embed = response.json().await.map_err(|_| Error::InvalidOperation)?; + + embeds.push(res); + } + + link_index = link_index + 1; + } + + // Prevent database update when no embeds are found + if embeds.len() > 0 { + Ok(embeds) + } else { + Err(Error::LabelMe) } } } From 6a29c7cf037fdfa053dd956004d37293a3752e66 Mon Sep 17 00:00:00 2001 From: goat <16760685+goaaats@users.noreply.github.com> Date: Wed, 8 Sep 2021 18:02:39 +0200 Subject: [PATCH 3/3] feat(january): add MAX_EMBED_COUNT variable --- src/database/entities/microservice/january.rs | 10 +++++----- src/util/variables.rs | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/database/entities/microservice/january.rs b/src/database/entities/microservice/january.rs index 4f92f636..e9dd37e1 100644 --- a/src/database/entities/microservice/january.rs +++ b/src/database/entities/microservice/january.rs @@ -1,6 +1,7 @@ use crate::util::{ result::{Error, Result}, variables::JANUARY_URL, + variables::MAX_EMBED_COUNT, }; use linkify::{LinkFinder, LinkKind}; use regex::Regex; @@ -126,7 +127,7 @@ impl Embed { let mut finder = LinkFinder::new(); finder.kinds(&[LinkKind::Url]); - let links: Vec<_> = finder.links(&content).take(5).collect(); + let links: Vec<_> = finder.links(&content).take(*MAX_EMBED_COUNT).collect(); if links.len() == 0 { return Err(Error::LabelMe); @@ -134,14 +135,13 @@ impl Embed { let mut embeds: Vec = Vec::new(); - // ! FIXME: allow configuration of number of embeds - // ! FIXME: batch request to january? let mut link_index = 0; + // ! FIXME: batch request to january? while link_index < links.len() { let link = &links[link_index]; - // Check if we did the same link already in for this message. + // Check if we already processed this link. if link_index != 0 && links.iter().take(link_index).any(|x| x.as_str() == link.as_str()) { link_index = link_index + 1; continue; @@ -169,7 +169,7 @@ impl Embed { link_index = link_index + 1; } - // Prevent database update when no embeds are found + // Prevent database update when no embeds are found. if embeds.len() > 0 { Ok(embeds) } else { diff --git a/src/util/variables.rs b/src/util/variables.rs index 8f087a24..bc05a272 100644 --- a/src/util/variables.rs +++ b/src/util/variables.rs @@ -64,6 +64,8 @@ lazy_static! { env::var("REVOLT_MAX_GROUP_SIZE").unwrap_or_else(|_| "50".to_string()).parse().unwrap(); pub static ref MAX_BOT_COUNT: usize = env::var("REVOLT_MAX_BOT_COUNT").unwrap_or_else(|_| "5".to_string()).parse().unwrap(); + pub static ref MAX_EMBED_COUNT: usize = + env::var("REVOLT_MAX_EMBED_COUNT").unwrap_or_else(|_| "5".to_string()).parse().unwrap(); pub static ref EARLY_ADOPTER_BADGE: i64 = env::var("REVOLT_EARLY_ADOPTER_BADGE").unwrap_or_else(|_| "0".to_string()).parse().unwrap(); }