Merge pull request #81 from goaaats/feat/embed_improvements

src/database/entities/microservice/january.rs

@@ -1,6 +1,7 @@
 use crate::util::{
     result::{Error, Result},
     variables::JANUARY_URL,
+    variables::MAX_EMBED_COUNT,
 };
 use linkify::{LinkFinder, LinkKind};
 use regex::Regex;
@@ -103,11 +104,15 @@ impl Embed {
     pub async fn generate(content: String) -> Result<Vec<Embed>> {
         lazy_static! {
             static ref RE_CODE: Regex = Regex::new("```(?:.|\n)+?```|`(?:.|\n)+?`").unwrap();
+            static ref RE_IGNORED: Regex = Regex::new("(<http.+>)").unwrap();
         }
 
         // Ignore code blocks.
         let content = RE_CODE.replace_all(&content, "");
 
+        // Ignore all content between angle brackets starting with http.
+        let content = RE_IGNORED.replace_all(&content, "");
+
         let content = content
             // Ignore quoted lines.
             .split("\n")
@@ -123,35 +128,55 @@ impl Embed {
             .collect::<Vec<&str>>()
             .join("\n");
 
-        // ! FIXME: allow multiple links
-        // ! FIXME: prevent generation if link is surrounded with < >
         let mut finder = LinkFinder::new();
         finder.kinds(&[LinkKind::Url]);
-        let links: Vec<_> = finder.links(&content).collect();
+        let links: Vec<_> = finder.links(&content).take(*MAX_EMBED_COUNT).collect();
 
         if links.len() == 0 {
             return Err(Error::LabelMe);
         }
 
-        let link = &links[0];
+        let mut embeds: Vec<Embed> = Vec::new();
 
-        let client = reqwest::Client::new();
+        let mut link_index = 0;
-        let result = client
-            .get(&format!("{}/embed", *JANUARY_URL))
-            .query(&[("url", link.as_str())])
-            .send()
-            .await;
 
-        match result {
+        // ! FIXME: batch request to january?
-            Err(_) => return Err(Error::LabelMe),
+        while link_index < links.len() {
-            Ok(result) => match result.status() {
+            let link = &links[link_index];
-                reqwest::StatusCode::OK => {
-                    let res: Embed = result.json().await.map_err(|_| Error::InvalidOperation)?;
 
-                    Ok(vec![res])
+            // Check if we already processed this link.
-                }
+            if link_index != 0 && links.iter().take(link_index).any(|x| x.as_str() == link.as_str()) {
-                _ => return Err(Error::LabelMe),
+                link_index = link_index + 1;
-            },
+                continue;
+            }
+
+            let client = reqwest::Client::new();
+            let result = client
+                .get(&format!("{}/embed", *JANUARY_URL))
+                .query(&[("url", link.as_str())])
+                .send()
+                .await;
+
+            if result.is_err() {
+                link_index = link_index + 1;
+                continue;
+            }
+
+            let response = result.unwrap();
+            if response.status().is_success() {
+                let res: Embed = response.json().await.map_err(|_| Error::InvalidOperation)?;
+
+                embeds.push(res);
+            }
+
+            link_index = link_index + 1;
+        }
+
+        // Prevent database update when no embeds are found.
+        if embeds.len() > 0 {
+            Ok(embeds)
+        } else {
+            Err(Error::LabelMe)
         }
     }
 }

src/util/variables.rs

@@ -64,6 +64,8 @@ lazy_static! {
         env::var("REVOLT_MAX_GROUP_SIZE").unwrap_or_else(|_| "50".to_string()).parse().unwrap();
     pub static ref MAX_BOT_COUNT: usize =
         env::var("REVOLT_MAX_BOT_COUNT").unwrap_or_else(|_| "5".to_string()).parse().unwrap();
+    pub static ref MAX_EMBED_COUNT: usize =
+        env::var("REVOLT_MAX_EMBED_COUNT").unwrap_or_else(|_| "5".to_string()).parse().unwrap();
     pub static ref MAX_SERVER_COUNT: usize =
         env::var("REVOLT_MAX_SERVER_COUNT").unwrap_or_else(|_| "100".to_string()).parse().unwrap();
     pub static ref EARLY_ADOPTER_BADGE: i64 =