feat: new parser for mention everything (#411)

* feat: implement custom parser for mentions

* feat: handle backticks in codeblocks and escapes

* chore: remove fancy-regex

* fix: dont serialise empty role mentions

* chore: cleanup role mentions logic

* chore: update lockfile

* fix: actually use everyone and online mentions from content

* fix: broken merge

---------

Signed-off-by: Zomatree <me@zomatree.live>

Cargo.lock

@@ -1084,6 +1084,12 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
 
+[[package]]
+name = "beef"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"
+
 [[package]]
 name = "binascii"
 version = "0.1.4"
@@ -1105,21 +1111,6 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed79c2a8151273c70956b5e3cdfdc1ff6c1a8b9779ba59c6807d281b32ee2f86"
 
-[[package]]
-name = "bit-set"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
-dependencies = [
- "bit-vec",
-]
-
-[[package]]
-name = "bit-vec"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
-
 [[package]]
 name = "bit_field"
 version = "0.10.2"
@@ -2257,17 +2248,6 @@ dependencies = [
  "zune-inflate",
 ]
 
-[[package]]
-name = "fancy-regex"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
-dependencies = [
- "bit-set",
- "regex-automata 0.4.9",
- "regex-syntax 0.8.5",
-]
-
 [[package]]
 name = "fastrand"
 version = "1.9.0"
@@ -4119,6 +4099,40 @@ dependencies = [
  "value-bag",
 ]
 
+[[package]]
+name = "logos"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab6f536c1af4c7cc81edf73da1f8029896e7e1e16a219ef09b184e76a296f3db"
+dependencies = [
+ "logos-derive",
+]
+
+[[package]]
+name = "logos-codegen"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "189bbfd0b61330abea797e5e9276408f2edbe4f822d7ad08685d67419aafb34e"
+dependencies = [
+ "beef",
+ "fnv",
+ "lazy_static",
+ "proc-macro2",
+ "quote 1.0.38",
+ "regex-syntax 0.8.5",
+ "rustc_version",
+ "syn 2.0.94",
+]
+
+[[package]]
+name = "logos-derive"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebfe8e1a19049ddbfccbd14ac834b215e11b85b90bab0c2dba7c7b92fb5d5cba"
+dependencies = [
+ "logos-codegen",
+]
+
 [[package]]
 name = "loom"
 version = "0.5.6"
@@ -6093,6 +6107,7 @@ dependencies = [
  "regex",
  "revolt-config",
  "revolt-models",
+ "revolt-parser",
  "revolt-permissions",
  "revolt-presence",
  "revolt-result",
@@ -6213,7 +6228,6 @@ dependencies = [
 name = "revolt-models"
 version = "0.8.4"
 dependencies = [
- "fancy-regex",
  "indexmap 1.9.3",
  "iso8601-timestamp 0.2.17",
  "num_enum 0.6.1",
@@ -6242,6 +6256,13 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "revolt-parser"
+version = "0.1.0"
+dependencies = [
+ "logos",
+]
+
 [[package]]
 name = "revolt-permissions"
 version = "0.8.4"

crates/core/database/Cargo.toml

@@ -36,6 +36,7 @@ revolt-permissions = { version = "0.8.4", path = "../permissions", features = [
     "serde",
     "bson",
 ] }
+revolt-parser = { version = "0.1.0", path = "../parser" }
 
 # Utility
 log = "0.4"

crates/core/database/src/models/messages/model.rs

@@ -5,8 +5,7 @@ use iso8601_timestamp::Timestamp;
 use revolt_config::{config, FeaturesLimits};
 use revolt_models::v0::{
     self, BulkMessageResponse, DataMessageSend, Embed, MessageAuthor, MessageFlags, MessageSort,
-    MessageWebhook, PushNotification, ReplyIntent, SendableEmbed, Text, RE_BLOCKS, RE_MENTION,
+    MessageWebhook, PushNotification, ReplyIntent, SendableEmbed, Text
-    RE_ROLE_MENTION,
 };
 use revolt_permissions::{calculate_channel_permissions, ChannelPermission, PermissionValue};
 use revolt_result::{ErrorType, Result};
@@ -369,52 +368,30 @@ impl Message {
         };
 
         // Parse mentions in message.
-        let mut mentions = HashSet::new();
-        let mut role_mentions = HashSet::new();
-        let mut role_model_mentions = vec![];
 
-        if let Some(raw_content) = &data.content {
+        let mut message_mentions = if let Some(raw_content) = &data.content {
-            // remove all codeblocks before running the mention regex
+            revolt_parser::parse_message(raw_content)
-            let modified_content = RE_BLOCKS.replace_all(raw_content, "");
+        } else {
+            revolt_parser::MessageResults::default()
+        };
 
-            for capture in RE_MENTION.captures_iter(&modified_content).flatten() {
+        message_mentions.mentions_everyone |= mentions_everyone;
-                if let Some(mention) = capture.get(1) {
+        message_mentions.mentions_online |= mentions_online;
-                    mentions.insert(mention.as_str().to_string());
-                }
-            }
-            if allow_mass_mentions && server_id.is_some() {
-                // Second step of mass mention resolution.
-                for capture in RE_ROLE_MENTION.captures_iter(&modified_content).flatten() {
-                    let raw = capture.get(0).expect("No capture?").as_str();
-                    if raw == "@everyone" {
-                        mentions_everyone = true;
-                        // Can't break early here since we want to capture any role ids
-                    } else if raw == "@online" {
-                        mentions_online = true;
-                    } else if let Some(role_match) = capture.get(1) {
-                        role_mentions.insert(role_match.as_str().to_string());
-                    }
-                }
 
-                if !role_mentions.is_empty() {
+        let revolt_parser::MessageResults {
-                    let server_data = db
+            mut user_mentions,
-                        .fetch_server(server_id.unwrap().as_str())
+            mut role_mentions,
-                        .await
+            mut mentions_everyone,
-                        .expect("Failed to fetch server");
+            mut mentions_online
+        } = message_mentions;
 
-                    role_mentions = role_mentions
+        if allow_mass_mentions && server_id.is_some() && !role_mentions.is_empty() {
-                        .iter()
+            let server_data = db
-                        .filter(|role_id| server_data.roles.contains_key(*role_id))
+                .fetch_server(server_id.unwrap().as_str())
-                        .cloned()
+                .await
-                        .collect();
+                .expect("Failed to fetch server");
 
-                    role_model_mentions.extend(
+            role_mentions.retain(|role_id| server_data.roles.contains_key(role_id));
-                        role_mentions
-                            .iter()
-                            .map(|role_id| server_data.roles.get(role_id).to_owned()),
-                    );
-                }
-            }
         }
 
         // Validate the user can perform a mass mention
@@ -479,7 +456,7 @@ impl Message {
                     // Referenced message exists
                     Ok(message) => {
                         if mention && allow_mentions {
-                            mentions.insert(message.author.to_owned());
+                            user_mentions.insert(message.author.to_owned());
                         }
 
                         replies.insert(message.id);
@@ -498,27 +475,25 @@ impl Message {
         }
 
         // Validate the mentions go to users in the channel/server
-        if !mentions.is_empty() {
+        if !user_mentions.is_empty() {
             match channel {
                 Channel::DirectMessage { ref recipients, .. }
                 | Channel::Group { ref recipients, .. } => {
-                    let recipients_hash: HashSet<&String, RandomState> =
+                    let recipients_hash = HashSet::<&String, RandomState>::from_iter(recipients);
-                        HashSet::from_iter(recipients);
+                    user_mentions.retain(|m| recipients_hash.contains(m));
-                    mentions.retain(|m| recipients_hash.contains(m));
                     role_mentions.clear();
                 }
                 Channel::TextChannel { ref server, .. }
                 | Channel::VoiceChannel { ref server, .. } => {
-                    let mentions_vec = Vec::from_iter(mentions.iter().cloned());
+                    let mentions_vec = Vec::from_iter(user_mentions.iter().cloned());
 
                     let valid_members = db.fetch_members(server.as_str(), &mentions_vec[..]).await;
                     if let Ok(valid_members) = valid_members {
-                        let valid_mentions: HashSet<&String, RandomState> =
+                        let valid_mentions = HashSet::<&String, RandomState>::from_iter(valid_members.iter().map(|m| &m.id.user));
-                            HashSet::from_iter(valid_members.iter().map(|m| &m.id.user));
 
-                        mentions.retain(|m| valid_mentions.contains(m)); // quick pass, validate mentions are in the server
+                        user_mentions.retain(|m| valid_mentions.contains(m)); // quick pass, validate mentions are in the server
 
-                        if !mentions.is_empty() {
+                        if !user_mentions.is_empty() {
                             // if there are still mentions, drill down to a channel-level
                             let member_channel_view_perms =
                                 BulkDatabasePermissionQuery::from_server_id(db, server)
@@ -528,8 +503,7 @@ impl Message {
                                     .members_can_see_channel()
                                     .await;
 
-                            mentions
+                            user_mentions.retain(|m| *member_channel_view_perms.get(m).unwrap_or(&false));
-                                .retain(|m| *member_channel_view_perms.get(m).unwrap_or(&false));
                         }
                     } else {
                         revolt_config::capture_error(&valid_members.unwrap_err());
@@ -537,13 +511,13 @@ impl Message {
                     }
                 }
                 Channel::SavedMessages { .. } => {
-                    mentions.clear();
+                    user_mentions.clear();
                 }
             }
         }
 
-        if !mentions.is_empty() {
+        if !user_mentions.is_empty() {
-            message.mentions.replace(mentions.into_iter().collect());
+            message.mentions.replace(user_mentions.into_iter().collect());
         }
 
         if !role_mentions.is_empty() {

crates/core/models/Cargo.toml

@@ -25,7 +25,6 @@ revolt-permissions = { version = "0.8.4", path = "../permissions" }
 
 # Utility
 regex = "1.11"
-fancy-regex = "0.14"
 indexmap = "1.9.3"
 once_cell = "1.17.1"
 num_enum = "0.6.1"

crates/core/models/src/v0/messages.rs

@@ -1,8 +1,6 @@
 use std::time::SystemTime;
 
-use fancy_regex::Regex;
 use indexmap::{IndexMap, IndexSet};
-use once_cell::sync::Lazy;
 use revolt_config::config;
 
 #[cfg(feature = "validator")]
@@ -15,15 +13,6 @@ use iso8601_timestamp::Timestamp;
 
 use super::{Channel, Embed, File, Member, MessageWebhook, User, Webhook, RE_COLOUR};
 
-pub static RE_MENTION: Lazy<Regex> =
-    Lazy::new(|| Regex::new(r"<@([0-9A-HJKMNP-TV-Z]{26})>").unwrap());
-
-pub static RE_ROLE_MENTION: Lazy<Regex> =
-    Lazy::new(|| Regex::new(r"<%([0-9A-HJKMNP-TV-Z]{26})>|@online|@everyone").unwrap());
-
-pub static RE_BLOCKS: Lazy<Regex> =
-    Lazy::new(|| Regex::new(r"(?s)(?<!\\)(?:(```.+?(?<!\\)```)|`[^`\n\r]+?(?<!\\)`)").unwrap());
-
 auto_derived_partial!(
     /// Message
     pub struct Message {
@@ -65,6 +54,7 @@ auto_derived_partial!(
         #[serde(skip_serializing_if = "Option::is_none")]
         pub mentions: Option<Vec<String>>,
         /// Array of role ids mentioned in this message
+        #[serde(skip_serializing_if = "Option::is_none")]
         pub role_mentions: Option<Vec<String>>,
         /// Array of message ids this message is replying to
         #[serde(skip_serializing_if = "Option::is_none")]

crates/core/parser/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "revolt-parser"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+logos = { version = "*" }

crates/core/parser/src/lib.rs

@@ -0,0 +1,245 @@
+use std::collections::{HashSet, VecDeque};
+
+use logos::Logos;
+
+#[derive(Debug, Clone, Logos, PartialEq)]
+#[logos(skip "\n")]
+pub enum MessageToken {
+    #[token("\\")]
+    Escape,
+    #[regex("(```[^`\n]*)|(``)|`", |lex| lex.slice().to_owned().chars().filter(|&c| c == '`').count())]
+    CodeblockMarker(usize),
+    #[regex("<@[0123456789ABCDEFGHJKMNPQRSTVWXYZ]{26}>", |lex| lex.slice()[2..lex.slice().len() - 1].to_owned())]
+    UserMention(String),
+    #[regex("<%[0123456789ABCDEFGHJKMNPQRSTVWXYZ]{26}>", |lex| lex.slice()[2..lex.slice().len() - 1].to_owned())]
+    RoleMention(String),
+    #[token("@everyone")]
+    MentionEveryone,
+    #[token("@online")]
+    MentionOnline
+}
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct MessageResults {
+    pub user_mentions: HashSet<String>,
+    pub role_mentions: HashSet<String>,
+    pub mentions_everyone: bool,
+    pub mentions_online: bool
+}
+
+struct MessageParserIterator<I> {
+    inner: I,
+    temp: VecDeque<MessageToken>
+}
+
+impl<I: Iterator<Item = MessageToken>> Iterator for MessageParserIterator<I> {
+    type Item = MessageToken;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if !self.temp.is_empty() {
+            self.temp.pop_front()
+        } else {
+            let token = self.inner.next();
+
+            if token == Some(MessageToken::Escape) {
+                self.inner.next();
+
+                token
+            } else if let Some(MessageToken::CodeblockMarker(ty)) = token {
+                loop {
+                    let next_token = self.inner.next();
+
+                    if next_token == Some(MessageToken::CodeblockMarker(ty)) {
+                        self.temp.clear();
+                        self.temp.push_back(MessageToken::CodeblockMarker(ty));
+                        break next_token
+                    } else if let Some(token) = next_token {
+                        self.temp.push_back(token);
+                    } else {
+                        break Some(MessageToken::CodeblockMarker(ty))
+                    }
+                }
+            } else {
+                token
+            }
+        }
+    }
+}
+
+pub fn parse_message_iter(text: &str) -> impl Iterator<Item = MessageToken> + '_ {
+    MessageParserIterator {
+        inner: MessageToken::lexer(text).flatten(),
+        temp: VecDeque::new()
+    }
+}
+
+pub fn parse_message(text: &str) -> MessageResults {
+    let mut results = MessageResults::default();
+
+    for token in parse_message_iter(text) {
+        match token {
+            MessageToken::Escape => {}
+            MessageToken::CodeblockMarker(_) => {},
+            MessageToken::UserMention(id) => { results.user_mentions.insert(id); },
+            MessageToken::RoleMention(id) => { results.role_mentions.insert(id); },
+            MessageToken::MentionEveryone => results.mentions_everyone = true,
+            MessageToken::MentionOnline => results.mentions_online = true,
+        };
+    };
+
+    results
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_no_nodes() {
+        let output = parse_message_iter("Hello everyone").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 0);
+    }
+
+    #[test]
+    fn test_simple_user_mention() {
+        let output = parse_message_iter("Hello <@01FD58YK5W7QRV5H3D64KTQYX3>.").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 1);
+        assert_eq!(output[0], MessageToken::UserMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string()));
+    }
+
+    #[test]
+    fn test_simple_role_mention() {
+        let output = parse_message_iter("Hello <%01FD58YK5W7QRV5H3D64KTQYX3>.").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 1);
+        assert_eq!(output[0], MessageToken::RoleMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string()));
+    }
+
+    #[test]
+    fn test_mention_everyone() {
+        let output = parse_message_iter("Hello @everyone.").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 1);
+        assert_eq!(output[0], MessageToken::MentionEveryone);
+    }
+
+    #[test]
+    fn test_mention_online() {
+        let output = parse_message_iter("Hello @online.").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 1);
+        assert_eq!(output[0], MessageToken::MentionOnline);
+    }
+
+    #[test]
+    fn test_everything() {
+        let output = parse_message_iter("Hello <@01FD58YK5W7QRV5H3D64KTQYX3>, <%01FD58YK5W7QRV5H3D64KTQYX3>, @everyone and @online.").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 4);
+        assert_eq!(output[0], MessageToken::UserMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string()));
+        assert_eq!(output[1], MessageToken::RoleMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string()));
+        assert_eq!(output[2], MessageToken::MentionEveryone);
+        assert_eq!(output[3], MessageToken::MentionOnline);
+    }
+
+    #[test]
+    fn test_everything_no_spaces() {
+        let output = parse_message_iter("<@01FD58YK5W7QRV5H3D64KTQYX3><%01FD58YK5W7QRV5H3D64KTQYX3>@everyone@online").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 4);
+        assert_eq!(output[0], MessageToken::UserMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string()));
+        assert_eq!(output[1], MessageToken::RoleMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string()));
+        assert_eq!(output[2], MessageToken::MentionEveryone);
+        assert_eq!(output[3], MessageToken::MentionOnline);
+    }
+
+    #[test]
+    fn test_codeblock_no_mentions() {
+        let output = parse_message_iter("```\n<@01FD58YK5W7QRV5H3D64KTQYX3><%01FD58YK5W7QRV5H3D64KTQYX3>@everyone@online\n```").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 2);
+        assert_eq!(output[0], MessageToken::CodeblockMarker(3));
+        assert_eq!(output[1], MessageToken::CodeblockMarker(3));
+    }
+
+    #[test]
+    fn test_uncontained_codeblock_should_mention() {
+        let output = parse_message_iter("```\n<@01FD58YK5W7QRV5H3D64KTQYX3><%01FD58YK5W7QRV5H3D64KTQYX3>@everyone@online").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 5);
+        assert_eq!(output[0], MessageToken::CodeblockMarker(3));
+        assert_eq!(output[1], MessageToken::UserMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string()));
+        assert_eq!(output[2], MessageToken::RoleMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string()));
+        assert_eq!(output[3], MessageToken::MentionEveryone);
+        assert_eq!(output[4], MessageToken::MentionOnline);
+    }
+
+    #[test]
+    fn test_inline_codeblock_no_mentions() {
+
+        let output = parse_message_iter("`<@01FD58YK5W7QRV5H3D64KTQYX3><%01FD58YK5W7QRV5H3D64KTQYX3>@everyone@online`").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 2);
+        assert_eq!(output[0], MessageToken::CodeblockMarker(1));
+        assert_eq!(output[0], MessageToken::CodeblockMarker(1));
+    }
+
+    #[test]
+    fn test_uncontained_inline_codeblock_should_mention() {
+        let output = parse_message_iter("`<@01FD58YK5W7QRV5H3D64KTQYX3><%01FD58YK5W7QRV5H3D64KTQYX3>@everyone@online").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 5);
+        assert_eq!(output[0], MessageToken::CodeblockMarker(1));
+        assert_eq!(output[1], MessageToken::UserMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string()));
+        assert_eq!(output[2], MessageToken::RoleMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string()));
+        assert_eq!(output[3], MessageToken::MentionEveryone);
+        assert_eq!(output[4], MessageToken::MentionOnline);
+    }
+
+    #[test]
+    fn test_double_inline_codeblock() {
+        let output = parse_message_iter("``this should not ping @everyone``").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 2);
+        assert_eq!(output[0], MessageToken::CodeblockMarker(2));
+        assert_eq!(output[1], MessageToken::CodeblockMarker(2));
+    }
+
+    #[test]
+    fn test_double_inline_codeblock_with_backticks_inside() {
+        let output = parse_message_iter("``this `should` not `ping` @everyone``").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 2);
+        assert_eq!(output[0], MessageToken::CodeblockMarker(2));
+        assert_eq!(output[1], MessageToken::CodeblockMarker(2));
+    }
+
+    #[test]
+    fn test_in_middle() {
+        let output = parse_message_iter("i am not pinging `@everyone`.").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 2);
+        assert_eq!(output[0], MessageToken::CodeblockMarker(1));
+        assert_eq!(output[1], MessageToken::CodeblockMarker(1));
+    }
+
+    #[test]
+    fn test_escaped_codeblock() {
+        let output = parse_message_iter("i am ~~not~~ pinging \\`@everyone` ok.").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 3);
+        assert_eq!(output[0], MessageToken::Escape);
+        assert_eq!(output[1], MessageToken::MentionEveryone);
+        assert_eq!(output[2], MessageToken::CodeblockMarker(1));
+    }
+
+    #[test]
+    fn test_escape_mention() {
+        let output = parse_message_iter("i wont ping \\@everyone").collect::<Vec<_>>();
+
+        assert_eq!(output.len(), 1);
+        assert_eq!(output[0], MessageToken::Escape);
+    }
+}