diff --git a/Cargo.lock b/Cargo.lock index b5cf3477..2e0ff711 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1084,6 +1084,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "binascii" version = "0.1.4" @@ -1105,21 +1111,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed79c2a8151273c70956b5e3cdfdc1ff6c1a8b9779ba59c6807d281b32ee2f86" -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - [[package]] name = "bit_field" version = "0.10.2" @@ -2257,17 +2248,6 @@ dependencies = [ "zune-inflate", ] -[[package]] -name = "fancy-regex" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" -dependencies = [ - "bit-set", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", -] - [[package]] name = "fastrand" version = "1.9.0" @@ -4119,6 +4099,40 @@ dependencies = [ "value-bag", ] +[[package]] +name = "logos" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6f536c1af4c7cc81edf73da1f8029896e7e1e16a219ef09b184e76a296f3db" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189bbfd0b61330abea797e5e9276408f2edbe4f822d7ad08685d67419aafb34e" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2", + "quote 1.0.38", + "regex-syntax 0.8.5", + "rustc_version", + "syn 2.0.94", +] + +[[package]] +name = "logos-derive" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebfe8e1a19049ddbfccbd14ac834b215e11b85b90bab0c2dba7c7b92fb5d5cba" +dependencies = [ + "logos-codegen", +] + [[package]] name = "loom" version = "0.5.6" @@ -6093,6 +6107,7 @@ dependencies = [ "regex", "revolt-config", "revolt-models", + "revolt-parser", "revolt-permissions", "revolt-presence", "revolt-result", @@ -6213,7 +6228,6 @@ dependencies = [ name = "revolt-models" version = "0.8.4" dependencies = [ - "fancy-regex", "indexmap 1.9.3", "iso8601-timestamp 0.2.17", "num_enum 0.6.1", @@ -6242,6 +6256,13 @@ dependencies = [ "serde", ] +[[package]] +name = "revolt-parser" +version = "0.1.0" +dependencies = [ + "logos", +] + [[package]] name = "revolt-permissions" version = "0.8.4" diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index ebb38c6b..d32e5268 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -36,6 +36,7 @@ revolt-permissions = { version = "0.8.4", path = "../permissions", features = [ "serde", "bson", ] } +revolt-parser = { version = "0.1.0", path = "../parser" } # Utility log = "0.4" diff --git a/crates/core/database/src/models/messages/model.rs b/crates/core/database/src/models/messages/model.rs index fd51d0ae..02ad765f 100644 --- a/crates/core/database/src/models/messages/model.rs +++ b/crates/core/database/src/models/messages/model.rs @@ -5,8 +5,7 @@ use iso8601_timestamp::Timestamp; use revolt_config::{config, FeaturesLimits}; use revolt_models::v0::{ self, BulkMessageResponse, DataMessageSend, Embed, MessageAuthor, MessageFlags, MessageSort, - MessageWebhook, PushNotification, ReplyIntent, SendableEmbed, Text, RE_BLOCKS, RE_MENTION, - RE_ROLE_MENTION, + MessageWebhook, PushNotification, ReplyIntent, SendableEmbed, Text }; use revolt_permissions::{calculate_channel_permissions, ChannelPermission, PermissionValue}; use revolt_result::{ErrorType, Result}; @@ -369,52 +368,30 @@ impl Message { }; // Parse mentions in message. - let mut mentions = HashSet::new(); - let mut role_mentions = HashSet::new(); - let mut role_model_mentions = vec![]; - if let Some(raw_content) = &data.content { - // remove all codeblocks before running the mention regex - let modified_content = RE_BLOCKS.replace_all(raw_content, ""); + let mut message_mentions = if let Some(raw_content) = &data.content { + revolt_parser::parse_message(raw_content) + } else { + revolt_parser::MessageResults::default() + }; - for capture in RE_MENTION.captures_iter(&modified_content).flatten() { - if let Some(mention) = capture.get(1) { - mentions.insert(mention.as_str().to_string()); - } - } - if allow_mass_mentions && server_id.is_some() { - // Second step of mass mention resolution. - for capture in RE_ROLE_MENTION.captures_iter(&modified_content).flatten() { - let raw = capture.get(0).expect("No capture?").as_str(); - if raw == "@everyone" { - mentions_everyone = true; - // Can't break early here since we want to capture any role ids - } else if raw == "@online" { - mentions_online = true; - } else if let Some(role_match) = capture.get(1) { - role_mentions.insert(role_match.as_str().to_string()); - } - } + message_mentions.mentions_everyone |= mentions_everyone; + message_mentions.mentions_online |= mentions_online; - if !role_mentions.is_empty() { - let server_data = db - .fetch_server(server_id.unwrap().as_str()) - .await - .expect("Failed to fetch server"); + let revolt_parser::MessageResults { + mut user_mentions, + mut role_mentions, + mut mentions_everyone, + mut mentions_online + } = message_mentions; - role_mentions = role_mentions - .iter() - .filter(|role_id| server_data.roles.contains_key(*role_id)) - .cloned() - .collect(); + if allow_mass_mentions && server_id.is_some() && !role_mentions.is_empty() { + let server_data = db + .fetch_server(server_id.unwrap().as_str()) + .await + .expect("Failed to fetch server"); - role_model_mentions.extend( - role_mentions - .iter() - .map(|role_id| server_data.roles.get(role_id).to_owned()), - ); - } - } + role_mentions.retain(|role_id| server_data.roles.contains_key(role_id)); } // Validate the user can perform a mass mention @@ -479,7 +456,7 @@ impl Message { // Referenced message exists Ok(message) => { if mention && allow_mentions { - mentions.insert(message.author.to_owned()); + user_mentions.insert(message.author.to_owned()); } replies.insert(message.id); @@ -498,27 +475,25 @@ impl Message { } // Validate the mentions go to users in the channel/server - if !mentions.is_empty() { + if !user_mentions.is_empty() { match channel { Channel::DirectMessage { ref recipients, .. } | Channel::Group { ref recipients, .. } => { - let recipients_hash: HashSet<&String, RandomState> = - HashSet::from_iter(recipients); - mentions.retain(|m| recipients_hash.contains(m)); + let recipients_hash = HashSet::<&String, RandomState>::from_iter(recipients); + user_mentions.retain(|m| recipients_hash.contains(m)); role_mentions.clear(); } Channel::TextChannel { ref server, .. } | Channel::VoiceChannel { ref server, .. } => { - let mentions_vec = Vec::from_iter(mentions.iter().cloned()); + let mentions_vec = Vec::from_iter(user_mentions.iter().cloned()); let valid_members = db.fetch_members(server.as_str(), &mentions_vec[..]).await; if let Ok(valid_members) = valid_members { - let valid_mentions: HashSet<&String, RandomState> = - HashSet::from_iter(valid_members.iter().map(|m| &m.id.user)); + let valid_mentions = HashSet::<&String, RandomState>::from_iter(valid_members.iter().map(|m| &m.id.user)); - mentions.retain(|m| valid_mentions.contains(m)); // quick pass, validate mentions are in the server + user_mentions.retain(|m| valid_mentions.contains(m)); // quick pass, validate mentions are in the server - if !mentions.is_empty() { + if !user_mentions.is_empty() { // if there are still mentions, drill down to a channel-level let member_channel_view_perms = BulkDatabasePermissionQuery::from_server_id(db, server) @@ -528,8 +503,7 @@ impl Message { .members_can_see_channel() .await; - mentions - .retain(|m| *member_channel_view_perms.get(m).unwrap_or(&false)); + user_mentions.retain(|m| *member_channel_view_perms.get(m).unwrap_or(&false)); } } else { revolt_config::capture_error(&valid_members.unwrap_err()); @@ -537,13 +511,13 @@ impl Message { } } Channel::SavedMessages { .. } => { - mentions.clear(); + user_mentions.clear(); } } } - if !mentions.is_empty() { - message.mentions.replace(mentions.into_iter().collect()); + if !user_mentions.is_empty() { + message.mentions.replace(user_mentions.into_iter().collect()); } if !role_mentions.is_empty() { diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 1d4c5db0..c2c667be 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -25,7 +25,6 @@ revolt-permissions = { version = "0.8.4", path = "../permissions" } # Utility regex = "1.11" -fancy-regex = "0.14" indexmap = "1.9.3" once_cell = "1.17.1" num_enum = "0.6.1" diff --git a/crates/core/models/src/v0/messages.rs b/crates/core/models/src/v0/messages.rs index 0c02903d..0324e8b1 100644 --- a/crates/core/models/src/v0/messages.rs +++ b/crates/core/models/src/v0/messages.rs @@ -1,8 +1,6 @@ use std::time::SystemTime; -use fancy_regex::Regex; use indexmap::{IndexMap, IndexSet}; -use once_cell::sync::Lazy; use revolt_config::config; #[cfg(feature = "validator")] @@ -15,15 +13,6 @@ use iso8601_timestamp::Timestamp; use super::{Channel, Embed, File, Member, MessageWebhook, User, Webhook, RE_COLOUR}; -pub static RE_MENTION: Lazy = - Lazy::new(|| Regex::new(r"<@([0-9A-HJKMNP-TV-Z]{26})>").unwrap()); - -pub static RE_ROLE_MENTION: Lazy = - Lazy::new(|| Regex::new(r"<%([0-9A-HJKMNP-TV-Z]{26})>|@online|@everyone").unwrap()); - -pub static RE_BLOCKS: Lazy = - Lazy::new(|| Regex::new(r"(?s)(?>, /// Array of role ids mentioned in this message + #[serde(skip_serializing_if = "Option::is_none")] pub role_mentions: Option>, /// Array of message ids this message is replying to #[serde(skip_serializing_if = "Option::is_none")] diff --git a/crates/core/parser/Cargo.toml b/crates/core/parser/Cargo.toml new file mode 100644 index 00000000..f9b0c700 --- /dev/null +++ b/crates/core/parser/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "revolt-parser" +version = "0.1.0" +edition = "2021" + +[dependencies] +logos = { version = "*" } \ No newline at end of file diff --git a/crates/core/parser/src/lib.rs b/crates/core/parser/src/lib.rs new file mode 100644 index 00000000..191540c1 --- /dev/null +++ b/crates/core/parser/src/lib.rs @@ -0,0 +1,245 @@ +use std::collections::{HashSet, VecDeque}; + +use logos::Logos; + +#[derive(Debug, Clone, Logos, PartialEq)] +#[logos(skip "\n")] +pub enum MessageToken { + #[token("\\")] + Escape, + #[regex("(```[^`\n]*)|(``)|`", |lex| lex.slice().to_owned().chars().filter(|&c| c == '`').count())] + CodeblockMarker(usize), + #[regex("<@[0123456789ABCDEFGHJKMNPQRSTVWXYZ]{26}>", |lex| lex.slice()[2..lex.slice().len() - 1].to_owned())] + UserMention(String), + #[regex("<%[0123456789ABCDEFGHJKMNPQRSTVWXYZ]{26}>", |lex| lex.slice()[2..lex.slice().len() - 1].to_owned())] + RoleMention(String), + #[token("@everyone")] + MentionEveryone, + #[token("@online")] + MentionOnline +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct MessageResults { + pub user_mentions: HashSet, + pub role_mentions: HashSet, + pub mentions_everyone: bool, + pub mentions_online: bool +} + +struct MessageParserIterator { + inner: I, + temp: VecDeque +} + +impl> Iterator for MessageParserIterator { + type Item = MessageToken; + + fn next(&mut self) -> Option { + if !self.temp.is_empty() { + self.temp.pop_front() + } else { + let token = self.inner.next(); + + if token == Some(MessageToken::Escape) { + self.inner.next(); + + token + } else if let Some(MessageToken::CodeblockMarker(ty)) = token { + loop { + let next_token = self.inner.next(); + + if next_token == Some(MessageToken::CodeblockMarker(ty)) { + self.temp.clear(); + self.temp.push_back(MessageToken::CodeblockMarker(ty)); + break next_token + } else if let Some(token) = next_token { + self.temp.push_back(token); + } else { + break Some(MessageToken::CodeblockMarker(ty)) + } + } + } else { + token + } + } + } +} + +pub fn parse_message_iter(text: &str) -> impl Iterator + '_ { + MessageParserIterator { + inner: MessageToken::lexer(text).flatten(), + temp: VecDeque::new() + } +} + +pub fn parse_message(text: &str) -> MessageResults { + let mut results = MessageResults::default(); + + for token in parse_message_iter(text) { + match token { + MessageToken::Escape => {} + MessageToken::CodeblockMarker(_) => {}, + MessageToken::UserMention(id) => { results.user_mentions.insert(id); }, + MessageToken::RoleMention(id) => { results.role_mentions.insert(id); }, + MessageToken::MentionEveryone => results.mentions_everyone = true, + MessageToken::MentionOnline => results.mentions_online = true, + }; + }; + + results +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_no_nodes() { + let output = parse_message_iter("Hello everyone").collect::>(); + + assert_eq!(output.len(), 0); + } + + #[test] + fn test_simple_user_mention() { + let output = parse_message_iter("Hello <@01FD58YK5W7QRV5H3D64KTQYX3>.").collect::>(); + + assert_eq!(output.len(), 1); + assert_eq!(output[0], MessageToken::UserMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string())); + } + + #[test] + fn test_simple_role_mention() { + let output = parse_message_iter("Hello <%01FD58YK5W7QRV5H3D64KTQYX3>.").collect::>(); + + assert_eq!(output.len(), 1); + assert_eq!(output[0], MessageToken::RoleMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string())); + } + + #[test] + fn test_mention_everyone() { + let output = parse_message_iter("Hello @everyone.").collect::>(); + + assert_eq!(output.len(), 1); + assert_eq!(output[0], MessageToken::MentionEveryone); + } + + #[test] + fn test_mention_online() { + let output = parse_message_iter("Hello @online.").collect::>(); + + assert_eq!(output.len(), 1); + assert_eq!(output[0], MessageToken::MentionOnline); + } + + #[test] + fn test_everything() { + let output = parse_message_iter("Hello <@01FD58YK5W7QRV5H3D64KTQYX3>, <%01FD58YK5W7QRV5H3D64KTQYX3>, @everyone and @online.").collect::>(); + + assert_eq!(output.len(), 4); + assert_eq!(output[0], MessageToken::UserMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string())); + assert_eq!(output[1], MessageToken::RoleMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string())); + assert_eq!(output[2], MessageToken::MentionEveryone); + assert_eq!(output[3], MessageToken::MentionOnline); + } + + #[test] + fn test_everything_no_spaces() { + let output = parse_message_iter("<@01FD58YK5W7QRV5H3D64KTQYX3><%01FD58YK5W7QRV5H3D64KTQYX3>@everyone@online").collect::>(); + + assert_eq!(output.len(), 4); + assert_eq!(output[0], MessageToken::UserMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string())); + assert_eq!(output[1], MessageToken::RoleMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string())); + assert_eq!(output[2], MessageToken::MentionEveryone); + assert_eq!(output[3], MessageToken::MentionOnline); + } + + #[test] + fn test_codeblock_no_mentions() { + let output = parse_message_iter("```\n<@01FD58YK5W7QRV5H3D64KTQYX3><%01FD58YK5W7QRV5H3D64KTQYX3>@everyone@online\n```").collect::>(); + + assert_eq!(output.len(), 2); + assert_eq!(output[0], MessageToken::CodeblockMarker(3)); + assert_eq!(output[1], MessageToken::CodeblockMarker(3)); + } + + #[test] + fn test_uncontained_codeblock_should_mention() { + let output = parse_message_iter("```\n<@01FD58YK5W7QRV5H3D64KTQYX3><%01FD58YK5W7QRV5H3D64KTQYX3>@everyone@online").collect::>(); + + assert_eq!(output.len(), 5); + assert_eq!(output[0], MessageToken::CodeblockMarker(3)); + assert_eq!(output[1], MessageToken::UserMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string())); + assert_eq!(output[2], MessageToken::RoleMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string())); + assert_eq!(output[3], MessageToken::MentionEveryone); + assert_eq!(output[4], MessageToken::MentionOnline); + } + + #[test] + fn test_inline_codeblock_no_mentions() { + + let output = parse_message_iter("`<@01FD58YK5W7QRV5H3D64KTQYX3><%01FD58YK5W7QRV5H3D64KTQYX3>@everyone@online`").collect::>(); + + assert_eq!(output.len(), 2); + assert_eq!(output[0], MessageToken::CodeblockMarker(1)); + assert_eq!(output[0], MessageToken::CodeblockMarker(1)); + } + + #[test] + fn test_uncontained_inline_codeblock_should_mention() { + let output = parse_message_iter("`<@01FD58YK5W7QRV5H3D64KTQYX3><%01FD58YK5W7QRV5H3D64KTQYX3>@everyone@online").collect::>(); + + assert_eq!(output.len(), 5); + assert_eq!(output[0], MessageToken::CodeblockMarker(1)); + assert_eq!(output[1], MessageToken::UserMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string())); + assert_eq!(output[2], MessageToken::RoleMention("01FD58YK5W7QRV5H3D64KTQYX3".to_string())); + assert_eq!(output[3], MessageToken::MentionEveryone); + assert_eq!(output[4], MessageToken::MentionOnline); + } + + #[test] + fn test_double_inline_codeblock() { + let output = parse_message_iter("``this should not ping @everyone``").collect::>(); + + assert_eq!(output.len(), 2); + assert_eq!(output[0], MessageToken::CodeblockMarker(2)); + assert_eq!(output[1], MessageToken::CodeblockMarker(2)); + } + + #[test] + fn test_double_inline_codeblock_with_backticks_inside() { + let output = parse_message_iter("``this `should` not `ping` @everyone``").collect::>(); + + assert_eq!(output.len(), 2); + assert_eq!(output[0], MessageToken::CodeblockMarker(2)); + assert_eq!(output[1], MessageToken::CodeblockMarker(2)); + } + + #[test] + fn test_in_middle() { + let output = parse_message_iter("i am not pinging `@everyone`.").collect::>(); + + assert_eq!(output.len(), 2); + assert_eq!(output[0], MessageToken::CodeblockMarker(1)); + assert_eq!(output[1], MessageToken::CodeblockMarker(1)); + } + + #[test] + fn test_escaped_codeblock() { + let output = parse_message_iter("i am ~~not~~ pinging \\`@everyone` ok.").collect::>(); + + assert_eq!(output.len(), 3); + assert_eq!(output[0], MessageToken::Escape); + assert_eq!(output[1], MessageToken::MentionEveryone); + assert_eq!(output[2], MessageToken::CodeblockMarker(1)); + } + + #[test] + fn test_escape_mention() { + let output = parse_message_iter("i wont ping \\@everyone").collect::>(); + + assert_eq!(output.len(), 1); + assert_eq!(output[0], MessageToken::Escape); + } +} \ No newline at end of file