jmug/stoatchat
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

feat: search for links and author type

Browse Source 
Signed-off-by: Zomatree <me@zomatree.live>
This commit is contained in:
Zomatree
2026-04-09 03:54:14 +01:00
parent b7ed543156
commit 9c62568132
27 changed files with 615 additions and 368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
crates/core/database/src/amqp/amqp.rs
View File

@@ -326,10 +326,14 @@ impl AMQP {
.await
}
pub async fn new_message_search(&self, message: Message) -> Result<(), AMQPError> {
pub async fn new_message_search(
&self,
message: Message,
user: Option<User>,
) -> Result<(), AMQPError> {
let config = revolt_config::config().await;
let payload = to_string(&message).unwrap();
let payload = to_string(&MessageCreatePayload { message, user }).unwrap();
debug!(
"Sending new message search payload on channel {}: {}",
@@ -351,10 +355,14 @@ impl AMQP {
.await
}
pub async fn edit_message_search(&self, message: Message) -> Result<(), AMQPError> {
pub async fn edit_message_search(
&self,
message: Message,
user: Option<User>,
) -> Result<(), AMQPError> {
let config = revolt_config::config().await;
let payload = to_string(&message).unwrap();
let payload = to_string(&MessageEditPayload { message, user }).unwrap();
debug!(
"Sending edit message search payload on channel {}: {}",

20
crates/core/database/src/events/rabbit.rs
View File

@@ -3,7 +3,7 @@ use std::collections::HashMap;
use revolt_models::v0::PushNotification;
use serde::{Deserialize, Serialize};
use crate::User;
use crate::{Message, User};
#[derive(Serialize, Deserialize)]
pub struct MessageSentPayload {
@@ -81,10 +81,22 @@ pub struct AckPayload {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MessageDeletePayload {
pub message_id: String
pub message_id: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChannelDeletePayload {
pub channel_id: String
}
pub channel_id: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MessageCreatePayload {
pub message: Message,
pub user: Option<User>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MessageEditPayload {
pub message: Message,
pub user: Option<User>,
}

208
crates/core/database/src/models/messages/model.rs
View File

@@ -1,6 +1,6 @@
use indexmap::{IndexMap, IndexSet};
use iso8601_timestamp::Timestamp;
use revolt_config::{FeaturesLimits, capture_error, config};
use revolt_config::{capture_error, config, FeaturesLimits};
use revolt_models::v0::{
self, BulkMessageResponse, DataMessageSend, Embed, MessageAuthor, MessageFlags, MessageSort,
MessageWebhook, PushNotification, ReplyIntent, SendableEmbed, Text,
@@ -18,7 +18,7 @@ use crate::{
bulk_permissions::BulkDatabasePermissionQuery, idempotency::IdempotencyKey,
permissions::DatabasePermissionQuery,
},
Channel, Database, Emoji, File, User, AMQP,
Channel, Database, Emoji, File, Member, User, AMQP,
};
#[cfg(feature = "tasks")]
@@ -210,6 +210,13 @@ auto_derived!(
pub enum FieldsMessage {
Pinned,
}
/// Message along with the user for the author
pub struct MessageWithUser {
#[serde(flatten)]
pub message: Message,
pub user: Option<User>,
}
);
pub struct MessageFlagsValue(pub u32);
@@ -272,8 +279,8 @@ impl Message {
channel: Channel,
data: DataMessageSend,
author: MessageAuthor<'_>,
user: Option<v0::User>,
member: Option<v0::Member>,
user: Option<User>,
member: Option<Member>,
limits: FeaturesLimits,
mut idempotency: IdempotencyKey,
generate_embeds: bool,
@@ -610,10 +617,10 @@ impl Message {
/// Send a message without any notifications
pub async fn send_without_notifications(
&mut self,
&self,
db: &Database,
user: Option<v0::User>,
member: Option<v0::Member>,
user: Option<User>,
member: Option<Member>,
is_dm: bool,
generate_embeds: bool,
// This determines if this function should queue the mentions task or if somewhere else will.
@@ -623,9 +630,18 @@ impl Message {
db.insert_message(self).await?;
// Fan out events
EventV1::Message(self.clone().into_model(user, member))
.p(self.channel.to_string())
.await;
EventV1::Message(self.clone().into_model(
match user {
Some(user) => {
let is_online = revolt_presence::is_online(&user.id).await;
Some(user.into_known_static(is_online).await)
}
None => None,
},
member.map(Into::into),
))
.p(self.channel.to_string())
.await;
// Update last_message_id
#[cfg(feature = "tasks")]
@@ -673,8 +689,8 @@ impl Message {
db: &Database,
amqp: Option<&AMQP>, // this is optional mostly for tests.
author: MessageAuthor<'_>,
user: Option<v0::User>,
member: Option<v0::Member>,
user: Option<User>,
member: Option<Member>,
channel: &Channel,
generate_embeds: bool,
) -> Result<()> {
@@ -704,7 +720,17 @@ impl Message {
messages: vec![(
Some(
PushNotification::from(
self.clone().into_model(user, member),
self.clone().into_model(
match user.clone() {
Some(user) => {
let is_online =
revolt_presence::is_online(&user.id).await;
Some(user.into_known_static(is_online).await)
}
None => None,
},
member.map(Into::into),
),
Some(author),
channel.to_owned().into(),
)
@@ -727,7 +753,7 @@ impl Message {
}
if let Some(amqp) = amqp {
if let Err(e) = amqp.new_message_search(self.clone()).await {
if let Err(e) = amqp.new_message_search(self.clone(), user).await {
log::error!("Error pushing message to RabbitMQ: {e}");
capture_error(&e);
}
@@ -785,6 +811,7 @@ impl Message {
pub async fn update(
&mut self,
db: &Database,
amqp: Option<&AMQP>,
partial: PartialMessage,
remove: Vec<FieldsMessage>,
) -> Result<()> {
@@ -806,9 +833,74 @@ impl Message {
.p(self.channel.clone())
.await;
if let Some(amqp) = amqp {
if let Err(e) = amqp
.edit_message_search(self.clone(), self.fetch_author(db).await)
.await
{
log::error!("Error pushing message to RabbitMQ: {e}");
capture_error(&e);
}
}
Ok(())
}
pub async fn fetch_users(
db: &Database,
messages: &[Message],
server_id: Option<&str>,
) -> Result<(Vec<User>, Vec<Member>)> {
let user_ids = messages
.iter()
.flat_map(|m| {
let mut users = vec![m.author.clone()];
if let Some(system) = &m.system {
match system {
SystemMessage::ChannelDescriptionChanged { by } => users.push(by.clone()),
SystemMessage::ChannelIconChanged { by } => users.push(by.clone()),
SystemMessage::ChannelOwnershipChanged { from, to, .. } => {
users.push(from.clone());
users.push(to.clone())
}
SystemMessage::ChannelRenamed { by, .. } => users.push(by.clone()),
SystemMessage::UserAdded { by, id, .. }
| SystemMessage::UserRemove { by, id, .. } => {
users.push(by.clone());
users.push(id.clone());
}
SystemMessage::UserBanned { id, .. }
| SystemMessage::UserKicked { id, .. }
| SystemMessage::UserJoined { id, .. }
| SystemMessage::UserLeft { id, .. } => {
users.push(id.clone());
}
SystemMessage::Text { .. } => {}
SystemMessage::MessagePinned { by, .. } => {
users.push(by.clone());
}
SystemMessage::MessageUnpinned { by, .. } => {
users.push(by.clone());
}
SystemMessage::CallStarted { by, .. } => users.push(by.clone()),
}
}
users
})
.collect::<HashSet<String>>()
.into_iter()
.collect::<Vec<String>>();
let users = db.fetch_users(&user_ids).await?;
let members = if let Some(server_id) = server_id {
db.fetch_members(server_id, &user_ids).await?
} else {
Vec::new()
};
Ok((users, members))
}
/// Helper function to fetch many messages with users
pub async fn fetch_with_users(
db: &Database,
@@ -817,74 +909,26 @@ impl Message {
include_users: Option<bool>,
server_id: Option<&str>,
) -> Result<BulkMessageResponse> {
let messages: Vec<v0::Message> = db
.fetch_messages(query)
.await?
.into_iter()
.map(|msg| msg.into_model(None, None))
.collect();
let messages = db.fetch_messages(query).await?;
if let Some(true) = include_users {
let user_ids = messages
.iter()
.flat_map(|m| {
let mut users = vec![m.author.clone()];
if let Some(system) = &m.system {
match system {
v0::SystemMessage::ChannelDescriptionChanged { by } => {
users.push(by.clone())
}
v0::SystemMessage::ChannelIconChanged { by } => users.push(by.clone()),
v0::SystemMessage::ChannelOwnershipChanged { from, to, .. } => {
users.push(from.clone());
users.push(to.clone())
}
v0::SystemMessage::ChannelRenamed { by, .. } => users.push(by.clone()),
v0::SystemMessage::UserAdded { by, id, .. }
| v0::SystemMessage::UserRemove { by, id, .. } => {
users.push(by.clone());
users.push(id.clone());
}
v0::SystemMessage::UserBanned { id, .. }
| v0::SystemMessage::UserKicked { id, .. }
| v0::SystemMessage::UserJoined { id, .. }
| v0::SystemMessage::UserLeft { id, .. } => {
users.push(id.clone());
}
v0::SystemMessage::Text { .. } => {}
v0::SystemMessage::MessagePinned { by, .. } => {
users.push(by.clone());
}
v0::SystemMessage::MessageUnpinned { by, .. } => {
users.push(by.clone());
}
v0::SystemMessage::CallStarted { by, .. } => users.push(by.clone()),
}
}
users
})
.collect::<HashSet<String>>()
.into_iter()
.collect::<Vec<String>>();
let users = User::fetch_many_ids_as_mutuals(db, perspective, &user_ids).await?;
let (users, members) = Message::fetch_users(db, &messages, server_id).await?;
Ok(BulkMessageResponse::MessagesAndUsers {
messages,
users,
members: if let Some(server_id) = server_id {
Some(
db.fetch_members(server_id, &user_ids)
.await?
.into_iter()
.map(Into::into)
.collect(),
)
} else {
None
},
messages: messages
.into_iter()
.map(|msg| msg.into_model(None, None))
.collect(),
users: User::into_mutuals(perspective, users).await,
members: Some(members.into_iter().map(Into::into).collect()),
})
} else {
Ok(BulkMessageResponse::JustMessages(messages))
Ok(BulkMessageResponse::JustMessages(
messages
.into_iter()
.map(|msg| msg.into_model(None, None))
.collect(),
))
}
}
@@ -898,7 +942,9 @@ impl Message {
) -> Result<()> {
if let Some(message) = db.append_message(&id, &append).await? {
if let Some(amqp) = amqp {
if let Err(e) = amqp.edit_message_search(message).await {
let author = message.fetch_author(db).await;
if let Err(e) = amqp.edit_message_search(message, author).await {
log::error!("Error pushing message to RabbitMQ: {e}");
capture_error(&e);
}
@@ -1136,6 +1182,14 @@ impl Message {
FieldsMessage::Pinned => self.pinned = None,
}
}
pub async fn fetch_author(&self, db: &Database) -> Option<User> {
if self.webhook.is_some() {
None
} else {
db.fetch_user(&self.author).await.ok()
}
}
}
impl SystemMessage {

20
crates/core/database/src/models/messages/ops.rs
View File

@@ -1,8 +1,11 @@
use revolt_result::Result;
use std::collections::HashMap;
use std::time::SystemTime;
use revolt_result::Result;
use crate::{AppendMessage, FieldsMessage, Message, MessageQuery, PartialMessage, util::ChunkedDatabaseGenerator};
use crate::{
util::ChunkedDatabaseGenerator, AppendMessage, FieldsMessage, Message, MessageQuery,
MessageWithUser, PartialMessage,
};
#[cfg(feature = "mongodb")]
mod mongodb;
@@ -23,7 +26,12 @@ pub trait AbstractMessages: Sync + Send {
async fn fetch_messages_by_id(&self, ids: &[String]) -> Result<Vec<Message>>;
/// Update a given message with new information
async fn update_message(&self, id: &str, message: &PartialMessage, remove: Vec<FieldsMessage>) -> Result<()>;
async fn update_message(
&self,
id: &str,
message: &PartialMessage,
remove: Vec<FieldsMessage>,
) -> Result<()>;
/// Append information to a given message
async fn append_message(&self, id: &str, append: &AppendMessage) -> Result<Option<Message>>;
@@ -48,7 +56,9 @@ pub trait AbstractMessages: Sync + Send {
&self,
channels: &[String],
author: &str,
since: SystemTime
since: SystemTime,
) -> Result<HashMap<String, Vec<String>>>;
async fn fetch_all_messages(&self) -> Result<ChunkedDatabaseGenerator<Message>>;
/// Fetches all messages along with their author from every message in decending order
async fn fetch_all_messages(&self) -> Result<ChunkedDatabaseGenerator<MessageWithUser>>;
}

34
crates/core/database/src/models/messages/ops/mongodb.rs
View File

@@ -9,8 +9,8 @@ use std::time::SystemTime;
use ulid::Ulid;
use crate::{
AppendMessage, DocumentId, FieldsMessage, IntoDocumentPath, Message, MessageQuery,
MessageTimePeriod, MongoDb, PartialMessage, util::ChunkedDatabaseGenerator,
util::ChunkedDatabaseGenerator, AppendMessage, DocumentId, FieldsMessage, IntoDocumentPath,
Message, MessageQuery, MessageTimePeriod, MessageWithUser, MongoDb, PartialMessage,
};
use super::AbstractMessages;
@@ -417,7 +417,8 @@ impl AbstractMessages for MongoDb {
Ok(deleted_messages)
}
async fn fetch_all_messages(&self) -> Result<ChunkedDatabaseGenerator<Message>> {
/// Fetches all messages along with their author from every message in decending order
async fn fetch_all_messages(&self) -> Result<ChunkedDatabaseGenerator<MessageWithUser>> {
let mut session = self
.start_session()
.await
@@ -431,13 +432,34 @@ impl AbstractMessages for MongoDb {
let cursor = self
.col::<Message>(COL)
.find(doc! {})
.sort(doc ! { "_id": -1i32 })
.aggregate([
doc! {
"$lookup": {
"from": "users",
"localField": "author",
"foreignField": "_id",
"as": "user"
}
},
doc! {
"$set": {
"user": {
"$first": "$user"
}
}
},
doc! {
"$sort": {
"_id": -1
}
},
])
.with_type::<MessageWithUser>()
.session(&mut session)
.batch_size(1000)
.await
.inspect_err(|e| log::error!("{e}"))
.map_err(|_| create_database_error!("find", COL))?;
.map_err(|_| create_database_error!("aggregate", COL))?;
Ok(ChunkedDatabaseGenerator::new_mongo(session, cursor))
}

46
crates/core/database/src/models/messages/ops/reference.rs
View File

@@ -1,13 +1,13 @@
use std::collections::HashMap;
use futures::future::try_join_all;
use indexmap::IndexSet;
use revolt_result::Result;
use std::collections::HashMap;
use std::time::SystemTime;
use ulid::Ulid;
use crate::{
util::ChunkedDatabaseGenerator, AppendMessage, FieldsMessage, Message, MessageQuery,
PartialMessage, ReferenceDb,
MessageWithUser, PartialMessage, ReferenceDb,
};
use super::AbstractMessages;
@@ -64,7 +64,7 @@ impl AbstractMessages for ReferenceDb {
if let Some(pinned) = query.filter.pinned {
if message.pinned.unwrap_or_default() == pinned {
return false
return false;
}
}
@@ -195,7 +195,12 @@ impl AbstractMessages for ReferenceDb {
}
/// Update a given message with new information
async fn update_message(&self, id: &str, message: &PartialMessage, remove: Vec<FieldsMessage>) -> Result<()> {
async fn update_message(
&self,
id: &str,
message: &PartialMessage,
remove: Vec<FieldsMessage>,
) -> Result<()> {
let mut messages = self.messages.lock().await;
if let Some(message_data) = messages.get_mut(id) {
message_data.apply_options(message.to_owned());
@@ -300,7 +305,7 @@ impl AbstractMessages for ReferenceDb {
&self,
channels: &[String],
author: &str,
since: SystemTime
since: SystemTime,
) -> Result<HashMap<String, Vec<String>>> {
let threshold_ulid = Ulid::from_datetime(since).to_string();
let mut deleted_messages: HashMap<String, Vec<String>> = HashMap::new();
@@ -341,22 +346,31 @@ impl AbstractMessages for ReferenceDb {
}
// Delete the messages
self.messages
.lock()
.await
.retain(|id, message| {
let should_keep = !(message.author == author
&& channels.contains(&message.channel)
&& id.as_str() >= threshold_ulid.as_str());
should_keep
});
self.messages.lock().await.retain(|id, message| {
let should_keep = !(message.author == author
&& channels.contains(&message.channel)
&& id.as_str() >= threshold_ulid.as_str());
should_keep
});
Ok(deleted_messages)
}
async fn fetch_all_messages(&self) -> Result<ChunkedDatabaseGenerator<Message>> {
/// Fetches all messages along with their author from every message in decending order
async fn fetch_all_messages(&self) -> Result<ChunkedDatabaseGenerator<MessageWithUser>> {
let users = self.users.lock().await;
Ok(ChunkedDatabaseGenerator::new_reference(
self.messages.lock().await.values().cloned().collect(),
self.messages
.lock()
.await
.values()
.cloned()
.map(|message| MessageWithUser {
user: users.get(&message.author).cloned(),
message,
})
.collect(),
))
}
}

11
crates/core/database/src/models/users/model.rs
View File

@@ -362,6 +362,17 @@ impl User {
)
}
pub async fn into_mutuals(perspective: &User, users: Vec<User>) -> Vec<v0::User> {
let online_ids =
filter_online(&users.iter().map(|user| user.id.clone()).collect::<Vec<_>>()).await;
join_all(users.into_iter().map(|user| async {
let is_online = online_ids.contains(&user.id);
user.into_known(perspective, is_online).await
}))
.await
}
/// Find a free discriminator for a given username
pub async fn find_discriminator(
db: &Database,

27
crates/core/database/src/util/chunked.rs
View File

@@ -1,5 +1,6 @@
#[cfg(feature = "mongodb")]
use ::mongodb::{ClientSession, SessionCursor};
use revolt_result::{Result, ToRevoltError};
use serde::Deserialize;
#[derive(Debug)]
@@ -12,7 +13,7 @@ pub enum ChunkedDatabaseGenerator<T> {
},
Reference {
offset: i32,
offset: usize,
data: Vec<T>,
},
}
@@ -20,33 +21,25 @@ pub enum ChunkedDatabaseGenerator<T> {
impl<T: for<'d> Deserialize<'d> + Clone> ChunkedDatabaseGenerator<T> {
#[cfg(feature = "mongodb")]
pub fn new_mongo(session: ClientSession, cursor: SessionCursor<T>) -> Self {
Self::MongoDb {
session,
cursor,
}
Self::MongoDb { session, cursor }
}
pub fn new_reference(data: Vec<T>) -> Self {
Self::Reference {
offset: 0,
data,
}
Self::Reference { offset: 0, data }
}
pub async fn next(&mut self) -> Option<T> {
pub async fn next(&mut self) -> Result<Option<T>> {
match self {
#[cfg(feature = "mongodb")]
Self::MongoDb { session, cursor } => {
let value = cursor.next(session).await;
value.map(|val| val.expect("Failed to fetch the next message"))
cursor.next(session).await.transpose().to_internal_error()
}
Self::Reference { offset, data } => {
if data.len() as i32 >= *offset {
None
} else {
let resp = &data[*offset as usize];
if let Some(value) = data.get(*offset) {
*offset += 1;
Some(resp.clone())
Ok(Some(value.clone()))
} else {
Ok(None)
}
}
}