jmug/stoatchat
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

feat: Add Mass Mentions to the backend (#394)

Browse Source 
* feat: create base of push daemon

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* Add outbound senders

* Make web_push send to rabbit instead (temp stuff)

* feat: stability and friend requests

* make vapid fr stuff not suck

* swap naming of queue

* move pushd into daemons folder

* fix cargo file for move into daemons folder

* feat: probably working fcm push notifs

* comment out fcm webpush stuff since the config keys dont exist

* fix fcm, name queues according to their prod status and configure routing keys

* add pushd to docker

* mix: Remove old code, add stuff to pushd

* fix: lockfile

* feat: update rocket to 5.0.1

* fix: fix queues and ack bugs

* Move rabbit messsage processing into ack queue

* chore: update readme

* chore: optimizations for ack database hits

* pushd flowchart

* misc: update flowchart

* exit dependancy hell

* add rocket_impl flag to authifier

* make the tests file of delta actually compile

* fix: don't silence every push message

* fix: don't silence all messages

* add debug logging for sending data to rabbit from message events

* validate mentions at a server membership level

* put back that import that was actually important

* minor fix to lockfile

* update delta authifier

* feat: proper permissions for push notifications

* add unit test for mention sanitization

* remove local file dependancy on authifier

* update ports to proper defaults

* fixTM the node bindings

* Theoretically configure docker releases for pushd

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* declare exchange in pushd and delta

* fix createbuckets script

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: reference db implementation

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: remove finally redundant code

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: changes

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: other changes

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: make channel name return result

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* Add role mention parsing

* feat: update to mongo 3.1, add member generator.

* integrate mass mentions into pushd

* patch redis-rs with updated versions

* feat: chunk role mentions

* move permission bits to 37/38 to avoid livekit conflict

* change role mention format to <%id>

* fix the lockfile from merge

* fix: PR change requests

* feat: add tests

* fix: i am a dumbass

* fix: tests, again

---------

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>
This commit is contained in:
Tom
2025-04-05 10:45:04 -07:00
committed by Zomatree
parent a110a087be
commit 2a36f5912d
51 changed files with 3530 additions and 1522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
crates/daemons/pushd/Cargo.toml
View File

@@ -10,6 +10,9 @@ revolt-database = { version = "0.8.4", path = "../../core/database" }
revolt-models = { version = "0.8.4", path = "../../core/models", features = [
"validator",
] }
revolt-presence = { version = "0.8.1", path = "../../core/presence", features = [
"redis-is-patched",
] }
amqprs = { version = "1.7.0" }
fcm_v1 = "0.3.0"
@@ -20,9 +23,10 @@ tokio = "1.39.2"
async-trait = "0.1.81"
ulid = "1.0.0"
authifier = "1.0.8"
authifier = "1.0.10"
log = "0.4.11"
pretty_env_logger = "0.4.0"
#serialization
serde_json = "1"

BIN
crates/daemons/pushd/Pushd Flowchart.graffle
View File

Binary file not shown.

10
crates/daemons/pushd/src/consumers/inbound/ack.rs
View File

@@ -71,8 +71,14 @@ impl AsyncConsumer for AckConsumer {
#[allow(clippy::disallowed_methods)]
let unreads = self.db.fetch_unread_mentions(&payload.user_id).await;
debug!("Processing unreads for {:}", &payload.user_id);
if let Ok(u) = &unreads {
if u.is_empty() {
debug!(
"Discarding unread task (no mentions found) for {:}",
&payload.user_id
);
return;
}
} else {
@@ -95,6 +101,10 @@ impl AsyncConsumer for AckConsumer {
.collect();
if apple_sessions.is_empty() {
debug!(
"Discarding unread task (no apn sessions found) for {:}",
&payload.user_id
);
return;
}

275
crates/daemons/pushd/src/consumers/inbound/mass_mention.rs Normal file
View File

@@ -0,0 +1,275 @@
use std::{
collections::{HashMap, HashSet},
hash::RandomState,
};
use crate::consumers::inbound::internal::*;
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use async_trait::async_trait;
use revolt_database::{
events::rabbit::*, util::bulk_permissions::BulkDatabasePermissionQuery, Database, Member,
MessageFlagsValue,
};
use revolt_models::v0::{MessageFlags, PushNotification};
pub struct MassMessageConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
}
impl Channeled for MassMessageConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl MassMessageConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> MassMessageConsumer {
MassMessageConsumer {
db,
authifier_db,
conn: None,
channel: None,
}
}
async fn fire_notification_for_users(&mut self, push: &PushNotification, users: &[String]) {
if let Ok(sessions) = self
.authifier_db
.find_sessions_with_subscription(users)
.await
{
let config = revolt_config::config().await;
for session in sessions {
if let Some(sub) = session.subscription {
let mut sendable = PayloadToService {
notification: PayloadKind::MessageNotification(push.clone()),
token: sub.auth,
user_id: session.user_id,
session_id: session.id,
extras: HashMap::new(),
};
let args: BasicPublishArguments;
if sub.endpoint == "apn" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
} else if sub.endpoint == "fcm" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.fcm.queue.as_str(),
)
.finish();
} else {
// web push (vapid)
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.vapid.queue.as_str(),
)
.finish();
sendable.extras.insert("p265dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), sub.endpoint.clone());
}
let payload = serde_json::to_string(&sendable).unwrap();
publish_message(self, payload.into(), args).await;
}
}
}
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for MassMessageConsumer {
/// This consumer handles adding mentions for all the users affected by a mass mention ping, and then sends out push notifications
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
let config = revolt_config::config().await;
let content = String::from_utf8(content).unwrap();
let payload: MassMessageSentPayload = serde_json::from_str(content.as_str()).unwrap();
debug!("Received mass message event");
// We should only ever receive clumped messages from a single channel, so it's safe to reuse this many times.
let mut query: Option<BulkDatabasePermissionQuery<'_>> = None;
let query_db = self.db.clone();
for push in payload.notifications {
if query.is_none() {
query = Some(
BulkDatabasePermissionQuery::from_server_id(&query_db, &payload.server_id)
.await
.from_channel_id(push.channel.id().to_string()) // wrong channel model, so fetch the right one
.await,
);
}
let existing_mentions: HashSet<String, RandomState> =
if let Some(ref mentions) = push.message.mentions {
HashSet::from_iter(mentions.iter().cloned())
} else {
HashSet::new()
};
// KNOWN QUIRK: if you mention @online and role(s), the offline members with the role(s) wont get pinged
if let Some(ref query) = query {
let flags = MessageFlagsValue(push.message.flags);
if flags.has(MessageFlags::MentionsEveryone) {
let mut db_query = self
.db
.fetch_all_members_chunked(&payload.server_id)
.await
.expect("Failed to fetch members from database");
let mut exhausted = false;
let ack_chnl = vec![push.channel.id().to_string()];
loop {
let mut chunk: Vec<Member> = vec![];
for _ in 0..config.pushd.mass_mention_chunk_size {
if let Some(member) = db_query.next().await {
chunk.push(member);
} else {
exhausted = true;
break;
}
}
let userids: Vec<String> =
chunk.iter().map(|member| member.id.user.clone()).collect();
debug!("Userids in chunk: {:?}", userids);
if let Err(err) = self
.db
.add_mention_to_many_unreads(push.channel.id(), &userids, &ack_chnl)
.await
{
revolt_config::capture_error(&err);
}
// ignore anyone in this list
let online_users = revolt_presence::filter_online(&userids).await;
let target_users: Vec<String> = userids
.iter()
.filter(|id| {
!online_users.contains(*id) && !existing_mentions.contains(*id)
})
.cloned()
.collect();
debug!(
"Userids after filter: {:?} (online: {:?}",
target_users, online_users
);
self.fire_notification_for_users(&push, &target_users).await;
if exhausted {
break;
}
}
} else if let Some(roles) = &push.message.role_mentions {
// role mentions
let _role_members = self
.db
.fetch_all_members_with_roles_chunked(&payload.server_id, roles)
.await;
debug!("role members: {:?}", _role_members);
if _role_members.is_err() {
revolt_config::capture_error(&_role_members.err().unwrap());
return;
}
let mut role_members = _role_members.unwrap();
let mut chunk = vec![];
let mut exhausted = false;
while !exhausted {
chunk.clear();
for _ in 0..config.pushd.mass_mention_chunk_size {
if let Some(member) = role_members.next().await {
chunk.push(member);
} else {
exhausted = true;
break;
}
}
let mut q = query.clone().members(&chunk);
let viewing_members: Vec<String> = q
.members_can_see_channel()
.await
.iter()
.filter_map(|(uid, viewable)| {
if *viewable && !existing_mentions.contains(uid) {
Some(uid.clone())
} else {
None
}
})
.collect();
debug!("viewing members: {:?}", viewing_members);
let online = revolt_presence::filter_online(&viewing_members).await;
debug!("online: {:?}", online);
let targets: Vec<String> = viewing_members
.iter()
.filter(|m| !online.contains(*m))
.cloned()
.collect();
debug!("targets: {:?}", targets);
self.fire_notification_for_users(&push, &targets).await;
}
}
}
}
}
}

1
crates/daemons/pushd/src/consumers/inbound/mod.rs
View File

@@ -3,4 +3,5 @@ pub mod fr_accepted;
pub mod fr_received;
pub mod generic;
mod internal;
pub mod mass_mention;
pub mod message;

23
crates/daemons/pushd/src/consumers/outbound/apn.rs
View File

@@ -82,7 +82,7 @@ impl ApnsOutboundConsumer {
}
}
println!("Got badge count for APN: {}", mention_count);
debug!("Got badge count for APN: {}", mention_count);
return Some(mention_count);
}
@@ -189,6 +189,10 @@ impl AsyncConsumer for ApnsOutboundConsumer {
data: BTreeMap::new(),
};
debug!(
"Sending friend request received for user: {:}",
&payload.user_id
);
resp = self.client.send(apn_payload).await;
}
@@ -231,6 +235,10 @@ impl AsyncConsumer for ApnsOutboundConsumer {
data: BTreeMap::new(),
};
debug!(
"Sending friend request accept for user: {:}",
&payload.user_id
);
resp = self.client.send(apn_payload).await;
}
PayloadKind::Generic(alert) => {
@@ -260,6 +268,10 @@ impl AsyncConsumer for ApnsOutboundConsumer {
data: BTreeMap::new(),
};
debug!(
"Sending generic notification for user: {:}",
&payload.user_id
);
resp = self.client.send(apn_payload).await;
}
@@ -295,6 +307,10 @@ impl AsyncConsumer for ApnsOutboundConsumer {
channel_name: alert.channel.name().unwrap_or(&title),
};
debug!(
"Sending message notification for user: {:}",
&payload.user_id
);
resp = self.client.send(apn_payload).await;
}
PayloadKind::BadgeUpdate(badge) => {
@@ -308,6 +324,7 @@ impl AsyncConsumer for ApnsOutboundConsumer {
data: BTreeMap::new(),
};
debug!("Sending badge update for user: {:}", &payload.user_id);
resp = self.client.send(apn_payload).await;
}
}
@@ -322,6 +339,10 @@ impl AsyncConsumer for ApnsOutboundConsumer {
}),
..
}) => {
info!(
"Removing APNS subscription id {:} (user: {:}) due to invalid token",
&payload.session_id, &payload.user_id
);
if let Err(err) = self
.db
.remove_push_subscription_by_session_id(&payload.session_id)

26
crates/daemons/pushd/src/main.rs
View File

@@ -1,3 +1,6 @@
#[macro_use]
extern crate log;
use amqprs::{
channel::{
BasicConsumeArguments, Channel, ExchangeDeclareArguments, QueueBindArguments,
@@ -14,7 +17,7 @@ mod consumers;
use consumers::{
inbound::{
ack::AckConsumer, fr_accepted::FRAcceptedConsumer, fr_received::FRReceivedConsumer,
generic::GenericConsumer, message::MessageConsumer,
generic::GenericConsumer, mass_mention::MassMessageConsumer, message::MessageConsumer,
},
outbound::{apn::ApnsOutboundConsumer, fcm::FcmOutboundConsumer, vapid::VapidOutboundConsumer},
};
@@ -22,6 +25,7 @@ use consumers::{
#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
async fn main() {
let config = config().await;
pretty_env_logger::init();
// Setup database
let db = revolt_database::DatabaseInfo::Auto.connect().await.unwrap();
@@ -96,6 +100,17 @@ async fn main() {
.await,
);
connections.push(
make_queue_and_consume(
&config,
&config.pushd.mass_mention_queue,
config.pushd.get_mass_mention_routing_key().as_str(),
None,
MassMessageConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
if !config.pushd.apn.pkcs8.is_empty() {
connections.push(
make_queue_and_consume(
@@ -223,11 +238,10 @@ where
.manual_ack(false)
.finish();
channel.basic_consume(consumer, args).await.unwrap();
log::info!(
"Consuming routing key {} as queue {}",
routing_key,
queue_name
let routing_key = channel.basic_consume(consumer, args).await.unwrap();
info!(
"Consuming routing key {} as queue {}, tag {}",
routing_key, queue_name, routing_key
);
(channel, connection)
}