HMC/handmade-revolt-backend
6
0
Fork 0
forked from jmug/stoatchat
Code Pull Requests Actions Packages Releases 2 Activity

merge: branch 'main' into insert/feat/crond

Browse Source
This commit is contained in:
Paul Makles
2025-02-10 19:02:19 +00:00
parent 249749e14d add3f40b23
commit 078380d305
96 changed files with 3779 additions and 1215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crates/core/config/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-config"
version = "0.7.19"
version = "0.8.2"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -15,7 +15,6 @@ default = ["test"]
[dependencies]
# Utility
dotenv = "0.15.0"
config = "0.13.3"
cached = "0.44.0"
once_cell = "1.18.0"
@@ -35,4 +34,4 @@ pretty_env_logger = "0.4.0"
sentry = "0.31.5"
# Core
revolt-result = { version = "0.7.19", path = "../result", optional = true }
revolt-result = { version = "0.8.2", path = "../result", optional = true }

6
crates/core/config/Revolt.test.toml
View File

@@ -1,3 +1,9 @@
[database]
mongodb = "mongodb://localhost"
redis = "redis://localhost/"
[rabbit]
host = "127.0.0.1"
port = 5672
username = "rabbituser"
password = "rabbitpass"

82
crates/core/config/Revolt.toml
View File

@@ -20,6 +20,12 @@ january = "http://local.revolt.chat/january"
voso_legacy = ""
voso_legacy_ws = ""
[rabbit]
host = "rabbit"
port = 5672
username = "rabbituser"
password = "rabbitpass"
[api]
[api.registration]
@@ -38,34 +44,6 @@ from_address = "noreply@example.com"
# port = 587
# use_tls = true
[api.vapid]
# Generate your own keys:
# 1. Run `openssl ecparam -name prime256v1 -genkey -noout -out vapid_private.pem`
# 2. Find `private_key` using `base64 vapid_private.pem`
# 3. Find `public_key` using `openssl ec -in vapid_private.pem -outform DER|tail -c 65|base64|tr '/+' '_-'|tr -d '\n'`
private_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJSUWpyTWxLRnBiVWhsUHpUbERvcEliYk1yeVNrNXpKYzVYVzIxSjJDS3hvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFWnkrQkg2TGJQZ2hEa3pEempXOG0rUXVPM3pCajRXT1phdkR6ZU00c0pqbmFwd1psTFE0WAp1ZDh2TzVodU94QWhMQlU3WWRldVovWHlBdFpWZmNyQi9BPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo"
public_key = "BGcvgR-i2z4IQ5Mw841vJvkLjt8wY-FjmWrw83jOLCY52qcGZS0OF7nfLzuYbjsQISwVO2HXrmf18gLWVX3Kwfw="
[api.fcm]
# Google Firebase Cloud Messaging Service Account Key
# Obtained from the cloud messaging console
key_type = ""
project_id = ""
private_key_id = ""
private_key = ""
client_email = ""
client_id = ""
auth_uri = ""
token_uri = ""
auth_provider_x509_cert_url = ""
client_x509_cert_url = ""
[api.apn]
# Apple Push Notifications keys for sending notifications
sandbox = false
pkcs8 = ""
key_id = ""
team_id = ""
[api.security]
# Authifier Shield API key
@@ -84,6 +62,46 @@ hcaptcha_sitekey = ""
# Maximum concurrent connections (to proxy server)
max_concurrent_connections = 50
[pushd]
# this changes the names of the queues to not overlap
# prod/beta if they happen to be on the same exchange/instance.
# Usually they have to be, so that messages sent from one or the other get sent to everyone
production = true
# none of these should need changing
exchange = "revolt.notifications"
message_queue = "notifications.origin.message"
fr_accepted_queue = "notifications.ingest.fr_accepted" # friend request accepted
fr_received_queue = "notifications.ingest.fr_received" # friend request received
generic_queue = "notifications.ingest.generic" # generic messages (title + body)
ack_queue = "notifications.process.ack" # updates badges for apple devices
[pushd.vapid]
queue = "notifications.outbound.vapid"
private_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJSUWpyTWxLRnBiVWhsUHpUbERvcEliYk1yeVNrNXpKYzVYVzIxSjJDS3hvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFWnkrQkg2TGJQZ2hEa3pEempXOG0rUXVPM3pCajRXT1phdkR6ZU00c0pqbmFwd1psTFE0WAp1ZDh2TzVodU94QWhMQlU3WWRldVovWHlBdFpWZmNyQi9BPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo"
public_key = "BGcvgR-i2z4IQ5Mw841vJvkLjt8wY-FjmWrw83jOLCY52qcGZS0OF7nfLzuYbjsQISwVO2HXrmf18gLWVX3Kwfw="
[pushd.fcm]
queue = "notifications.outbound.fcm"
key_type = ""
project_id = ""
private_key_id = ""
private_key = ""
client_email = ""
client_id = ""
auth_uri = ""
token_uri = ""
auth_provider_x509_cert_url = ""
client_x509_cert_url = ""
[pushd.apn]
sandbox = false
queue = "notifications.outbound.apn"
pkcs8 = ""
key_id = ""
team_id = ""
[files]
# Encryption key for stored files
# Generate your own key using `openssl rand -base64 32`
@@ -149,10 +167,11 @@ region = "minio"
access_key_id = "minioautumn"
# S3 protocol access key
secret_access_key = "minioautumn"
# Bucket to upload to by default
default_bucket = "revolt-uploads"
[features]
# Bucket to upload to by default
# Feature gate options
webhooks_enabled = false
@@ -228,6 +247,11 @@ icons = 2_500_000
banners = 6_000_000
emojis = 500_000
[features.advanced]
# The max amount of messages the rabbitmq provider/db mention adder job will delay for before forcing handling of a channel.
# default: 5
process_message_delay_limit = 5
[sentry]
# Configuration for Sentry error reporting
api = ""

95
crates/core/config/src/lib.rs
View File

@@ -23,6 +23,17 @@ macro_rules! report_error {
};
}
#[cfg(feature = "report-macros")]
#[macro_export]
macro_rules! capture_internal_error {
( $expr: expr ) => {
$crate::capture_message(
&format!("{:?} ({}:{}:{})", $expr, file!(), line!(), column!()),
$crate::Level::Error,
);
};
}
#[cfg(feature = "report-macros")]
#[macro_export]
macro_rules! report_internal_error {
@@ -79,6 +90,14 @@ pub struct Database {
pub redis: String,
}
#[derive(Deserialize, Debug, Clone)]
pub struct Rabbit {
pub host: String,
pub port: u16,
pub username: String,
pub password: String,
}
#[derive(Deserialize, Debug, Clone)]
pub struct Hosts {
pub app: String,
@@ -107,13 +126,15 @@ pub struct ApiSmtp {
}
#[derive(Deserialize, Debug, Clone)]
pub struct ApiVapid {
pub struct PushVapid {
pub queue: String,
pub private_key: String,
pub public_key: String,
}
#[derive(Deserialize, Debug, Clone)]
pub struct ApiFcm {
pub struct PushFcm {
pub queue: String,
pub key_type: String,
pub project_id: String,
pub private_key_id: String,
@@ -127,7 +148,8 @@ pub struct ApiFcm {
}
#[derive(Deserialize, Debug, Clone)]
pub struct ApiApn {
pub struct PushApn {
pub queue: String,
pub sandbox: bool,
pub pkcs8: String,
pub key_id: String,
@@ -157,13 +179,54 @@ pub struct ApiWorkers {
pub struct Api {
pub registration: ApiRegistration,
pub smtp: ApiSmtp,
pub vapid: ApiVapid,
pub fcm: ApiFcm,
pub apn: ApiApn,
pub security: ApiSecurity,
pub workers: ApiWorkers,
}
#[derive(Deserialize, Debug, Clone)]
pub struct Pushd {
pub production: bool,
pub exchange: String,
pub message_queue: String,
pub fr_accepted_queue: String,
pub fr_received_queue: String,
pub generic_queue: String,
pub ack_queue: String,
pub vapid: PushVapid,
pub fcm: PushFcm,
pub apn: PushApn,
}
impl Pushd {
fn get_routing_key(&self, key: String) -> String {
match self.production {
true => key + "-prd",
false => key + "-tst",
}
}
pub fn get_ack_routing_key(&self) -> String {
self.get_routing_key(self.ack_queue.clone())
}
pub fn get_message_routing_key(&self) -> String {
self.get_routing_key(self.message_queue.clone())
}
pub fn get_fr_accepted_routing_key(&self) -> String {
self.get_routing_key(self.fr_accepted_queue.clone())
}
pub fn get_fr_received_routing_key(&self) -> String {
self.get_routing_key(self.fr_received_queue.clone())
}
pub fn get_generic_routing_key(&self) -> String {
self.get_routing_key(self.generic_queue.clone())
}
}
#[derive(Deserialize, Debug, Clone)]
pub struct FilesLimit {
pub min_file_size: usize,
@@ -233,10 +296,26 @@ pub struct FeaturesLimitsCollection {
pub roles: HashMap<String, FeaturesLimits>,
}
#[derive(Deserialize, Debug, Clone)]
pub struct FeaturesAdvanced {
#[serde(default)]
pub process_message_delay_limit: u16,
}
impl Default for FeaturesAdvanced {
fn default() -> Self {
Self {
process_message_delay_limit: 5,
}
}
}
#[derive(Deserialize, Debug, Clone)]
pub struct Features {
pub limits: FeaturesLimitsCollection,
pub webhooks_enabled: bool,
#[serde(default)]
pub advanced: FeaturesAdvanced,
}
#[derive(Deserialize, Debug, Clone)]
@@ -251,8 +330,10 @@ pub struct Sentry {
#[derive(Deserialize, Debug, Clone)]
pub struct Settings {
pub database: Database,
pub rabbit: Rabbit,
pub hosts: Hosts,
pub api: Api,
pub pushd: Pushd,
pub files: Files,
pub features: Features,
pub sentry: Sentry,
@@ -288,8 +369,6 @@ pub async fn config() -> Settings {
/// Configure logging and common Rust variables
pub async fn setup_logging(release: &'static str, dsn: String) -> Option<sentry::ClientInitGuard> {
dotenv::dotenv().ok();
if std::env::var("RUST_LOG").is_err() {
std::env::set_var("RUST_LOG", "info");
}

21
crates/core/database/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-database"
version = "0.7.19"
version = "0.8.2"
edition = "2021"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -24,15 +24,15 @@ default = ["mongodb", "async-std-runtime", "tasks"]
[dependencies]
# Core
revolt-config = { version = "0.7.19", path = "../config", features = [
revolt-config = { version = "0.8.2", path = "../config", features = [
"report-macros",
] }
revolt-result = { version = "0.7.19", path = "../result" }
revolt-models = { version = "0.7.19", path = "../models", features = [
revolt-result = { version = "0.8.2", path = "../result" }
revolt-models = { version = "0.8.2", path = "../models", features = [
"validator",
] }
revolt-presence = { version = "0.7.19", path = "../presence" }
revolt-permissions = { version = "0.7.19", path = "../permissions", features = [
revolt-presence = { version = "0.8.2", path = "../presence" }
revolt-permissions = { version = "0.8.2", path = "../permissions", features = [
"serde",
"bson",
] }
@@ -84,11 +84,11 @@ axum = { version = "0.7.5", optional = true }
# Rocket Impl
schemars = { version = "0.8.8", optional = true }
rocket = { version = "0.5.0-rc.2", default-features = false, features = [
rocket = { version = "0.5.1", default-features = false, features = [
"json",
], optional = true }
revolt_okapi = { version = "0.9.1", optional = true }
revolt_rocket_okapi = { version = "0.9.1", optional = true }
revolt_rocket_okapi = { version = "0.10.0", optional = true }
# Notifications
fcm_v1 = "0.3.0"
@@ -96,4 +96,7 @@ web-push = "0.10.0"
revolt_a2 = { version = "0.10", default-features = false, features = ["ring"] }
# Authifier
authifier = { version = "1.0.8" }
authifier = { version = "1.0.9", features = ["rocket_impl"] }
# RabbitMQ
amqprs = { version = "1.7.0" }

211
crates/core/database/src/amqp/amqp.rs Normal file
View File

@@ -0,0 +1,211 @@
use std::collections::HashSet;
use crate::events::rabbit::*;
use crate::User;
use amqprs::channel::BasicPublishArguments;
use amqprs::{channel::Channel, connection::Connection, error::Error as AMQPError};
use amqprs::{BasicProperties, FieldTable};
use revolt_models::v0::PushNotification;
use revolt_presence::filter_online;
use serde_json::to_string;
#[derive(Clone)]
pub struct AMQP {
#[allow(unused)]
connection: Connection,
channel: Channel,
}
impl AMQP {
pub fn new(connection: Connection, channel: Channel) -> AMQP {
AMQP {
connection,
channel,
}
}
pub async fn friend_request_accepted(
&self,
accepted_request_user: &User,
sent_request_user: &User,
) -> Result<(), AMQPError> {
let config = revolt_config::config().await;
let payload = FRAcceptedPayload {
accepted_user: accepted_request_user.to_owned(),
user: sent_request_user.id.clone(),
};
let payload = to_string(&payload).unwrap();
debug!(
"Sending friend request accept payload on channel {}: {}",
config.pushd.get_fr_accepted_routing_key(),
payload
);
self.channel
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_fr_accepted_routing_key(),
),
)
.await
}
pub async fn friend_request_received(
&self,
received_request_user: &User,
sent_request_user: &User,
) -> Result<(), AMQPError> {
let config = revolt_config::config().await;
let payload = FRReceivedPayload {
from_user: sent_request_user.to_owned(),
user: received_request_user.id.clone(),
};
let payload = to_string(&payload).unwrap();
debug!(
"Sending friend request received payload on channel {}: {}",
config.pushd.get_fr_received_routing_key(),
payload
);
self.channel
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_fr_received_routing_key(),
),
)
.await
}
pub async fn generic_message(
&self,
user: &User,
title: String,
body: String,
icon: Option<String>,
) -> Result<(), AMQPError> {
let config = revolt_config::config().await;
let payload = GenericPayload {
title,
body,
icon,
user: user.to_owned(),
};
let payload = to_string(&payload).unwrap();
debug!(
"Sending generic payload on channel {}: {}",
config.pushd.get_generic_routing_key(),
payload
);
self.channel
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_generic_routing_key(),
),
)
.await
}
pub async fn message_sent(
&self,
recipients: Vec<String>,
payload: PushNotification,
) -> Result<(), AMQPError> {
if recipients.is_empty() {
return Ok(());
}
let config = revolt_config::config().await;
let online_ids = filter_online(&recipients).await;
let recipients = (&recipients.into_iter().collect::<HashSet<String>>() - &online_ids)
.into_iter()
.collect::<Vec<String>>();
let payload = MessageSentPayload {
notification: payload,
users: recipients,
};
let payload = to_string(&payload).unwrap();
debug!(
"Sending message payload on channel {}: {}",
config.pushd.get_message_routing_key(),
payload
);
self.channel
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_message_routing_key(),
),
)
.await
}
pub async fn ack_message(
&self,
user_id: String,
channel_id: String,
message_id: String,
) -> Result<(), AMQPError> {
let config = revolt_config::config().await;
let payload = AckPayload {
user_id: user_id.clone(),
channel_id: channel_id.clone(),
message_id,
};
let payload = to_string(&payload).unwrap();
info!(
"Sending ack payload on channel {}: {}",
config.pushd.ack_queue, payload
);
let mut headers = FieldTable::new();
headers.insert(
"x-deduplication-header".try_into().unwrap(),
format!("{}-{}", &user_id, &channel_id).into(),
);
self.channel
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
//.with_headers(headers)
.finish(),
payload.into(),
BasicPublishArguments::new(&config.pushd.exchange, &config.pushd.ack_queue),
)
.await
}
}

2
crates/core/database/src/amqp/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
#[allow(clippy::module_inception)]
pub mod amqp;

1
crates/core/database/src/events/mod.rs
View File

@@ -1,2 +1,3 @@
pub mod client;
pub mod rabbit;
pub mod server;

59
crates/core/database/src/events/rabbit.rs Normal file
View File

@@ -0,0 +1,59 @@
use std::collections::HashMap;
use revolt_models::v0::PushNotification;
use serde::{Deserialize, Serialize};
use crate::User;
#[derive(Serialize, Deserialize)]
pub struct MessageSentPayload {
pub notification: PushNotification,
pub users: Vec<String>,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct FRAcceptedPayload {
pub accepted_user: User,
pub user: String,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct FRReceivedPayload {
pub from_user: User,
pub user: String,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct GenericPayload {
pub title: String,
pub body: String,
pub icon: Option<String>,
pub user: User,
}
#[derive(Serialize, Deserialize)]
#[serde(tag = "type", content = "data")]
#[allow(clippy::large_enum_variant)]
pub enum PayloadKind {
MessageNotification(PushNotification),
FRAccepted(FRAcceptedPayload),
FRReceived(FRReceivedPayload),
BadgeUpdate(usize),
Generic(GenericPayload),
}
#[derive(Serialize, Deserialize)]
pub struct PayloadToService {
pub notification: PayloadKind,
pub user_id: String,
pub session_id: String,
pub token: String,
pub extras: HashMap<String, String>,
}
#[derive(Serialize, Deserialize)]
pub struct AckPayload {
pub user_id: String,
pub channel_id: String,
pub message_id: String,
}

23
crates/core/database/src/lib.rs
View File

@@ -25,6 +25,26 @@ pub use mongodb;
#[macro_use]
extern crate bson;
#[macro_export]
#[cfg(debug_assertions)]
macro_rules! query {
( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
Ok($self.$type($collection, $($rest),+).await.unwrap())
};
}
#[macro_export]
#[cfg(not(debug_assertions))]
macro_rules! query {
( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
$self.$type($collection, $($rest),+).await
.map_err(|err| {
revolt_config::capture_internal_error!(err);
create_database_error!(stringify!($type), $collection)
})
};
}
macro_rules! database_derived {
( $( $item:item )+ ) => {
$(
@@ -85,6 +105,9 @@ pub use models::*;
pub mod events;
pub mod tasks;
mod amqp;
pub use amqp::amqp::AMQP;
/// Utility function to check if a boolean value is false
pub fn if_false(t: &bool) -> bool {
!t

3
crates/core/database/src/models/channel_unreads/ops.rs
View File

@@ -26,6 +26,9 @@ pub trait AbstractChannelUnreads: Sync + Send {
message_ids: &[String],
) -> Result<()>;
/// Fetch all unreads with mentions for a user.
async fn fetch_unread_mentions(&self, user_id: &str) -> Result<Vec<ChannelUnread>>;
/// Fetch all channel unreads for a user.
async fn fetch_unreads(&self, user_id: &str) -> Result<Vec<ChannelUnread>>;

15
crates/core/database/src/models/channel_unreads/ops/mongodb.rs
View File

@@ -30,7 +30,7 @@ impl AbstractChannelUnreads for MongoDb {
doc! {
"$pull": {
"mentions": {
"$lt": message_id
"$lte": message_id
}
},
"$set": {
@@ -123,6 +123,18 @@ impl AbstractChannelUnreads for MongoDb {
)
}
async fn fetch_unread_mentions(&self, user_id: &str) -> Result<Vec<ChannelUnread>> {
query! {
self,
find,
COL,
doc! {
"_id.user": user_id,
"mentions": {"$ne": null}
}
}
}
/// Fetch unread for a specific user in a channel.
async fn fetch_unread(&self, user_id: &str, channel_id: &str) -> Result<Option<ChannelUnread>> {
query!(
@@ -135,5 +147,4 @@ impl AbstractChannelUnreads for MongoDb {
}
)
}
}

19
crates/core/database/src/models/channel_unreads/ops/reference.rs
View File

@@ -78,6 +78,15 @@ impl AbstractChannelUnreads for ReferenceDb {
Ok(())
}
async fn fetch_unread_mentions(&self, user_id: &str) -> Result<Vec<ChannelUnread>> {
let unreads = self.channel_unreads.lock().await;
Ok(unreads
.values()
.filter(|unread| unread.id.user == user_id && unread.mentions.is_some())
.cloned()
.collect())
}
/// Fetch all channel unreads for a user.
async fn fetch_unreads(&self, user_id: &str) -> Result<Vec<ChannelUnread>> {
let unreads = self.channel_unreads.lock().await;
@@ -92,9 +101,11 @@ impl AbstractChannelUnreads for ReferenceDb {
async fn fetch_unread(&self, user_id: &str, channel_id: &str) -> Result<Option<ChannelUnread>> {
let unreads = self.channel_unreads.lock().await;
Ok(unreads.get(&ChannelCompositeKey {
channel: channel_id.to_string(),
user: user_id.to_string()
}).cloned())
Ok(unreads
.get(&ChannelCompositeKey {
channel: channel_id.to_string(),
user: user_id.to_string(),
})
.cloned())
}
}

9
crates/core/database/src/models/channels/model.rs
View File

@@ -9,7 +9,7 @@ use ulid::Ulid;
use crate::{
events::client::EventV1, tasks::ack::AckEvent, Database, File, IntoDocumentPath, PartialServer,
Server, SystemMessage, User,
Server, SystemMessage, User, AMQP,
};
auto_derived!(
@@ -337,6 +337,7 @@ impl Channel {
pub async fn add_user_to_group(
&mut self,
db: &Database,
amqp: &AMQP,
user: &User,
by_id: &str,
) -> Result<()> {
@@ -373,6 +374,7 @@ impl Channel {
.into_message(id.to_string())
.send(
db,
Some(amqp),
MessageAuthor::System {
username: &user.username,
avatar: user.avatar.as_ref().map(|file| file.id.as_ref()),
@@ -639,7 +641,7 @@ impl Channel {
.private(user.to_string())
.await;
crate::tasks::ack::queue(
crate::tasks::ack::queue_ack(
self.id().to_string(),
user.to_string(),
AckEvent::AckMessage {
@@ -655,6 +657,7 @@ impl Channel {
pub async fn remove_user_from_group(
&self,
db: &Database,
amqp: &AMQP,
user: &User,
by_id: Option<&str>,
silent: bool,
@@ -686,6 +689,7 @@ impl Channel {
.into_message(id.to_string())
.send(
db,
Some(amqp),
MessageAuthor::System {
username: name,
avatar: None,
@@ -725,6 +729,7 @@ impl Channel {
.into_message(id.to_string())
.send(
db,
Some(amqp),
MessageAuthor::System {
username: &user.username,
avatar: user.avatar.as_ref().map(|file| file.id.as_ref()),

2
crates/core/database/src/models/file_hashes/model.rs
View File

@@ -45,7 +45,7 @@ auto_derived!(
Image {
width: isize,
height: isize,
// animated: bool // TODO: https://docs.rs/image/latest/image/trait.AnimationDecoder.html
// animated: bool // TODO: https://docs.rs/image/latest/image/trait.AnimationDecoder.html for APNG support
},
/// File is a video with specific dimensions
Video { width: isize, height: isize },

102
crates/core/database/src/models/messages/model.rs
View File

@@ -15,8 +15,8 @@ use validator::Validate;
use crate::{
events::client::EventV1,
tasks::{self, ack::AckEvent},
util::idempotency::IdempotencyKey,
Channel, Database, Emoji, File, User,
util::{bulk_permissions::BulkDatabasePermissionQuery, idempotency::IdempotencyKey},
Channel, Database, Emoji, File, User, AMQP,
};
auto_derived_partial!(
@@ -230,6 +230,7 @@ impl Message {
#[allow(clippy::too_many_arguments)]
pub async fn create_from_api(
db: &Database,
amqp: Option<&AMQP>,
channel: Channel,
data: DataMessageSend,
author: MessageAuthor<'_>,
@@ -337,35 +338,52 @@ impl Message {
}
}
// Validate the mentions go to users in the channel/server
if !mentions.is_empty() {
// FIXME: temp fix to stop spam attacks
match channel {
Channel::DirectMessage { ref recipients, .. }
| Channel::Group { ref recipients, .. } => {
let recipients_hash: HashSet<&String, RandomState> =
HashSet::from_iter(recipients.iter());
HashSet::from_iter(recipients);
mentions.retain(|m| recipients_hash.contains(m));
}
Channel::TextChannel { ref server, .. }
| Channel::VoiceChannel { ref server, .. } => {
let mentions_vec = Vec::from_iter(mentions.iter().cloned());
let valid_members = db.fetch_members(server.as_str(), &mentions_vec[..]).await;
if let Ok(valid_members) = valid_members {
let valid_ids: HashSet<String, RandomState> = HashSet::from_iter(
valid_members.iter().map(|member| member.id.user.clone()),
);
mentions.retain(|m| valid_ids.contains(m));
let valid_mentions: HashSet<&String, RandomState> =
HashSet::from_iter(valid_members.iter().map(|m| &m.id.user));
mentions.retain(|m| valid_mentions.contains(m)); // quick pass, validate mentions are in the server
if !mentions.is_empty() {
// if there are still mentions, drill down to a channel-level
let member_channel_view_perms =
BulkDatabasePermissionQuery::from_server_id(db, server)
.await
.channel(&channel)
.members(&valid_members)
.members_can_see_channel()
.await;
mentions
.retain(|m| *member_channel_view_perms.get(m).unwrap_or(&false));
}
} else {
revolt_config::capture_error(&valid_members.unwrap_err());
return Err(create_error!(InternalError));
}
}
Channel::SavedMessages { .. } => mentions.clear(),
Channel::SavedMessages { .. } => {
mentions.clear();
}
}
}
if !mentions.is_empty() {
message.mentions.replace(mentions.into_iter().collect());
}
if !mentions.is_empty() {
message.mentions.replace(mentions.into_iter().collect());
}
if !replies.is_empty() {
@@ -418,7 +436,7 @@ impl Message {
// Send the message
message
.send(db, author, user, member, &channel, generate_embeds)
.send(db, amqp, author, user, member, &channel, generate_embeds)
.await?;
Ok(message)
@@ -432,6 +450,9 @@ impl Message {
member: Option<v0::Member>,
is_dm: bool,
generate_embeds: bool,
// This determines if this function should queue the mentions task or if somewhere else will.
// If this is true, you MUST call tasks::ack::queue yourself.
mentions_elsewhere: bool,
) -> Result<()> {
db.insert_message(self).await?;
@@ -444,13 +465,12 @@ impl Message {
tasks::last_message_id::queue(self.channel.to_string(), self.id.to_string(), is_dm).await;
// Add mentions for affected users
if let Some(mentions) = &self.mentions {
for user in mentions {
tasks::ack::queue(
if !mentions_elsewhere {
if let Some(mentions) = &self.mentions {
tasks::ack::queue_message(
self.channel.to_string(),
user.to_string(),
AckEvent::AddMention {
ids: vec![self.id.to_string()],
AckEvent::ProcessMessage {
messages: vec![(None, self.clone(), mentions.clone(), true)],
},
)
.await;
@@ -473,9 +493,11 @@ impl Message {
}
/// Send a message
#[allow(clippy::too_many_arguments)]
pub async fn send(
&mut self,
db: &Database,
amqp: Option<&AMQP>, // this is optional mostly for tests.
author: MessageAuthor<'_>,
user: Option<v0::User>,
member: Option<v0::Member>,
@@ -488,26 +510,36 @@ impl Message {
member.clone(),
matches!(channel, Channel::DirectMessage { .. }),
generate_embeds,
true,
)
.await?;
if !self.has_suppressed_notifications() {
// Push out Web Push notifications
crate::tasks::web_push::queue(
{
match channel {
Channel::DirectMessage { recipients, .. }
| Channel::Group { recipients, .. } => recipients.clone(),
Channel::TextChannel { .. } => self.mentions.clone().unwrap_or_default(),
_ => vec![],
}
// send Push notifications
tasks::ack::queue_message(
self.channel.to_string(),
AckEvent::ProcessMessage {
messages: vec![(
Some(
PushNotification::from(
self.clone().into_model(user, member),
Some(author),
channel.to_owned().into(),
)
.await,
),
self.clone(),
match channel {
Channel::DirectMessage { recipients, .. }
| Channel::Group { recipients, .. } => recipients.clone(),
Channel::TextChannel { .. } => {
self.mentions.clone().unwrap_or_default()
}
_ => vec![],
},
self.has_suppressed_notifications(),
)],
},
PushNotification::from(
self.clone().into_model(user, member),
Some(author),
channel.id(),
)
.await,
)
.await;
}

4
crates/core/database/src/models/messages/ops/mongodb.rs
View File

@@ -95,7 +95,7 @@ impl AbstractMessages for MongoDb {
COL,
older_message_filter,
FindOptions::builder()
.limit(limit / 2)
.limit(limit / 2 + 1)
.sort(doc! {
"_id": -1_i32
})
@@ -171,7 +171,7 @@ impl AbstractMessages for MongoDb {
self.find_with_options(
COL,
doc! {
"ids": {
"_id": {
"$in": ids
}
},

4
crates/core/database/src/models/server_members/model.rs
View File

@@ -150,7 +150,7 @@ impl Member {
id: user.id.clone(),
}
.into_message(id.to_string())
.send_without_notifications(db, None, None, false, false)
.send_without_notifications(db, None, None, false, false, false)
.await
.ok();
}
@@ -251,7 +251,7 @@ impl Member {
}
.into_message(id.to_string())
// TODO: support notifications here in the future?
.send_without_notifications(db, None, None, false, false)
.send_without_notifications(db, None, None, false, false, false)
.await
.ok();
}

8
crates/core/database/src/models/server_members/ops/reference.rs
View File

@@ -53,17 +53,17 @@ impl AbstractServerMembers for ReferenceDb {
/// Fetch multiple members by their ids
async fn fetch_members<'a>(&self, server_id: &str, ids: &'a [String]) -> Result<Vec<Member>> {
let server_members = self.server_members.lock().await;
ids.iter()
.map(|id| {
Ok(ids
.iter()
.filter_map(|id| {
server_members
.get(&MemberCompositeKey {
server: server_id.to_string(),
user: id.to_string(),
})
.cloned()
.ok_or_else(|| create_error!(NotFound))
})
.collect()
.collect())
}
/// Fetch member count of a server

13
crates/core/database/src/models/users/model.rs
View File

@@ -1,6 +1,6 @@
use std::{collections::HashSet, str::FromStr, time::Duration};
use crate::{events::client::EventV1, Database, File, RatelimitEvent};
use crate::{events::client::EventV1, Database, File, RatelimitEvent, AMQP};
use authifier::config::{EmailVerificationConfig, Template};
use iso8601_timestamp::Timestamp;
@@ -504,7 +504,12 @@ impl User {
}
/// Add another user as a friend
pub async fn add_friend(&mut self, db: &Database, target: &mut User) -> Result<()> {
pub async fn add_friend(
&mut self,
db: &Database,
amqp: &AMQP,
target: &mut User,
) -> Result<()> {
match self.relationship_with(&target.id) {
RelationshipStatus::User => Err(create_error!(NoEffect)),
RelationshipStatus::Friend => Err(create_error!(AlreadyFriends)),
@@ -513,6 +518,8 @@ impl User {
RelationshipStatus::BlockedOther => Err(create_error!(BlockedByOther)),
RelationshipStatus::Incoming => {
// Accept incoming friend request
_ = amqp.friend_request_accepted(self, target).await;
self.apply_relationship(
db,
target,
@@ -541,6 +548,8 @@ impl User {
}));
}
_ = amqp.friend_request_received(target, self).await;
// Send the friend request
self.apply_relationship(
db,

2
crates/core/database/src/models/users/rocket.rs
View File

@@ -38,7 +38,7 @@ impl<'r> FromRequest<'r> for User {
if let Some(user) = user {
Outcome::Success(user.clone())
} else {
Outcome::Failure((Status::Unauthorized, authifier::Error::InvalidSession))
Outcome::Error((Status::Unauthorized, authifier::Error::InvalidSession))
}
}
}

178
crates/core/database/src/tasks/ack.rs
View File

@@ -1,21 +1,27 @@
// Queue Type: Debounced
use crate::Database;
use crate::{Database, Message, AMQP};
use deadqueue::limited::Queue;
use once_cell::sync::Lazy;
use std::{collections::HashMap, time::Duration};
use revolt_models::v0::PushNotification;
use rocket::form::validate::Contains;
use std::{
collections::{HashMap, HashSet},
time::Duration,
};
use validator::HasLen;
use revolt_result::Result;
use super::{apple_notifications::{self, ApnJob}, DelayedTask};
use super::DelayedTask;
/// Enumeration of possible events
#[derive(Debug, Eq, PartialEq)]
pub enum AckEvent {
/// Add mentions for a user in a channel
AddMention {
/// Message IDs
ids: Vec<String>,
/// Add mentions for a channel
ProcessMessage {
/// push notification, message, recipients, push silenced
messages: Vec<(Option<PushNotification>, Message, Vec<String>, bool)>,
},
/// Acknowledge message in a channel for a user
@@ -30,7 +36,7 @@ struct Data {
/// Channel to ack
channel: String,
/// User to ack for
user: String,
user: Option<String>,
/// Event
event: AckEvent,
}
@@ -43,21 +49,49 @@ struct Task {
static Q: Lazy<Queue<Data>> = Lazy::new(|| Queue::new(10_000));
/// Queue a new task for a worker
pub async fn queue(channel: String, user: String, event: AckEvent) {
pub async fn queue_ack(channel: String, user: String, event: AckEvent) {
Q.try_push(Data {
channel,
user,
user: Some(user),
event,
})
.ok();
info!("Queue is using {} slots from {}.", Q.len(), Q.capacity());
info!(
"Queue is using {} slots from {}. Queued type: ACK",
Q.len(),
Q.capacity()
);
}
pub async fn handle_ack_event(event: &AckEvent, db: &Database, authifier_db: &authifier::Database, user: &str, channel: &str) -> Result<()> {
pub async fn queue_message(channel: String, event: AckEvent) {
Q.try_push(Data {
channel,
user: None,
event,
})
.ok();
info!(
"Queue is using {} slots from {}. Queued type: MENTION",
Q.len(),
Q.capacity()
);
}
pub async fn handle_ack_event(
event: &AckEvent,
db: &Database,
amqp: &AMQP,
user: &Option<String>,
channel: &str,
) -> Result<()> {
match &event {
#[allow(clippy::disallowed_methods)] // event is sent by higher level function
AckEvent::AckMessage { id } => {
let user = user.as_ref().unwrap();
let user: &str = user.as_str();
let unread = db.fetch_unread(user, channel).await?;
let updated = db.acknowledge_message(channel, user, id).await?;
@@ -68,21 +102,67 @@ pub async fn handle_ack_event(event: &AckEvent, db: &Database, authifier_db: &au
let mentions_acked = before_mentions - after_mentions;
if mentions_acked > 0 {
if let Ok(sessions) = authifier_db.find_sessions(user).await {
for session in sessions {
if let Some(sub) = session.subscription {
if sub.endpoint == "apn" {
apple_notifications::queue(ApnJob::from_ack(session.id, user.to_string(), sub.auth)).await;
}
}
}
if let Err(err) = amqp
.ack_message(user.to_string(), channel.to_string(), id.to_owned())
.await
{
revolt_config::capture_error(&err);
}
};
}
},
AckEvent::AddMention { ids } => {
db.add_mention_to_unread(channel, user, ids).await?;
}
AckEvent::ProcessMessage { messages } => {
let mut users: HashSet<&String> = HashSet::new();
debug!(
"Processing {} messages from channel {}",
messages.len(),
messages[0].1.channel
);
// find all the users we'll be notifying
messages
.iter()
.for_each(|(_, _, recipents, _)| users.extend(recipents.iter()));
debug!("Found {} users to notify.", users.len());
for user in users {
let message_ids: Vec<String> = messages
.iter()
.filter(|(_, _, recipients, _)| recipients.contains(user))
.map(|(_, message, _, _)| message.id.clone())
.collect();
if !message_ids.is_empty() {
db.add_mention_to_unread(channel, user, &message_ids)
.await?;
}
debug!("Added {} mentions for user {}", message_ids.len(), &user);
}
for (push, _, recipients, silenced) in messages {
if *silenced || recipients.is_empty() || push.is_none() {
debug!(
"Rejecting push: silenced: {}, recipient count: {}, push exists: {:?}",
*silenced,
recipients.length(),
push
);
continue;
}
debug!(
"Sending push event to AMQP; message {} for {} users",
push.as_ref().unwrap().message.id,
recipients.len()
);
if let Err(err) = amqp
.message_sent(recipients.clone(), push.clone().unwrap())
.await
{
revolt_config::capture_error(&err);
}
}
}
};
@@ -90,9 +170,9 @@ pub async fn handle_ack_event(event: &AckEvent, db: &Database, authifier_db: &au
}
/// Start a new worker
pub async fn worker(db: Database, authifier_db: authifier::Database) {
let mut tasks = HashMap::<(String, String), DelayedTask<Task>>::new();
let mut keys = vec![];
pub async fn worker(db: Database, amqp: AMQP) {
let mut tasks = HashMap::<(Option<String>, String, u8), DelayedTask<Task>>::new();
let mut keys: Vec<(Option<String>, String, u8)> = vec![];
loop {
// Find due tasks.
@@ -106,12 +186,13 @@ pub async fn worker(db: Database, authifier_db: authifier::Database) {
for key in &keys {
if let Some(task) = tasks.remove(key) {
let Task { event } = task.data;
let (user, channel) = key;
let (user, channel, _) = key;
if let Err(err) = handle_ack_event(&event, &db, &authifier_db, user, channel).await {
error!("{err:?} for {event:?}. ({user}, {channel})");
if let Err(err) = handle_ack_event(&event, &db, &amqp, user, channel).await {
revolt_config::capture_error(&err);
error!("{err:?} for {event:?}. ({user:?}, {channel})");
} else {
info!("User {user} ack in {channel} with {event:?}");
info!("User {user:?} ack in {channel} with {event:?}");
}
}
}
@@ -126,20 +207,41 @@ pub async fn worker(db: Database, authifier_db: authifier::Database) {
mut event,
}) = Q.try_pop()
{
let key = (user, channel);
let key: (Option<String>, String, u8) = (
user,
channel,
match &event {
AckEvent::AckMessage { .. } => 0,
AckEvent::ProcessMessage { .. } => 1,
},
);
if let Some(task) = tasks.get_mut(&key) {
task.delay();
match &mut event {
AckEvent::AddMention { ids } => {
if let AckEvent::AddMention { ids: existing } = &mut task.data.event {
existing.append(ids);
AckEvent::ProcessMessage { messages: new_data } => {
if let AckEvent::ProcessMessage { messages: existing } =
&mut task.data.event
{
// add the new message to the list of messages to be processed.
existing.append(new_data);
// put a cap on the amount of messages that can be queued, for particularly active channels
if (existing.length() as u16)
< revolt_config::config()
.await
.features
.advanced
.process_message_delay_limit
{
task.delay();
}
} else {
task.data.event = event;
panic!("Somehow got an ack message in the add mention arm");
}
}
AckEvent::AckMessage { .. } => {
// replace the last acked message with the new acked message
task.data.event = event;
task.delay();
}
}
} else {

314
crates/core/database/src/tasks/apple_notifications.rs
View File

@@ -1,314 +0,0 @@
use std::io::Cursor;
use base64::{
engine::{self},
Engine as _,
};
use deadqueue::limited::Queue;
use once_cell::sync::Lazy;
use revolt_a2::{
request::{
notification::{DefaultAlert, NotificationOptions},
payload::{APSAlert, APSSound, PayloadLike, APS},
},
Client, ClientConfig, Endpoint, Error, ErrorBody, ErrorReason, Priority, PushType, Response,
};
use revolt_config::config;
use revolt_models::v0::{Message, PushNotification};
use crate::Database;
/// Payload information, before assembly
#[derive(Debug)]
#[allow(non_snake_case)]
pub struct ApnPayload {
message: Message,
url: String,
authorAvatar: String,
authorDisplayName: String,
channelName: String,
}
#[derive(Serialize, Debug)]
#[allow(non_snake_case)]
struct Payload<'a> {
aps: APS<'a>,
#[serde(skip_serializing)]
options: NotificationOptions<'a>,
#[serde(skip_serializing)]
device_token: &'a str,
message: &'a Message,
url: &'a str,
authorAvatar: &'a str,
authorDisplayName: &'a str,
channelName: &'a str,
}
impl<'a> PayloadLike for Payload<'a> {
fn get_device_token(&self) -> &'a str {
self.device_token
}
fn get_options(&self) -> &NotificationOptions {
&self.options
}
}
/// Task information
#[derive(Debug)]
pub struct AlertJob {
/// Session Id
session_id: String,
/// Device token
device_token: String,
/// User Id
user_id: String,
/// Title
title: String,
/// Body
body: String,
/// Thread Id
thread_id: String,
/// Category (informs the client what kind of notification is being sent.)
category: String,
/// Payload used by the iOS client to modify the notification
custom_payload: ApnPayload,
}
impl AlertJob {
fn format_title(notification: &PushNotification) -> String {
// ideally this changes depending on context
// in a server, it would look like "Sendername, #channelname in servername"
// in a group, it would look like "Sendername in groupname"
// in a dm it should just be "Sendername".
// not sure how feasible all those are given the PushNotification object as it currently stands.
format!(
"{} in {}",
notification.author, notification.message.channel
) // TODO: this absolutely needs a channel name
}
}
#[derive(Debug)]
pub struct BadgeJob {
/// Session Id
session_id: String,
/// Device token
device_token: String,
/// User Id
user_id: String,
}
#[derive(Debug)]
pub enum JobType {
Alert(AlertJob),
Badge(BadgeJob),
}
#[derive(Debug)]
pub struct ApnJob {
job_type: JobType,
}
impl ApnJob {
pub fn from_notification(
session_id: String,
user_id: String,
device_token: String,
notification: &PushNotification,
) -> ApnJob {
ApnJob {
job_type: JobType::Alert(AlertJob {
session_id,
device_token,
user_id,
title: AlertJob::format_title(notification),
body: notification.body.to_string(),
thread_id: notification.tag.to_string(),
category: "ALERT_MESSAGE".to_string(),
custom_payload: ApnPayload {
message: notification.message.clone(),
url: notification.url.clone(),
authorAvatar: notification.icon.clone(),
authorDisplayName: notification.author.clone(),
channelName: "#fetchchannelnamehere".to_string(), // TODO: get actual channel name
},
}),
}
}
pub fn from_ack(session_id: String, user_id: String, device_token: String) -> ApnJob {
ApnJob {
job_type: JobType::Badge(BadgeJob {
session_id,
device_token,
user_id,
}),
}
}
}
enum AssembledPayload<'a> {
Alert(Payload<'a>),
Default(revolt_a2::request::payload::Payload<'a>),
}
static Q: Lazy<Queue<ApnJob>> = Lazy::new(|| Queue::new(10_000));
/// Queue a new task for a worker
pub async fn queue(task: ApnJob) {
Q.try_push(task).ok();
info!("Queue is using {} slots from {}.", Q.len(), Q.capacity());
}
async fn get_badge_count(db: &Database, user: &str) -> Option<u32> {
if let Ok(unreads) = db.fetch_unreads(user).await {
let mut mention_count = 0;
for channel in unreads {
if let Some(mentions) = channel.mentions {
mention_count += mentions.len() as u32
}
}
return Some(mention_count);
}
None
}
/// Start a new worker
pub async fn worker(db: Database) {
let config = config().await;
if config.api.apn.pkcs8.is_empty()
|| config.api.apn.key_id.is_empty()
|| config.api.apn.team_id.is_empty()
{
eprintln!("Missing APN keys.");
return;
}
let endpoint = if config.api.apn.sandbox {
Endpoint::Sandbox
} else {
Endpoint::Production
};
let pkcs8 = engine::general_purpose::STANDARD
.decode(config.api.apn.pkcs8)
.expect("valid `pcks8`");
let client_config = ClientConfig::new(endpoint);
let client = Client::token(
&mut Cursor::new(pkcs8),
config.api.apn.key_id,
config.api.apn.team_id,
client_config,
)
.expect("could not create APN client");
let payload_options = NotificationOptions {
apns_id: None,
apns_push_type: Some(PushType::Alert),
apns_expiration: None,
apns_priority: Some(Priority::High),
apns_topic: Some("chat.revolt.app"),
apns_collapse_id: None,
};
loop {
let task = Q.pop().await;
let payload: AssembledPayload;
match task.job_type {
JobType::Alert(ref alert) => {
payload = AssembledPayload::Alert(Payload {
aps: APS {
alert: Some(APSAlert::Default(DefaultAlert {
title: Some(&alert.title),
subtitle: None,
body: Some(&alert.body),
title_loc_key: None,
title_loc_args: None,
action_loc_key: None,
loc_key: None,
loc_args: None,
launch_image: None,
})),
badge: get_badge_count(&db, &alert.user_id).await,
sound: Some(APSSound::Sound("default")),
thread_id: Some(&alert.thread_id),
content_available: None,
category: Some(&alert.category),
mutable_content: Some(1),
url_args: None,
},
device_token: &alert.device_token,
options: payload_options.clone(),
message: &alert.custom_payload.message,
url: &alert.custom_payload.url,
authorAvatar: &alert.custom_payload.authorAvatar,
authorDisplayName: &alert.custom_payload.authorDisplayName,
channelName: &alert.custom_payload.channelName,
});
}
JobType::Badge(ref alert) => {
payload = AssembledPayload::Default(revolt_a2::request::payload::Payload {
aps: APS {
alert: None,
badge: get_badge_count(&db, &alert.user_id).await,
sound: None,
thread_id: None,
content_available: None,
category: None,
mutable_content: None,
url_args: None,
},
device_token: &alert.device_token,
options: payload_options.clone(),
data: std::collections::BTreeMap::new(),
})
}
}
let resp = match payload {
AssembledPayload::Alert(p) => client.send(p).await,
AssembledPayload::Default(p) => client.send(p).await,
};
//println!("response from APNS: {:?}", resp);
if let Err(err) = resp {
match err {
Error::ResponseError(Response {
error:
Some(ErrorBody {
reason: ErrorReason::BadDeviceToken | ErrorReason::Unregistered,
..
}),
..
}) => {
if let Err(err) = db
.remove_push_subscription_by_session_id(match task.job_type {
JobType::Alert(ref a) => &a.session_id.as_str(),
JobType::Badge(ref a) => &a.session_id.as_str(),
})
.await
{
revolt_config::capture_error(&err);
}
}
err => {
revolt_config::capture_error(&err);
}
}
}
}
}

10
crates/core/database/src/tasks/mod.rs
View File

@@ -1,6 +1,6 @@
//! Semi-important background task management
use crate::Database;
use crate::{Database, AMQP};
use async_std::task;
use std::time::Instant;
@@ -8,22 +8,18 @@ use std::time::Instant;
const WORKER_COUNT: usize = 5;
pub mod ack;
pub mod apple_notifications;
pub mod authifier_relay;
pub mod last_message_id;
pub mod process_embeds;
pub mod web_push;
/// Spawn background workers
pub fn start_workers(db: Database, authifier_db: authifier::Database) {
pub fn start_workers(db: Database, amqp: AMQP) {
task::spawn(authifier_relay::worker());
task::spawn(apple_notifications::worker(db.clone()));
for _ in 0..WORKER_COUNT {
task::spawn(ack::worker(db.clone(), authifier_db.clone()));
task::spawn(ack::worker(db.clone(), amqp.clone()));
task::spawn(last_message_id::worker(db.clone()));
task::spawn(process_embeds::worker(db.clone()));
task::spawn(web_push::worker(db.clone(), authifier_db.clone()));
}
}

198
crates/core/database/src/tasks/web_push.rs
View File

@@ -1,198 +0,0 @@
use std::{
collections::{HashMap, HashSet},
time::Duration,
};
use authifier::Database as AuthifierDatabase;
use base64::{
engine::{self},
Engine as _,
};
use deadqueue::limited::Queue;
use fcm_v1::auth::{Authenticator, ServiceAccountKey};
use once_cell::sync::Lazy;
use revolt_config::{config, report_internal_error};
use revolt_models::v0::PushNotification;
use revolt_presence::filter_online;
use serde_json::json;
use web_push::{
ContentEncoding, IsahcWebPushClient, SubscriptionInfo, SubscriptionKeys, VapidSignatureBuilder,
WebPushClient, WebPushMessageBuilder,
};
use crate::Database;
use super::apple_notifications;
/// Task information
#[derive(Debug)]
struct PushTask {
/// User IDs of the targets that are to receive this notification
recipients: Vec<String>,
/// Push Notification
payload: PushNotification,
}
static Q: Lazy<Queue<PushTask>> = Lazy::new(|| Queue::new(10_000));
/// Queue a new task for a worker
pub async fn queue(recipients: Vec<String>, payload: PushNotification) {
if recipients.is_empty() {
return;
}
let online_ids = filter_online(&recipients).await;
let recipients = (&recipients.into_iter().collect::<HashSet<String>>() - &online_ids)
.into_iter()
.collect::<Vec<String>>();
Q.try_push(PushTask {
recipients,
payload,
})
.ok();
info!("Queue is using {} slots from {}.", Q.len(), Q.capacity());
}
/// Start a new worker
pub async fn worker(db: Database, authifier_db: AuthifierDatabase) {
let config = config().await;
let web_push_client = IsahcWebPushClient::new().unwrap();
let fcm_client = if config.api.fcm.key_type.is_empty() {
None
} else {
Some(fcm_v1::Client::new(
Authenticator::service_account::<&str>(ServiceAccountKey {
key_type: Some(config.api.fcm.key_type),
project_id: Some(config.api.fcm.project_id.clone()),
private_key_id: Some(config.api.fcm.private_key_id),
private_key: config.api.fcm.private_key,
client_email: config.api.fcm.client_email,
client_id: Some(config.api.fcm.client_id),
auth_uri: Some(config.api.fcm.auth_uri),
token_uri: config.api.fcm.token_uri,
auth_provider_x509_cert_url: Some(config.api.fcm.auth_provider_x509_cert_url),
client_x509_cert_url: Some(config.api.fcm.client_x509_cert_url),
})
.await
.unwrap(),
config.api.fcm.project_id,
false,
Duration::from_secs(5),
))
};
let web_push_private_key = engine::general_purpose::URL_SAFE_NO_PAD
.decode(config.api.vapid.private_key)
.expect("valid `VAPID_PRIVATE_KEY`");
loop {
let task = Q.pop().await;
if let Ok(sessions) = authifier_db
.find_sessions_with_subscription(&task.recipients)
.await
{
for session in sessions {
if let Some(sub) = session.subscription {
if sub.endpoint == "fcm" {
// Use Firebase Cloud Messaging
if let Some(client) = &fcm_client {
let message = fcm_v1::message::Message {
token: Some(sub.auth),
data: Some(HashMap::from([(
"payload".to_owned(),
serde_json::Value::String(json!(&task.payload).to_string()),
)])),
..Default::default()
};
if let Err(err) = client.send(&message).await {
error!("Failed to send FCM notification! {:?}", err);
if let fcm_v1::Error::FCM(fcm_error) = err {
if fcm_error.contains("404 (Not Found)") {
println!("Unregistering {:?}", session.id);
report_internal_error!(
db.remove_push_subscription_by_session_id(&session.id)
.await
)
.ok();
}
}
} else {
info!("Sent FCM notification to {:?}.", session.id);
}
} else {
info!("No FCM token was specified!");
}
} else if sub.endpoint == "apn" {
apple_notifications::queue(apple_notifications::ApnJob::from_notification(
session.id,
session.user_id,
sub.auth,
&task.payload,
))
.await;
} else {
// Use Web Push Standard
let subscription = SubscriptionInfo {
endpoint: sub.endpoint,
keys: SubscriptionKeys {
auth: sub.auth,
p256dh: sub.p256dh,
},
};
match VapidSignatureBuilder::from_pem(
std::io::Cursor::new(&web_push_private_key),
&subscription,
) {
Ok(sig_builder) => match sig_builder.build() {
Ok(signature) => {
let mut builder = WebPushMessageBuilder::new(&subscription);
builder.set_vapid_signature(signature);
let payload = json!(task.payload).to_string();
builder
.set_payload(ContentEncoding::AesGcm, payload.as_bytes());
match builder.build() {
Ok(msg) => match web_push_client.send(msg).await {
Ok(_) => {
info!(
"Sent Web Push notification to {:?}.",
session.id
)
}
Err(err) => {
error!("Hit error sending Web Push! {:?}", err)
}
},
Err(err) => {
error!(
"Failed to build message for {}! {:?}",
session.user_id, err
)
}
}
}
Err(err) => error!(
"Failed to build signature for {}! {:?}",
session.user_id, err
),
},
Err(err) => error!(
"Failed to create signature builder for {}! {:?}",
session.user_id, err
),
}
}
}
}
}
}
}

337
crates/core/database/src/util/bulk_permissions.rs Normal file
View File

@@ -0,0 +1,337 @@
use std::{collections::HashMap, hash::RandomState};
use revolt_permissions::{
ChannelPermission, ChannelType, Override, OverrideField, PermissionValue, ALLOW_IN_TIMEOUT,
DEFAULT_PERMISSION_DIRECT_MESSAGE,
};
use crate::{Channel, Database, Member, Server, User};
#[derive(Clone)]
pub struct BulkDatabasePermissionQuery<'a> {
#[allow(dead_code)]
database: &'a Database,
server: Server,
channel: Option<Channel>,
users: Option<Vec<User>>,
members: Option<Vec<Member>>,
// In case the users or members are fetched as part of the permissions checking operation
pub(crate) cached_users: Option<Vec<User>>,
pub(crate) cached_members: Option<Vec<Member>>,
cached_member_perms: Option<HashMap<String, PermissionValue>>,
}
impl<'z, 'x> BulkDatabasePermissionQuery<'x> {
pub async fn members_can_see_channel(&'z mut self) -> HashMap<String, bool>
where
'z: 'x,
{
let member_perms = if self.cached_member_perms.is_some() {
// This isn't done as an if let to prevent borrow checker errors with the mut self call when the perms aren't cached.
let perms = self.cached_member_perms.as_ref().unwrap();
perms
.iter()
.map(|(m, p)| {
(
m.clone(),
p.has_channel_permission(ChannelPermission::ViewChannel),
)
})
.collect()
} else {
calculate_members_permissions(self)
.await
.iter()
.map(|(m, p)| {
(
m.clone(),
p.has_channel_permission(ChannelPermission::ViewChannel),
)
})
.collect()
};
member_perms
}
}
impl<'z> BulkDatabasePermissionQuery<'z> {
pub fn new(database: &Database, server: Server) -> BulkDatabasePermissionQuery<'_> {
BulkDatabasePermissionQuery {
database,
server,
channel: None,
users: None,
members: None,
cached_members: None,
cached_users: None,
cached_member_perms: None,
}
}
pub async fn from_server_id<'a>(
db: &'a Database,
server: &str,
) -> BulkDatabasePermissionQuery<'a> {
BulkDatabasePermissionQuery {
database: db,
server: db.fetch_server(server).await.unwrap(),
channel: None,
users: None,
members: None,
cached_members: None,
cached_users: None,
cached_member_perms: None,
}
}
pub fn channel(self, channel: &'z Channel) -> BulkDatabasePermissionQuery {
BulkDatabasePermissionQuery {
channel: Some(channel.clone()),
..self
}
}
pub fn members(self, members: &'z [Member]) -> BulkDatabasePermissionQuery {
BulkDatabasePermissionQuery {
members: Some(members.to_owned()),
..self
}
}
pub fn users(self, users: &'z [User]) -> BulkDatabasePermissionQuery {
BulkDatabasePermissionQuery {
users: Some(users.to_owned()),
..self
}
}
/// Get the default channel permissions
/// Group channel defaults should be mapped to an allow-only override
#[allow(dead_code)]
async fn get_default_channel_permissions(&mut self) -> Override {
if let Some(channel) = &self.channel {
match channel {
Channel::Group { permissions, .. } => Override {
allow: permissions.unwrap_or(*DEFAULT_PERMISSION_DIRECT_MESSAGE as i64) as u64,
deny: 0,
},
Channel::TextChannel {
default_permissions,
..
}
| Channel::VoiceChannel {
default_permissions,
..
} => default_permissions.unwrap_or_default().into(),
_ => Default::default(),
}
} else {
Default::default()
}
}
#[allow(dead_code)]
fn get_channel_type(&mut self) -> ChannelType {
if let Some(channel) = &self.channel {
match channel {
Channel::DirectMessage { .. } => ChannelType::DirectMessage,
Channel::Group { .. } => ChannelType::Group,
Channel::SavedMessages { .. } => ChannelType::SavedMessages,
Channel::TextChannel { .. } | Channel::VoiceChannel { .. } => {
ChannelType::ServerChannel
}
}
} else {
ChannelType::Unknown
}
}
/// Get the ordered role overrides (from lowest to highest) for this member in this channel
#[allow(dead_code)]
async fn get_channel_role_overrides(&mut self) -> &HashMap<String, OverrideField> {
if let Some(channel) = &self.channel {
match channel {
Channel::TextChannel {
role_permissions, ..
}
| Channel::VoiceChannel {
role_permissions, ..
} => role_permissions,
_ => panic!("Not supported for non-server channels"),
}
} else {
panic!("No channel added to query")
}
}
}
/// Calculate members permissions in a server channel.
async fn calculate_members_permissions<'a>(
query: &'a mut BulkDatabasePermissionQuery<'a>,
) -> HashMap<String, PermissionValue> {
let mut resp = HashMap::new();
let (_, channel_role_permissions, channel_default_permissions) = match query
.channel
.as_ref()
.expect("A channel must be assigned to calculate channel permissions")
.clone()
{
Channel::TextChannel {
id,
role_permissions,
default_permissions,
..
}
| Channel::VoiceChannel {
id,
role_permissions,
default_permissions,
..
} => (id, role_permissions, default_permissions),
_ => panic!("Calculation of member permissions must be done on a server channel"),
};
if query.users.is_none() {
let ids: Vec<String> = query
.members
.as_ref()
.expect("No users or members added to the query")
.iter()
.map(|m| m.id.user.clone())
.collect();
query.cached_users = Some(
query
.database
.fetch_users(&ids[..])
.await
.expect("Failed to get data from the db"),
);
query.users = Some(query.cached_users.as_ref().unwrap().to_vec())
}
let users = query.users.as_ref().unwrap();
if query.members.is_none() {
let ids: Vec<String> = query
.users
.as_ref()
.expect("No users or members added to the query")
.iter()
.map(|m| m.id.clone())
.collect();
query.cached_members = Some(
query
.database
.fetch_members(&query.server.id, &ids[..])
.await
.expect("Failed to get data from the db"),
);
query.members = Some(query.cached_members.as_ref().unwrap().to_vec())
}
let members: HashMap<&String, &Member, RandomState> = HashMap::from_iter(
query
.members
.as_ref()
.unwrap()
.iter()
.map(|m| (&m.id.user, m)),
);
for user in users {
let member = members.get(&user.id);
// User isn't a part of the server
if member.is_none() {
resp.insert(user.id.clone(), 0_u64.into());
continue;
}
let member = *member.unwrap();
if user.privileged {
resp.insert(
user.id.clone(),
PermissionValue::from(ChannelPermission::GrantAllSafe),
);
continue;
}
if user.id == query.server.owner {
resp.insert(
user.id.clone(),
PermissionValue::from(ChannelPermission::GrantAllSafe),
);
continue;
}
// Get the user's server permissions
let mut permission = calculate_server_permissions(&query.server, user, member);
if let Some(defaults) = channel_default_permissions {
permission.apply(defaults.into());
}
// Get the applicable role overrides
let mut roles = channel_role_permissions
.iter()
.filter(|(id, _)| member.roles.contains(id))
.filter_map(|(id, permission)| {
query.server.roles.get(id).map(|role| {
let v: Override = (*permission).into();
(role.rank, v)
})
})
.collect::<Vec<(i64, Override)>>();
roles.sort_by(|a, b| b.0.cmp(&a.0));
let overrides = roles.into_iter().map(|(_, v)| v);
for role_override in overrides {
permission.apply(role_override)
}
resp.insert(user.id.clone(), permission);
}
resp
}
/// Calculates a member's server permissions
fn calculate_server_permissions(server: &Server, user: &User, member: &Member) -> PermissionValue {
if user.privileged || server.owner == user.id {
return ChannelPermission::GrantAllSafe.into();
}
let mut permissions: PermissionValue = server.default_permissions.into();
let mut roles = server
.roles
.iter()
.filter(|(id, _)| member.roles.contains(id))
.map(|(_, role)| {
let v: Override = role.permissions.into();
(role.rank, v)
})
.collect::<Vec<(i64, Override)>>();
roles.sort_by(|a, b| b.0.cmp(&a.0));
let role_overrides: Vec<Override> = roles.into_iter().map(|(_, v)| v).collect();
for role in role_overrides {
permissions.apply(role);
}
if member.in_timeout() {
permissions.restrict(*ALLOW_IN_TIMEOUT);
}
permissions
}

4
crates/core/database/src/util/idempotency.rs
View File

@@ -102,7 +102,7 @@ impl<'r> FromRequest<'r> for IdempotencyKey {
.map(|k| k.to_string())
{
if key.len() > 64 {
return Outcome::Failure((
return Outcome::Error((
Status::BadRequest,
create_error!(FailedValidation {
error: "idempotency key too long".to_string(),
@@ -113,7 +113,7 @@ impl<'r> FromRequest<'r> for IdempotencyKey {
let idempotency = IdempotencyKey { key };
let mut cache = TOKEN_CACHE.lock().await;
if cache.get(&idempotency.key).is_some() {
return Outcome::Failure((Status::Conflict, create_error!(DuplicateNonce)));
return Outcome::Error((Status::Conflict, create_error!(DuplicateNonce)));
}
cache.put(idempotency.key.clone(), ());

1
crates/core/database/src/util/mod.rs
View File

@@ -1,4 +1,5 @@
pub mod bridge;
pub mod bulk_permissions;
pub mod idempotency;
pub mod permissions;
pub mod reference;

6
crates/core/files/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-files"
version = "0.7.19"
version = "0.8.2"
edition = "2021"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -20,10 +20,10 @@ typenum = "1.17.0"
aws-config = "1.5.5"
aws-sdk-s3 = { version = "1.46.0", features = ["behavior-version-latest"] }
revolt-config = { version = "0.7.19", path = "../config", features = [
revolt-config = { version = "0.8.2", path = "../config", features = [
"report-macros",
] }
revolt-result = { version = "0.7.19", path = "../result" }
revolt-result = { version = "0.8.2", path = "../result" }
# image processing
jxl-oxide = "0.8.1"

6
crates/core/models/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-models"
version = "0.7.19"
version = "0.8.2"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -20,8 +20,8 @@ default = ["serde", "partials", "rocket"]
[dependencies]
# Core
revolt-config = { version = "0.7.19", path = "../config" }
revolt-permissions = { version = "0.7.19", path = "../permissions" }
revolt-config = { version = "0.8.2", path = "../config" }
revolt-permissions = { version = "0.8.2", path = "../permissions" }
# Utility
regex = "1"

14
crates/core/models/src/v0/channels.rs
View File

@@ -306,4 +306,18 @@ impl Channel {
| Channel::VoiceChannel { id, .. } => id,
}
}
/// This returns a Result because the recipient name can't be determined here without a db call,
/// which can't be done since this is models, which can't reference the database crate.
///
/// If it returns Err, you need to fetch the name from the db.
pub fn name(&self) -> Result<&str, ()> {
match self {
Channel::DirectMessage { .. } => Err(()),
Channel::SavedMessages { .. } => Ok("Saved Messages"),
Channel::TextChannel { name, .. }
| Channel::Group { name, .. }
| Channel::VoiceChannel { name, .. } => Ok(name),
}
}
}

15
crates/core/models/src/v0/messages.rs
View File

@@ -13,7 +13,7 @@ use rocket::{FromForm, FromFormField};
use iso8601_timestamp::Timestamp;
use super::{Embed, File, Member, MessageWebhook, User, Webhook, RE_COLOUR};
use super::{Channel, Embed, File, Member, MessageWebhook, User, Webhook, RE_COLOUR};
pub static RE_MENTION: Lazy<Regex> =
Lazy::new(|| Regex::new(r"<@([0-9A-HJKMNP-TV-Z]{26})>").unwrap());
@@ -209,6 +209,8 @@ auto_derived!(
pub url: String,
/// The message object itself, to send to clients for processing
pub message: Message,
/// The channel object itself, for clients to process
pub channel: Channel,
}
/// Representation of a text embed before it is sent.
@@ -278,7 +280,7 @@ auto_derived!(
pub struct OptionsQueryMessages {
/// Maximum number of messages to fetch
///
/// For fetching nearby messages, this is \`(limit + 1)\`.
/// For fetching nearby messages, this is \`(limit + 2)\`.
#[cfg_attr(feature = "validator", validate(range(min = 1, max = 100)))]
pub limit: Option<i64>,
/// Message id before which messages should be fetched
@@ -369,7 +371,7 @@ auto_derived!(
/// Optional fields on message
pub enum FieldsMessage {
Pinned
Pinned,
}
);
@@ -442,7 +444,7 @@ impl From<SystemMessage> for String {
impl PushNotification {
/// Create a new notification from a given message, author and channel ID
pub async fn from(msg: Message, author: Option<MessageAuthor<'_>>, channel_id: &str) -> Self {
pub async fn from(msg: Message, author: Option<MessageAuthor<'_>>, channel: Channel) -> Self {
let config = config().await;
let icon = if let Some(author) = &author {
@@ -496,10 +498,11 @@ impl PushNotification {
icon,
image,
body,
tag: channel_id.to_string(),
tag: channel.id().to_string(),
timestamp,
url: format!("{}/channel/{}/{}", config.hosts.app, channel_id, msg.id),
url: format!("{}/channel/{}/{}", config.hosts.app, channel.id(), msg.id),
message: msg,
channel,
}
}
}

4
crates/core/permissions/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-permissions"
version = "0.7.19"
version = "0.8.2"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] }
[dependencies]
# Core
revolt-result = { version = "0.7.19", path = "../result" }
revolt-result = { version = "0.8.2", path = "../result" }
# Utility
auto_ops = "0.3.0"

2
crates/core/presence/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-presence"
version = "0.7.19"
version = "0.8.2"
edition = "2021"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <me@insrt.uk>"]

4
crates/core/result/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-result"
version = "0.7.19"
version = "0.8.2"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -29,7 +29,7 @@ utoipa = { version = "4.2.3", optional = true }
# Rocket
rocket = { optional = true, version = "0.5.0-rc.2", default-features = false }
revolt_rocket_okapi = { version = "0.9.1", optional = true }
revolt_rocket_okapi = { version = "0.10.0", optional = true }
revolt_okapi = { version = "0.9.1", optional = true }
# Axum

18
crates/core/result/src/lib.rs
View File

@@ -116,6 +116,7 @@ pub enum ErrorType {
max: usize,
},
AlreadyInServer,
CannotTimeoutYourself,
// ? Bot related errors
ReachedMaximumBots,
@@ -189,23 +190,6 @@ macro_rules! create_database_error {
};
}
#[macro_export]
#[cfg(debug_assertions)]
macro_rules! query {
( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
Ok($self.$type($collection, $($rest),+).await.unwrap())
};
}
#[macro_export]
#[cfg(not(debug_assertions))]
macro_rules! query {
( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
$self.$type($collection, $($rest),+).await
.map_err(|_| create_database_error!(stringify!($type), $collection))
};
}
#[cfg(test)]
mod tests {
use crate::ErrorType;