jmug/stoatchat
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

feat: Add Mass Mentions to the backend (#394)

Browse Source 
* feat: create base of push daemon

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* Add outbound senders

* Make web_push send to rabbit instead (temp stuff)

* feat: stability and friend requests

* make vapid fr stuff not suck

* swap naming of queue

* move pushd into daemons folder

* fix cargo file for move into daemons folder

* feat: probably working fcm push notifs

* comment out fcm webpush stuff since the config keys dont exist

* fix fcm, name queues according to their prod status and configure routing keys

* add pushd to docker

* mix: Remove old code, add stuff to pushd

* fix: lockfile

* feat: update rocket to 5.0.1

* fix: fix queues and ack bugs

* Move rabbit messsage processing into ack queue

* chore: update readme

* chore: optimizations for ack database hits

* pushd flowchart

* misc: update flowchart

* exit dependancy hell

* add rocket_impl flag to authifier

* make the tests file of delta actually compile

* fix: don't silence every push message

* fix: don't silence all messages

* add debug logging for sending data to rabbit from message events

* validate mentions at a server membership level

* put back that import that was actually important

* minor fix to lockfile

* update delta authifier

* feat: proper permissions for push notifications

* add unit test for mention sanitization

* remove local file dependancy on authifier

* update ports to proper defaults

* fixTM the node bindings

* Theoretically configure docker releases for pushd

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* declare exchange in pushd and delta

* fix createbuckets script

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: reference db implementation

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: remove finally redundant code

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: changes

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: other changes

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: make channel name return result

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* Add role mention parsing

* feat: update to mongo 3.1, add member generator.

* integrate mass mentions into pushd

* patch redis-rs with updated versions

* feat: chunk role mentions

* move permission bits to 37/38 to avoid livekit conflict

* change role mention format to <%id>

* fix the lockfile from merge

* fix: PR change requests

* feat: add tests

* fix: i am a dumbass

* fix: tests, again

---------

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>
This commit is contained in:
Tom
2025-04-05 10:45:04 -07:00
committed by GitHub
parent 93a5de769d
commit 2540860129
51 changed files with 4083 additions and 2553 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/core/database/src/models/admin_migrations/ops/mongodb.rs
View File

@@ -10,7 +10,7 @@ impl AbstractMigrations for MongoDb {
#[cfg(test)]
/// Drop the database
async fn drop_database(&self) {
self.db().drop(None).await.ok();
self.db().drop().await.ok();
}
/// Migrate the database
@@ -18,7 +18,7 @@ impl AbstractMigrations for MongoDb {
info!("Migrating the database.");
let list = self
.list_database_names(None, None)
.list_database_names()
.await
.expect("Failed to fetch database names.");

342
crates/core/database/src/models/admin_migrations/ops/mongodb/init.rs
View File

@@ -8,278 +8,254 @@ pub async fn create_database(db: &MongoDb) {
info!("Creating database.");
let db = db.db();
db.create_collection("accounts", None)
db.create_collection("accounts")
.await
.expect("Failed to create accounts collection.");
db.create_collection("users", None)
db.create_collection("users")
.await
.expect("Failed to create users collection.");
db.create_collection("channels", None)
db.create_collection("channels")
.await
.expect("Failed to create channels collection.");
db.create_collection("messages", None)
db.create_collection("messages")
.await
.expect("Failed to create messages collection.");
db.create_collection("servers", None)
db.create_collection("servers")
.await
.expect("Failed to create servers collection.");
db.create_collection("server_members", None)
db.create_collection("server_members")
.await
.expect("Failed to create server_members collection.");
db.create_collection("server_bans", None)
db.create_collection("server_bans")
.await
.expect("Failed to create server_bans collection.");
db.create_collection("channel_invites", None)
db.create_collection("channel_invites")
.await
.expect("Failed to create channel_invites collection.");
db.create_collection("channel_unreads", None)
db.create_collection("channel_unreads")
.await
.expect("Failed to create channel_unreads collection.");
db.create_collection("channel_webhooks", None)
db.create_collection("channel_webhooks")
.await
.expect("Failed to create channel_webhooks collection.");
db.create_collection("migrations", None)
db.create_collection("migrations")
.await
.expect("Failed to create migrations collection.");
db.create_collection("attachments", None)
db.create_collection("attachments")
.await
.expect("Failed to create attachments collection.");
db.create_collection("attachment_hashes", None)
db.create_collection("attachment_hashes")
.await
.expect("Failed to create attachment_hashes collection.");
db.create_collection("user_settings", None)
db.create_collection("user_settings")
.await
.expect("Failed to create user_settings collection.");
db.create_collection("safety_reports", None)
db.create_collection("safety_reports")
.await
.expect("Failed to create safety_reports collection.");
db.create_collection("safety_snapshots", None)
db.create_collection("safety_snapshots")
.await
.expect("Failed to create safety_snapshots collection.");
db.create_collection("safety_strikes", None)
db.create_collection("safety_strikes")
.await
.expect("Failed to create safety_strikes collection.");
db.create_collection("bots", None)
db.create_collection("bots")
.await
.expect("Failed to create bots collection.");
db.create_collection("ratelimit_events", None)
db.create_collection("ratelimit_events")
.await
.expect("Failed to create ratelimit_events collection.");
db.create_collection(
"pubsub",
CreateCollectionOptions::builder()
.capped(true)
.size(1_000_000)
.build(),
)
.await
.expect("Failed to create pubsub collection.");
db.create_collection("pubsub")
.with_options(
CreateCollectionOptions::builder()
.capped(true)
.size(1_000_000)
.build(),
)
.await
.expect("Failed to create pubsub collection.");
db.run_command(
doc! {
"createIndexes": "users",
"indexes": [
{
"key": {
"username": 1_i32
},
"name": "username",
"unique": false,
"collation": {
"locale": "en",
"strength": 2_i32
}
db.run_command(doc! {
"createIndexes": "users",
"indexes": [
{
"key": {
"username": 1_i32
},
{
"key": {
"username": 1_i32,
"discriminator": 1_i32
},
"name": "username_discriminator",
"unique": true,
"collation": {
"locale": "en",
"strength": 2_i32
}
"name": "username",
"unique": false,
"collation": {
"locale": "en",
"strength": 2_i32
}
]
},
None,
)
},
{
"key": {
"username": 1_i32,
"discriminator": 1_i32
},
"name": "username_discriminator",
"unique": true,
"collation": {
"locale": "en",
"strength": 2_i32
}
}
]
})
.await
.expect("Failed to create username index.");
db.run_command(
doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"content": "text"
},
"name": "content"
db.run_command(doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"content": "text"
},
{
"key": {
"channel": 1_i32,
"_id": 1_i32
},
"name": "channel_id_compound"
"name": "content"
},
{
"key": {
"channel": 1_i32,
"_id": 1_i32
},
{
"key": {
"author": 1_i32
},
"name": "author"
"name": "channel_id_compound"
},
{
"key": {
"author": 1_i32
},
{
"key": {
"channel": 1_i32,
"pinned": 1_i32
},
"name": "channel_pinned_compound"
"name": "author"
},
{
"key": {
"channel": 1_i32,
"pinned": 1_i32
},
]
},
None,
)
"name": "channel_pinned_compound"
},
]
})
.await
.expect("Failed to create message index.");
db.run_command(
doc! {
"createIndexes": "channel_unreads",
"indexes": [
{
"key": {
"_id.channel": 1_i32,
"_id.user": 1_i32,
},
"name": "compound_id"
db.run_command(doc! {
"createIndexes": "channel_unreads",
"indexes": [
{
"key": {
"_id.channel": 1_i32,
"_id.user": 1_i32,
},
{
"key": {
"_id.user": 1_i32,
},
"name": "user_id"
}
]
},
None,
)
"name": "compound_id"
},
{
"key": {
"_id.user": 1_i32,
},
"name": "user_id"
}
]
})
.await
.expect("Failed to create channel_unreads index.");
db.run_command(
doc! {
"createIndexes": "server_members",
"indexes": [
{
"key": {
"_id.server": 1_i32,
"_id.user": 1_i32,
},
"name": "compound_id"
db.run_command(doc! {
"createIndexes": "server_members",
"indexes": [
{
"key": {
"_id.server": 1_i32,
"_id.user": 1_i32,
},
{
"key": {
"_id.user": 1_i32,
},
"name": "user_id"
}
]
},
None,
)
"name": "compound_id"
},
{
"key": {
"_id.user": 1_i32,
},
"name": "user_id"
}
]
})
.await
.expect("Failed to create server_members index.");
db.run_command(
doc! {
"createIndexes": "attachments",
"indexes": [
{
"key": {
"hash": 1_i32
},
"name": "hash"
db.run_command(doc! {
"createIndexes": "attachments",
"indexes": [
{
"key": {
"hash": 1_i32
},
{
"key": {
"used_for.id": 1_i32
},
"name": "used_for_id"
}
]
},
None,
)
"name": "hash"
},
{
"key": {
"used_for.id": 1_i32
},
"name": "used_for_id"
}
]
})
.await
.expect("Failed to create attachments index.");
db.run_command(
doc! {
"createIndexes": "attachment_hashes",
"indexes": [
{
"key": {
"processed_hash": 1_i32
},
"name": "processed_hash"
}
]
},
None,
)
db.run_command(doc! {
"createIndexes": "attachment_hashes",
"indexes": [
{
"key": {
"processed_hash": 1_i32
},
"name": "processed_hash"
}
]
})
.await
.expect("Failed to create attachment_hashes index.");
db.collection("migrations")
.insert_one(
doc! {
"_id": 0_i32,
"revision": LATEST_REVISION
},
None,
)
.insert_one(doc! {
"_id": 0_i32,
"revision": LATEST_REVISION
})
.await
.expect("Failed to save migration info.");
db.run_command(
doc! {
"createIndexes": "ratelimit_events",
"indexes": [
{
"key": {
"_id": 1_i32,
"target_id": 1_i32,
"event_type": 1_i32,
},
"name": "compound_key"
}
]
},
None,
)
db.run_command(doc! {
"createIndexes": "ratelimit_events",
"indexes": [
{
"key": {
"_id": 1_i32,
"target_id": 1_i32,
"event_type": 1_i32,
},
"name": "compound_key"
}
]
})
.await
.expect("Failed to create ratelimit_events index.");

571
crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs
View File

@@ -26,7 +26,7 @@ pub const LATEST_REVISION: i32 = 31;
pub async fn migrate_database(db: &MongoDb) {
let migrations = db.col::<Document>("migrations");
let data = migrations
.find_one(None, None)
.find_one(doc! {})
.await
.expect("Failed to fetch migration data.");
@@ -46,7 +46,6 @@ pub async fn migrate_database(db: &MongoDb) {
"revision": revision
}
},
None,
)
.await
.expect("Failed to commit migration information.");
@@ -74,7 +73,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
.update_many(
doc! { "attachment": { "$exists": 1_i32 } },
doc! { "$set": { "attachment.tag": "attachments", "attachment.size": 0_i32 } },
None,
)
.await
.expect("Failed to update messages.");
@@ -83,7 +81,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
.update_many(
doc! {},
doc! { "$set": { "tag": "attachments", "size": 0_i32 } },
None,
)
.await
.expect("Failed to update attachments.");
@@ -93,7 +90,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
info!("Running migration [revision 2 / 2021-05-08]: Add servers collection.");
db.db()
.create_collection("servers", None)
.create_collection("servers")
.await
.expect("Failed to create servers collection.");
}
@@ -103,12 +100,12 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
let messages = db.col::<Document>("messages");
let mut cursor = messages
.find(
doc! {
"attachment": {
"$exists": 1_i32
}
},
.find(doc! {
"attachment": {
"$exists": 1_i32
}
})
.with_options(
FindOptions::builder()
.projection(doc! {
"_id": 1_i32,
@@ -128,19 +125,18 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
.update_one(
doc! { "_id": id },
doc! { "$unset": { "attachment": 1_i32 }, "$set": { "attachments": attachments } },
None,
)
.await
.unwrap();
}
db.db()
.create_collection("channel_unreads", None)
.create_collection("channel_unreads")
.await
.expect("Failed to create channel_unreads collection.");
db.db()
.create_collection("user_settings", None)
.create_collection("user_settings")
.await
.expect("Failed to create user_settings collection.");
}
@@ -149,17 +145,17 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
info!("Running migration [revision 4 / 2021-06-01]: Add more server collections.");
db.db()
.create_collection("server_members", None)
.create_collection("server_members")
.await
.expect("Failed to create server_members collection.");
db.db()
.create_collection("server_bans", None)
.create_collection("server_bans")
.await
.expect("Failed to create server_bans collection.");
db.db()
.create_collection("channel_invites", None)
.create_collection("channel_invites")
.await
.expect("Failed to create channel_invites collection.");
}
@@ -182,7 +178,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
doc! {
"$set": to_document(&server).unwrap()
},
None,
)
.await
.expect("Failed to migrate servers.");
@@ -192,20 +187,17 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
info!("Running migration [revision 6 / 2021-07-09]: Add message text index.");
db.db()
.run_command(
doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"content": "text"
},
"name": "content"
}
]
},
None,
)
.run_command(doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"content": "text"
},
"name": "content"
}
]
})
.await
.expect("Failed to create message index.");
}
@@ -214,7 +206,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
info!("Running migration [revision 7 / 2021-08-11]: Add message text index.");
db.db()
.create_collection("bots", None)
.create_collection("bots")
.await
.expect("Failed to create bots collection.");
}
@@ -223,22 +215,15 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
info!("Running migration [revision 8 / 2021-09-10]: Update to Authifier version 1.");
db.db()
.run_command(
doc! {
"dropIndexes": "accounts",
"index": ["email", "email_normalised"]
},
None,
)
.run_command(doc! {
"dropIndexes": "accounts",
"index": ["email", "email_normalised"]
})
.await
.expect("Failed to delete legacy account indexes.");
let col = db.col::<Document>("sessions");
let mut cursor = db
.col::<Document>("accounts")
.find(doc! {}, None)
.await
.unwrap();
let mut cursor = db.col::<Document>("accounts").find(doc! {}).await.unwrap();
while let Some(doc) = cursor.next().await {
if let Ok(account) = doc {
@@ -267,7 +252,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
doc.insert("subscription", sub);
}
col.insert_one(doc, None).await.ok();
col.insert_one(doc).await.ok();
}
} else {
info!("Account doesn't have any sessions!");
@@ -288,7 +273,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
}
}
},
None,
)
.await
.unwrap();
@@ -297,11 +281,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
if revision <= 9 {
info!("Running migration [revision 9 / 2021-09-14]: Switch from last_message to last_message_id.");
let mut cursor = db
.col::<Document>("channels")
.find(doc! {}, None)
.await
.unwrap();
let mut cursor = db.col::<Document>("channels").find(doc! {}).await.unwrap();
while let Some(doc) = cursor.next().await {
if let Ok(channel) = doc {
@@ -340,7 +320,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
"last_message": 1_i32,
}
},
None,
)
.await
.unwrap();
@@ -362,7 +341,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
"nonce": 1_i32,
}
},
None,
)
.await
.unwrap();
@@ -375,7 +353,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
"nonce": 1_i32,
}
},
None,
)
.await
.unwrap();
@@ -385,70 +362,61 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
info!("Running migration [revision 11 / 2021-11-14]: Add indexes to database.");
db.db()
.run_command(
doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"channel": 1_i32
},
"name": "channel"
}
]
},
None,
)
.run_command(doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"channel": 1_i32
},
"name": "channel"
}
]
})
.await
.expect("Failed to create message index.");
db.db()
.run_command(
doc! {
"createIndexes": "channel_unreads",
"indexes": [
{
"key": {
"_id.channel": 1_i32,
"_id.user": 1_i32,
},
"name": "compound_id"
.run_command(doc! {
"createIndexes": "channel_unreads",
"indexes": [
{
"key": {
"_id.channel": 1_i32,
"_id.user": 1_i32,
},
{
"key": {
"_id.user": 1_i32,
},
"name": "user_id"
}
]
},
None,
)
"name": "compound_id"
},
{
"key": {
"_id.user": 1_i32,
},
"name": "user_id"
}
]
})
.await
.expect("Failed to create channel_unreads index.");
db.db()
.run_command(
doc! {
"createIndexes": "server_members",
"indexes": [
{
"key": {
"_id.server": 1_i32,
"_id.user": 1_i32,
},
"name": "compound_id"
.run_command(doc! {
"createIndexes": "server_members",
"indexes": [
{
"key": {
"_id.server": 1_i32,
"_id.user": 1_i32,
},
{
"key": {
"_id.user": 1_i32,
},
"name": "user_id"
}
]
},
None,
)
"name": "compound_id"
},
{
"key": {
"_id.user": 1_i32,
},
"name": "user_id"
}
]
})
.await
.expect("Failed to create server_members index.");
}
@@ -457,21 +425,18 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
info!("Running migration [revision 12 / 2021-11-21]: Add indexes to database.");
db.db()
.run_command(
doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"channel": 1_i32,
"_id": 1_i32
},
"name": "channel_id_compound"
}
]
},
None,
)
.run_command(doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"channel": 1_i32,
"_id": 1_i32
},
"name": "channel_id_compound"
}
]
})
.await
.expect("Failed to create message index.");
}
@@ -485,7 +450,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
async_std::task::sleep(Duration::from_secs(10)).await;
let servers = db.col::<Document>("servers");
let mut cursor = servers.find(doc! {}, None).await.unwrap();
let mut cursor = servers.find(doc! {}).await.unwrap();
while let Some(Ok(mut document)) = cursor.next().await {
let id = document.get_str("_id").unwrap().to_string();
@@ -529,13 +494,13 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
}
servers
.update_one(doc! { "_id": id }, doc! { "$set": update }, None)
.update_one(doc! { "_id": id }, doc! { "$set": update })
.await
.unwrap();
}
let channels = db.col::<Document>("channels");
let mut cursor = channels.find(doc! {}, None).await.unwrap();
let mut cursor = channels.find(doc! {}).await.unwrap();
while let Some(Ok(document)) = cursor.next().await {
let id = document.get_str("_id").unwrap().to_string();
@@ -575,7 +540,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
}
channels
.update_one(doc! { "_id": id }, update, None)
.update_one(doc! { "_id": id }, update)
.await
.unwrap();
}
@@ -596,7 +561,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
"content": "system"
}
},
None,
)
.await
.unwrap();
@@ -621,25 +585,22 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
.unwrap();
db.db()
.create_collection("emojis", None)
.create_collection("emojis")
.await
.expect("Failed to create emojis collection.");
db.db()
.run_command(
doc! {
"createIndexes": "emojis",
"indexes": [
{
"key": {
"parent.id": 1_i32,
},
"name": "parent_id"
}
]
},
None,
)
.run_command(doc! {
"createIndexes": "emojis",
"indexes": [
{
"key": {
"parent.id": 1_i32,
},
"name": "parent_id"
}
]
})
.await
.expect("Failed to create emoji parent index.");
}
@@ -652,10 +613,9 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
doc! {},
doc! {
"$set": {
"joined_at": DateTime::now().to_rfc3339_string()
"joined_at": DateTime::now().try_to_rfc3339_string().expect("Failed to convert the date to rfc3339")
}
},
None,
)
.await
.expect("Failed to update server members.");
@@ -666,13 +626,10 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
if db
.db()
.run_command(
doc! {
"dropIndexes": "messages",
"index": ["channel"]
},
None,
)
.run_command(doc! {
"dropIndexes": "messages",
"index": ["channel"]
})
.await
.is_err()
{
@@ -680,20 +637,17 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
}
db.db()
.run_command(
doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"author": 1_i32,
},
"name": "author"
}
]
},
None,
)
.run_command(doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"author": 1_i32,
},
"name": "author"
}
]
})
.await
.expect("Failed to create messages author index.");
}
@@ -703,35 +657,26 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
"Running migration [revision 19 / 27-02-2023]: Create report / snapshot collections."
);
db.db()
.create_collection("safety_reports", None)
.await
.unwrap();
db.db().create_collection("safety_reports").await.unwrap();
db.db()
.create_collection("safety_snapshots", None)
.await
.unwrap();
db.db().create_collection("safety_snapshots").await.unwrap();
}
if revision <= 20 {
info!("Running migration [revision 20 / 28-02-2023]: Add index `snapshot.report_id`.");
db.db()
.run_command(
doc! {
"createIndexes": "safety_snapshots",
"indexes": [
{
"key": {
"report_id": 1_i32
},
"name": "report_id"
}
]
},
None,
)
.run_command(doc! {
"createIndexes": "safety_snapshots",
"indexes": [
{
"key": {
"report_id": 1_i32
},
"name": "report_id"
}
]
})
.await
.expect("Failed to create safety snapshot index.");
}
@@ -739,10 +684,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
if revision <= 21 {
info!("Running migration [revision 21 / 31-05-2023]: Add collection `safety_strikes`.");
db.db()
.create_collection("safety_strikes", None)
.await
.unwrap();
db.db().create_collection("safety_strikes").await.unwrap();
}
if revision <= 22 {
@@ -756,7 +698,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
"moderator_id": "01EX2NCWQ0CHS3QJF0FEQS1GR4"
}
},
None,
)
.await
.expect("Failed to update server members.");
@@ -766,13 +707,10 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
info!("Running migration [revision 23 / 10-06-2023]: Generate discriminators for users.");
db.db()
.run_command(
doc! {
"dropIndexes": "users",
"index": "username"
},
None,
)
.run_command(doc! {
"dropIndexes": "users",
"index": "username"
})
.await
.expect("Failed to drop existing username index.");
@@ -787,7 +725,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
let users: Vec<UserInformation> = db
.col::<UserInformation>("users")
.find(doc! {}, None)
.find(doc! {})
.await
.unwrap()
.map(|doc| doc.expect("id and username"))
@@ -839,7 +777,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
"discriminator": discriminator
}
},
None,
)
.await
.unwrap();
@@ -887,7 +824,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
"display_name": &info.username
}
},
None,
)
.await
.unwrap();
@@ -898,43 +834,37 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
if revision <= 24 {
info!("Running migration [revision 24 / 09-06-2023]: Add collection `channel_webhooks` if not exists, update users index.");
db.db()
.create_collection("channel_webhooks", None)
.await
.ok();
db.db().create_collection("channel_webhooks").await.ok();
db.db()
.run_command(
doc! {
"createIndexes": "users",
"indexes": [
{
"key": {
"username": 1_i32
},
"name": "username",
"unique": false,
"collation": {
"locale": "en",
"strength": 2_i32
}
.run_command(doc! {
"createIndexes": "users",
"indexes": [
{
"key": {
"username": 1_i32
},
{
"key": {
"username": 1_i32,
"discriminator": 1_i32
},
"name": "username_discriminator",
"unique": true,
"collation": {
"locale": "en",
"strength": 2_i32
}
"name": "username",
"unique": false,
"collation": {
"locale": "en",
"strength": 2_i32
}
]
},
None,
)
},
{
"key": {
"username": 1_i32,
"discriminator": 1_i32
},
"name": "username_discriminator",
"unique": true,
"collation": {
"locale": "en",
"strength": 2_i32
}
}
]
})
.await
.expect("Failed to create username index.");
};
@@ -950,7 +880,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
"permissions": *DEFAULT_WEBHOOK_PERMISSIONS as i64
}
},
None,
)
.await
.expect("Failed to update webhooks.");
@@ -959,28 +888,22 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
if revision <= 25 {
info!("Running migration [revision 25 / 15-06-2023]: Add collection `ratelimit_events` with index.");
db.db()
.create_collection("ratelimit_events", None)
.await
.ok();
db.db().create_collection("ratelimit_events").await.ok();
db.db()
.run_command(
doc! {
"createIndexes": "ratelimit_events",
"indexes": [
{
"key": {
"_id": 1_i32,
"target_id": 1_i32,
"event_type": 1_i32,
},
"name": "compound_key"
}
]
},
None,
)
.run_command(doc! {
"createIndexes": "ratelimit_events",
"indexes": [
{
"key": {
"_id": 1_i32,
"target_id": 1_i32,
"event_type": 1_i32,
},
"name": "compound_key"
}
]
})
.await
.expect("Failed to create ratelimit_events index.");
}
@@ -1016,12 +939,9 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
let invites = db
.db()
.collection::<Outer>("channel_invites")
.find(
doc! {
"type": { "$exists": false }
},
None,
)
.find(doc! {
"type": { "$exists": false }
})
.await
.expect("failed to find invites")
.filter_map(|s| async { s.ok() })
@@ -1055,18 +975,15 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
if !invites.is_empty() {
db.db()
.collection("channel_invites")
.insert_many(invites, None)
.insert_many(invites)
.await
.expect("failed to insert corrected invite");
db.db()
.collection::<Outer>("channel_invites")
.delete_many(
doc! {
"type": { "$exists": false }
},
None,
)
.delete_many(doc! {
"type": { "$exists": false }
})
.await
.expect("failed to find invites");
}
@@ -1076,21 +993,18 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
info!("Running migration [revision 27 / 21-07-2024]: create message pinned index.");
db.db()
.run_command(
doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"channel": 1_i32,
"pinned": 1_i32
},
"name": "channel_pinned_compound"
}
]
},
None,
)
.run_command(doc! {
"createIndexes": "messages",
"indexes": [
{
"key": {
"channel": 1_i32,
"pinned": 1_i32
},
"name": "channel_pinned_compound"
}
]
})
.await
.expect("Failed to create message index.");
}
@@ -1098,44 +1012,35 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
if revision <= 28 {
info!("Running migration [revision 28 / 10-09-2024]: Add support for new Autumn.");
db.db()
.create_collection("attachment_hashes", None)
.await
.ok();
db.db().create_collection("attachment_hashes").await.ok();
db.db()
.run_command(
doc! {
"createIndexes": "attachments",
"indexes": [
{
"key": {
"hash": 1_i32
},
"name": "hash"
}
]
},
None,
)
.run_command(doc! {
"createIndexes": "attachments",
"indexes": [
{
"key": {
"hash": 1_i32
},
"name": "hash"
}
]
})
.await
.expect("Failed to create attachments index.");
db.db()
.run_command(
doc! {
"createIndexes": "attachment_hashes",
"indexes": [
{
"key": {
"processed_hash": 1_i32
},
"name": "processed_hash"
}
]
},
None,
)
.run_command(doc! {
"createIndexes": "attachment_hashes",
"indexes": [
{
"key": {
"processed_hash": 1_i32
},
"name": "processed_hash"
}
]
})
.await
.expect("Failed to create attachment_hashes index.");
}
@@ -1146,20 +1051,17 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
info!("Running migration [revision 30 / 29-09-2024]: Add index for used_for.id to attachments.");
db.db()
.run_command(
doc! {
"createIndexes": "attachments",
"indexes": [
{
"key": {
"used_for.id": 1_i32
},
"name": "used_for_id"
}
]
},
None,
)
.run_command(doc! {
"createIndexes": "attachments",
"indexes": [
{
"key": {
"used_for.id": 1_i32
},
"name": "used_for_id"
}
]
})
.await
.expect("Failed to create attachments index.");
}
@@ -1176,7 +1078,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
let webhooks = db
.db()
.collection::<WebhookShell>("channel_webhooks")
.find(doc! {}, None)
.find(doc! {})
.await
.expect("webhooks")
.filter_map(|s| async { s.ok() })
@@ -1207,7 +1109,6 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
"creator_id": creator_id
}
},
None,
)
.await
.expect("update webhook");
@@ -1218,7 +1119,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
}) => {
db.db()
.collection::<WebhookShell>("channel_webhooks")
.delete_one(doc! { "_id": webhook._id }, None)
.delete_one(doc! { "_id": webhook._id })
.await
.expect("failed to delete invalid webhook");
}

9
crates/core/database/src/models/channel_invites/ops/mongodb.rs
View File

@@ -24,12 +24,9 @@ impl AbstractChannelInvites for MongoDb {
async fn fetch_invites_for_server(&self, server_id: &str) -> Result<Vec<Invite>> {
Ok(self
.col::<Invite>(COL)
.find(
doc! {
"server": server_id,
},
None,
)
.find(doc! {
"server": server_id,
})
.await
.map_err(|_| create_database_error!("find", COL))?
.filter_map(|s| async {

8
crates/core/database/src/models/channel_unreads/ops.rs
View File

@@ -26,6 +26,14 @@ pub trait AbstractChannelUnreads: Sync + Send {
message_ids: &[String],
) -> Result<()>;
/// Add a mention.
async fn add_mention_to_many_unreads<'a>(
&self,
channel_id: &str,
user_ids: &[String],
message_ids: &[String],
) -> Result<()>;
/// Fetch all unreads with mentions for a user.
async fn fetch_unread_mentions(&self, user_id: &str) -> Result<Vec<ChannelUnread>>;

47
crates/core/database/src/models/channel_unreads/ops/mongodb.rs
View File

@@ -37,6 +37,8 @@ impl AbstractChannelUnreads for MongoDb {
"last_id": message_id
}
},
)
.with_options(
FindOneAndUpdateOptions::builder()
.upsert(true)
.return_document(ReturnDocument::After)
@@ -51,15 +53,12 @@ impl AbstractChannelUnreads for MongoDb {
let current_time = Ulid::new().to_string();
self.col::<Document>(COL)
.delete_many(
doc! {
"_id.channel": {
"$in": channel_ids
},
"_id.user": user_id
.delete_many(doc! {
"_id.channel": {
"$in": channel_ids
},
None,
)
"_id.user": user_id
})
.await
.map_err(|_| create_database_error!("delete_many", COL))?;
@@ -77,7 +76,6 @@ impl AbstractChannelUnreads for MongoDb {
}
})
.collect::<Vec<Document>>(),
None,
)
.await
.map(|_| ())
@@ -104,13 +102,42 @@ impl AbstractChannelUnreads for MongoDb {
}
}
},
UpdateOptions::builder().upsert(true).build(),
)
.with_options(UpdateOptions::builder().upsert(true).build())
.await
.map(|_| ())
.map_err(|_| create_database_error!("update_one", COL))
}
/// Add a mention to multiple users.
async fn add_mention_to_many_unreads<'a>(
&self,
channel_id: &str,
user_ids: &[String],
message_ids: &[String],
) -> Result<()> {
self.col::<Document>(COL)
.update_many(
doc! {
"_id.channel": channel_id,
"_id.user": {
"$in": user_ids
},
},
doc! {
"$push": {
"mentions": {
"$each": message_ids
}
}
},
)
.with_options(UpdateOptions::builder().upsert(true).build())
.await
.map(|_| ())
.map_err(|_| create_database_error!("update_many", COL))
}
/// Fetch all channel unreads for a user.
async fn fetch_unreads(&self, user_id: &str) -> Result<Vec<ChannelUnread>> {
query!(

32
crates/core/database/src/models/channel_unreads/ops/reference.rs
View File

@@ -78,6 +78,38 @@ impl AbstractChannelUnreads for ReferenceDb {
Ok(())
}
/// Add a mention to multiple users.
async fn add_mention_to_many_unreads<'a>(
&self,
channel_id: &str,
user_ids: &[String],
message_ids: &[String],
) -> Result<()> {
let mut unreads = self.channel_unreads.lock().await;
for user_id in user_ids {
let key = ChannelCompositeKey {
channel: channel_id.to_string(),
user: user_id.to_string(),
};
if let Some(unread) = unreads.get_mut(&key) {
unread.mentions.replace(message_ids.to_vec());
} else {
unreads.insert(
key.clone(),
ChannelUnread {
id: key,
last_id: None,
mentions: Some(message_ids.to_vec()),
},
);
}
}
Ok(())
}
async fn fetch_unread_mentions(&self, user_id: &str) -> Result<Vec<ChannelUnread>> {
let unreads = self.channel_unreads.lock().await;
Ok(unreads

9
crates/core/database/src/models/channel_webhooks/ops/mongodb.rs
View File

@@ -24,12 +24,9 @@ impl AbstractWebhooks for MongoDb {
async fn fetch_webhooks_for_channel(&self, channel_id: &str) -> Result<Vec<Webhook>> {
Ok(self
.col::<Webhook>(COL)
.find(
doc! {
"channel_id": channel_id,
},
None,
)
.find(doc! {
"channel_id": channel_id,
})
.await
.map_err(|_| create_database_error!("find", COL))?
.filter_map(|s| async {

44
crates/core/database/src/models/channels/ops/mongodb.rs
View File

@@ -23,14 +23,11 @@ impl AbstractChannels for MongoDb {
async fn fetch_channels<'a>(&self, ids: &'a [String]) -> Result<Vec<Channel>> {
Ok(self
.col::<Channel>(COL)
.find(
doc! {
"_id": {
"$in": ids
}
},
None,
)
.find(doc! {
"_id": {
"$in": ids
}
})
.await
.map_err(|_| create_database_error!("fetch", "channels"))?
.filter_map(|s| async {
@@ -119,7 +116,6 @@ impl AbstractChannels for MongoDb {
"recipients": user
}
},
None,
)
.await
.map(|_| ())
@@ -141,7 +137,6 @@ impl AbstractChannels for MongoDb {
"role_permissions.".to_owned() + role: permissions
}
},
None,
)
.await
.map(|_| ())
@@ -179,7 +174,6 @@ impl AbstractChannels for MongoDb {
"recipients": user
}
},
None,
)
.await
.map(|_| ())
@@ -253,7 +247,6 @@ impl AbstractChannels for MongoDb {
"_id": server.id
},
update,
None,
)
.await
.map_err(|_| create_database_error!("update_one", "servers"))?;
@@ -274,23 +267,17 @@ impl MongoDb {
pub async fn delete_associated_channel_objects(&self, id: Bson) -> Result<()> {
// Delete all invites to these channels.
self.col::<Document>("channel_invites")
.delete_many(
doc! {
"channel": &id
},
None,
)
.delete_many(doc! {
"channel": &id
})
.await
.map_err(|_| create_database_error!("delete_many", "channel_invites"))?;
// Delete unread message objects on channels.
self.col::<Document>("channel_unreads")
.delete_many(
doc! {
"_id.channel": &id
},
None,
)
.delete_many(doc! {
"_id.channel": &id
})
.await
.map_err(|_| create_database_error!("delete_many", "channel_unreads"))
.map(|_| ())?;
@@ -299,12 +286,9 @@ impl MongoDb {
// Delete all webhooks on this channel.
self.col::<Document>("webhooks")
.delete_many(
doc! {
"channel": &id
},
None,
)
.delete_many(doc! {
"channel": &id
})
.await
.map_err(|_| create_database_error!("delete_many", "webhooks"))
.map(|_| ())

1
crates/core/database/src/models/emojis/ops/mongodb.rs
View File

@@ -60,7 +60,6 @@ impl AbstractEmojis for MongoDb {
}
}
},
None,
)
.await
.map(|_| ())

1
crates/core/database/src/models/file_hashes/ops/mongodb.rs
View File

@@ -42,7 +42,6 @@ impl AbstractAttachmentHashes for MongoDb {
"iv": nonce
}
},
None,
)
.await
.map(|_| ())

5
crates/core/database/src/models/files/ops/mongodb.rs
View File

@@ -110,7 +110,6 @@ impl AbstractAttachments for MongoDb {
"uploader_id": uploader_id
}
},
None,
)
.await
.map_err(|_| create_database_error!("update_one", COL))?;
@@ -130,7 +129,6 @@ impl AbstractAttachments for MongoDb {
"reported": true
}
},
None,
)
.await
.map(|_| ())
@@ -149,7 +147,6 @@ impl AbstractAttachments for MongoDb {
"deleted": true
}
},
None,
)
.await
.map(|_| ())
@@ -170,7 +167,6 @@ impl AbstractAttachments for MongoDb {
"deleted": true
}
},
None,
)
.await
.map(|_| ())
@@ -193,7 +189,6 @@ impl MongoDb {
"deleted": true
}
},
None,
)
.await
.map(|_| ())

212
crates/core/database/src/models/messages/model.rs
View File

@@ -5,9 +5,10 @@ use iso8601_timestamp::Timestamp;
use revolt_config::{config, FeaturesLimits};
use revolt_models::v0::{
self, BulkMessageResponse, DataMessageSend, Embed, MessageAuthor, MessageFlags, MessageSort,
MessageWebhook, PushNotification, ReplyIntent, SendableEmbed, Text, RE_MENTION,
MessageWebhook, PushNotification, ReplyIntent, SendableEmbed, Text, RE_BLOCKS, RE_MENTION,
RE_ROLE_MENTION,
};
use revolt_permissions::{ChannelPermission, PermissionValue};
use revolt_permissions::{calculate_channel_permissions, ChannelPermission, PermissionValue};
use revolt_result::{ErrorType, Result};
use ulid::Ulid;
use validator::Validate;
@@ -15,7 +16,10 @@ use validator::Validate;
use crate::{
events::client::EventV1,
tasks::{self, ack::AckEvent},
util::{bulk_permissions::BulkDatabasePermissionQuery, idempotency::IdempotencyKey},
util::{
bulk_permissions::BulkDatabasePermissionQuery, idempotency::IdempotencyKey,
permissions::DatabasePermissionQuery,
},
Channel, Database, Emoji, File, User, AMQP,
};
@@ -53,6 +57,9 @@ auto_derived_partial!(
/// Array of user ids mentioned in this message
#[serde(skip_serializing_if = "Option::is_none")]
pub mentions: Option<Vec<String>>,
/// Array of role ids mentioned in this message
#[serde(skip_serializing_if = "Option::is_none")]
pub role_mentions: Option<Vec<String>>,
/// Array of message ids this message is replying to
#[serde(skip_serializing_if = "Option::is_none")]
pub replies: Option<Vec<String>>,
@@ -71,7 +78,7 @@ auto_derived_partial!(
/// Bitfield of message flags
#[serde(skip_serializing_if = "Option::is_none")]
pub flags: Option<i32>,
pub flags: Option<u32>,
},
"PartialMessage"
);
@@ -199,6 +206,30 @@ auto_derived!(
}
);
pub struct MessageFlagsValue(pub u32);
impl MessageFlagsValue {
pub fn has(&self, flag: MessageFlags) -> bool {
self.has_value(flag as u32)
}
pub fn has_value(&self, bit: u32) -> bool {
let mask = 1 << bit;
self.0 & mask == mask
}
pub fn set(&mut self, flag: MessageFlags, toggle: bool) -> &mut Self {
self.set_value(flag as u32, toggle)
}
pub fn set_value(&mut self, bit: u32, toggle: bool) -> &mut Self {
if toggle {
self.0 |= 1 << bit;
} else {
self.0 &= !(1 << bit);
}
self
}
}
#[allow(clippy::derivable_impls)]
impl Default for Message {
fn default() -> Self {
@@ -214,6 +245,7 @@ impl Default for Message {
edited: None,
embeds: None,
mentions: None,
role_mentions: None,
replies: None,
reactions: Default::default(),
interactions: Default::default(),
@@ -262,13 +294,43 @@ impl Message {
return Err(create_error!(EmptyMessage));
}
// Ensure flags are either not set or have permissible values
if let Some(flags) = &data.flags {
if flags != &0 && flags != &1 {
let allow_mass_mentions = allow_mentions && config.features.mass_mentions_enabled;
let mut mentions_everyone = false;
let mut mentions_online = false;
let mut suppress_notifications = false;
if let Some(raw_flags) = &data.flags {
if raw_flags > &7 {
// quick path to failure: bigger than all the bits combined
return Err(create_error!(InvalidProperty));
}
// First step of mass mention resolution
let flags = MessageFlagsValue(*raw_flags);
suppress_notifications = flags.has(MessageFlags::SuppressNotifications);
mentions_everyone = allow_mentions && flags.has(MessageFlags::MentionsEveryone);
mentions_online = allow_mentions && flags.has(MessageFlags::MentionsOnline);
// Not a bot, and attempting to set mention flags
if user.as_ref().is_some_and(|u| u.bot.as_ref().is_none())
&& (mentions_everyone || mentions_online)
{
return Err(create_error!(IsNotBot));
}
if mentions_everyone && mentions_online {
return Err(create_error!(InvalidFlagValue));
}
}
let server_id = match channel {
Channel::TextChannel { ref server, .. } | Channel::VoiceChannel { ref server, .. } => {
Some(server.clone())
}
_ => None,
};
// Ensure restrict_reactions is not specified without reactions list
if let Some(interactions) = &data.interactions {
if interactions.restrict_reactions {
@@ -302,19 +364,99 @@ impl Message {
.unwrap_or_default(),
author: author_id,
webhook: webhook.map(|w| w.into()),
flags: data.flags.map(|v| v as i32),
flags: data.flags,
..Default::default()
};
// Parse mentions in message.
let mut mentions = HashSet::new();
if allow_mentions {
if let Some(content) = &data.content {
for capture in RE_MENTION.captures_iter(content) {
if let Some(mention) = capture.get(1) {
mentions.insert(mention.as_str().to_string());
let mut role_mentions = HashSet::new();
let mut role_model_mentions = vec![];
if let Some(raw_content) = &data.content {
// remove all codeblocks before running the mention regex
let modified_content = RE_BLOCKS.replace_all(raw_content, "");
for capture in RE_MENTION.captures_iter(&modified_content).flatten() {
if let Some(mention) = capture.get(1) {
mentions.insert(mention.as_str().to_string());
}
}
if allow_mass_mentions && server_id.is_some() {
// Second step of mass mention resolution.
for capture in RE_ROLE_MENTION.captures_iter(&modified_content).flatten() {
let raw = capture.get(0).expect("No capture?").as_str();
if raw == "@everyone" {
mentions_everyone = true;
// Can't break early here since we want to capture any role ids
} else if raw == "@online" {
mentions_online = true;
} else if let Some(role_match) = capture.get(1) {
role_mentions.insert(role_match.as_str().to_string());
}
}
if !role_mentions.is_empty() {
let server_data = db
.fetch_server(server_id.unwrap().as_str())
.await
.expect("Failed to fetch server");
role_mentions = role_mentions
.iter()
.filter(|role_id| server_data.roles.contains_key(*role_id))
.cloned()
.collect();
role_model_mentions.extend(
role_mentions
.iter()
.map(|role_id| server_data.roles.get(role_id).to_owned()),
);
}
}
}
// Validate the user can perform a mass mention
if !config.features.mass_mentions_enabled
&& (mentions_everyone || mentions_online || !role_mentions.is_empty())
{
mentions_everyone = false;
mentions_online = false;
role_mentions.clear();
} else if mentions_everyone || mentions_online || !role_mentions.is_empty() {
debug!(
"Mentioned everyone: {}, mentioned online: {}, mentioned roles: {:?}",
mentions_everyone, mentions_online, &role_mentions
);
if let Some(user) = match author {
MessageAuthor::User(user) => Some(Ok(user)),
MessageAuthor::System { .. } => Some(Err(())), // DISALLOWED
MessageAuthor::Webhook(..) => None, // Bypass check
} {
if user.is_err() {
return Err(create_error!(InvalidProperty));
}
let owned_user: User = user.unwrap().to_owned().into();
let mut query = DatabasePermissionQuery::new(db, &owned_user).channel(&channel);
let perms = calculate_channel_permissions(&mut query).await;
if (mentions_everyone || mentions_online)
&& !perms.has_channel_permission(ChannelPermission::MentionEveryone)
{
return Err(create_error!(MissingPermission {
permission: ChannelPermission::MentionEveryone.to_string()
}));
}
if !role_mentions.is_empty()
&& !perms.has_channel_permission(ChannelPermission::MentionRoles)
{
return Err(create_error!(MissingPermission {
permission: ChannelPermission::MentionRoles.to_string()
}));
}
}
}
@@ -363,6 +505,7 @@ impl Message {
let recipients_hash: HashSet<&String, RandomState> =
HashSet::from_iter(recipients);
mentions.retain(|m| recipients_hash.contains(m));
role_mentions.clear();
}
Channel::TextChannel { ref server, .. }
| Channel::VoiceChannel { ref server, .. } => {
@@ -403,12 +546,27 @@ impl Message {
message.mentions.replace(mentions.into_iter().collect());
}
if !role_mentions.is_empty() {
message
.role_mentions
.replace(role_mentions.into_iter().collect());
}
if !replies.is_empty() {
message
.replies
.replace(replies.into_iter().collect::<Vec<String>>());
}
// Calculate final message flags
let mut flag_value = MessageFlagsValue(0);
flag_value
.set(MessageFlags::SuppressNotifications, suppress_notifications)
.set(MessageFlags::MentionsEveryone, mentions_everyone)
.set(MessageFlags::MentionsOnline, mentions_online);
message.flags = Some(flag_value.0);
// Add attachments to message.
let mut attachments = vec![];
if data
@@ -487,7 +645,12 @@ impl Message {
tasks::ack::queue_message(
self.channel.to_string(),
AckEvent::ProcessMessage {
messages: vec![(None, self.clone(), mentions.clone(), true)],
messages: vec![(
None,
self.clone(),
mentions.clone(),
self.has_suppressed_notifications(),
)],
},
)
.await;
@@ -531,7 +694,9 @@ impl Message {
)
.await?;
if !self.has_suppressed_notifications() {
if !self.has_suppressed_notifications()
&& (self.mentions.is_some() || self.contains_mass_push_mention())
{
// send Push notifications
tasks::ack::queue_message(
self.channel.to_string(),
@@ -554,7 +719,7 @@ impl Message {
}
_ => vec![],
},
self.has_suppressed_notifications(),
false, // branch already dictates this
)],
},
)
@@ -591,13 +756,24 @@ impl Message {
/// Whether this message has suppressed notifications
pub fn has_suppressed_notifications(&self) -> bool {
if let Some(flags) = self.flags {
flags & MessageFlags::SuppressNotifications as i32
== MessageFlags::SuppressNotifications as i32
flags & MessageFlags::SuppressNotifications as u32
== MessageFlags::SuppressNotifications as u32
} else {
false
}
}
pub fn contains_mass_push_mention(&self) -> bool {
let ping = if let Some(flags) = self.flags {
let flags = MessageFlagsValue(flags);
flags.has(MessageFlags::MentionsEveryone)
} else {
false
};
ping || self.role_mentions.is_some()
}
/// Update message data
pub async fn update(
&mut self,

22
crates/core/database/src/models/messages/ops/mongodb.rs
View File

@@ -228,7 +228,6 @@ impl AbstractMessages for MongoDb {
"_id": id
},
query,
None,
)
.await
.map(|_| ())
@@ -247,7 +246,6 @@ impl AbstractMessages for MongoDb {
format!("reactions.{emoji}"): user
}
},
None,
)
.await
.map(|_| ())
@@ -266,7 +264,6 @@ impl AbstractMessages for MongoDb {
format!("reactions.{emoji}"): user
}
},
None,
)
.await
.map(|_| ())
@@ -285,7 +282,6 @@ impl AbstractMessages for MongoDb {
format!("reactions.{emoji}"): 1
}
},
None,
)
.await
.map(|_| ())
@@ -300,15 +296,12 @@ impl AbstractMessages for MongoDb {
/// Delete messages from a channel by their ids and corresponding channel id
async fn delete_messages(&self, channel: &str, ids: &[String]) -> Result<()> {
self.col::<Document>(COL)
.delete_many(
doc! {
"channel": channel,
"_id": {
"$in": ids
}
},
None,
)
.delete_many(doc! {
"channel": channel,
"_id": {
"$in": ids
}
})
.await
.map(|_| ())
.map_err(|_| create_database_error!("delete_many", COL))
@@ -362,7 +355,6 @@ impl MongoDb {
"deleted": true
}
},
None,
)
.await
.map_err(|_| create_database_error!("update_many", "attachments"))?;
@@ -370,7 +362,7 @@ impl MongoDb {
// And then delete said messages.
self.col::<Document>(COL)
.delete_many(projection, None)
.delete_many(projection)
.await
.map(|_| ())
.map_err(|_| create_database_error!("delete_many", COL))

15
crates/core/database/src/models/ratelimit_events/ops/mongodb.rs
View File

@@ -23,16 +23,13 @@ impl AbstractRatelimitEvents for MongoDb {
count: usize,
) -> Result<bool> {
self.col::<RatelimitEvent>(COL)
.count_documents(
doc! {
"_id": {
"$gte": Ulid::from_datetime(SystemTime::now() - period).to_string()
},
"target_id": target_id,
"event_type": event_type.to_string()
.count_documents(doc! {
"_id": {
"$gte": Ulid::from_datetime(SystemTime::now() - period).to_string()
},
None,
)
"target_id": target_id,
"event_type": event_type.to_string()
})
.await
.map(|c| c as usize >= count)
.map_err(|_| create_database_error!("count_documents", COL))

79
crates/core/database/src/models/server_members/ops.rs
View File

@@ -1,3 +1,4 @@
use ::mongodb::SessionCursor;
use revolt_result::Result;
use crate::{FieldsMember, Member, MemberCompositeKey, PartialMember};
@@ -5,6 +6,66 @@ use crate::{FieldsMember, Member, MemberCompositeKey, PartialMember};
mod mongodb;
mod reference;
#[derive(Debug)]
pub enum ChunkedServerMembersGenerator {
#[cfg(feature = "mongodb")]
MongoDb {
session: ::mongodb::ClientSession,
cursor: Option<SessionCursor<Member>>,
},
Reference {
offset: i32,
data: Option<Vec<Member>>,
},
}
impl ChunkedServerMembersGenerator {
#[cfg(feature = "mongodb")]
pub fn new_mongo(session: ::mongodb::ClientSession, cursor: SessionCursor<Member>) -> Self {
ChunkedServerMembersGenerator::MongoDb {
session,
cursor: Some(cursor),
}
}
pub fn new_reference(data: Vec<Member>) -> Self {
ChunkedServerMembersGenerator::Reference {
offset: 0,
data: Some(data),
}
}
pub async fn next(&mut self) -> Option<Member> {
match self {
#[cfg(feature = "mongodb")]
ChunkedServerMembersGenerator::MongoDb { session, cursor } => {
if let Some(cursor) = cursor {
let value = cursor.next(session).await;
value.map(|val| val.expect("Failed to fetch the next member"))
} else {
warn!("Attempted to access a (MongoDb) server member generator without first setting a cursor");
None
}
}
ChunkedServerMembersGenerator::Reference { offset, data } => {
if let Some(data) = data {
if data.len() as i32 >= *offset {
None
} else {
let resp = &data[*offset as usize];
*offset += 1;
Some(resp.clone())
}
} else {
warn!("Attempted to access a (Reference) server member generator without first providing data");
None
}
}
}
}
}
#[async_trait]
pub trait AbstractServerMembers: Sync + Send {
/// Insert a new server member into the database
@@ -16,6 +77,24 @@ pub trait AbstractServerMembers: Sync + Send {
/// Fetch all members in a server
async fn fetch_all_members<'a>(&self, server_id: &str) -> Result<Vec<Member>>;
/// Fetch all members in a server as an iterator
async fn fetch_all_members_chunked(
&self,
server_id: &str,
) -> Result<ChunkedServerMembersGenerator>;
async fn fetch_all_members_with_roles(
&self,
server_id: &str,
roles: &[String],
) -> Result<Vec<Member>>;
async fn fetch_all_members_with_roles_chunked(
&self,
server_id: &str,
roles: &[String],
) -> Result<ChunkedServerMembersGenerator>;
/// Fetch all memberships for a user
async fn fetch_all_memberships<'a>(&self, user_id: &str) -> Result<Vec<Member>>;

142
crates/core/database/src/models/server_members/ops/mongodb.rs
View File

@@ -1,10 +1,11 @@
use futures::StreamExt;
use mongodb::options::ReadConcern;
use revolt_result::Result;
use crate::{FieldsMember, Member, MemberCompositeKey, PartialMember};
use crate::{IntoDocumentPath, MongoDb};
use super::AbstractServerMembers;
use super::{AbstractServerMembers, ChunkedServerMembersGenerator};
static COL: &str = "server_members";
@@ -33,12 +34,9 @@ impl AbstractServerMembers for MongoDb {
async fn fetch_all_members<'a>(&self, server_id: &str) -> Result<Vec<Member>> {
Ok(self
.col::<Member>(COL)
.find(
doc! {
"_id.server": server_id
},
None,
)
.find(doc! {
"_id.server": server_id
})
.await
.map_err(|_| create_database_error!("find", COL))?
.filter_map(|s| async {
@@ -52,16 +50,101 @@ impl AbstractServerMembers for MongoDb {
.await)
}
/// Fetch all members in a server as a generator.
/// Uses config key pushd.mass_mention_chunk_size as the batch size.
async fn fetch_all_members_chunked(
&self,
server_id: &str,
) -> Result<ChunkedServerMembersGenerator> {
let config = revolt_config::config().await;
let mut session = self
.start_session()
.await
.map_err(|_| create_database_error!("start_session", COL))?;
session
.start_transaction()
.read_concern(ReadConcern::snapshot())
.await
.map_err(|_| create_database_error!("start_transaction", COL))?;
let cursor = self
.col::<Member>(COL)
.find(doc! {
"_id.server": server_id
})
.session(&mut session)
.batch_size(config.pushd.mass_mention_chunk_size as u32)
.await
.map_err(|_| create_database_error!("find", COL))?;
Ok(ChunkedServerMembersGenerator::new_mongo(session, cursor))
}
async fn fetch_all_members_with_roles(
&self,
server_id: &str,
roles: &[String],
) -> Result<Vec<Member>> {
Ok(self
.col::<Member>(COL)
.find(doc! {
"_id.server": server_id,
"roles": {"$in": roles}
})
.await
.map_err(|_| create_database_error!("find", COL))?
.filter_map(|s| async {
if cfg!(debug_assertions) {
Some(s.unwrap())
} else {
s.ok()
}
})
.collect()
.await)
}
async fn fetch_all_members_with_roles_chunked(
&self,
server_id: &str,
roles: &[String],
) -> Result<ChunkedServerMembersGenerator> {
let config = revolt_config::config().await;
let mut session = self
.start_session()
.await
.map_err(|_| create_database_error!("start_session", COL))?;
session
.start_transaction()
.read_concern(ReadConcern::snapshot())
.await
.map_err(|_| create_database_error!("start_transaction", COL))?;
let cursor = self
.col::<Member>(COL)
.find(doc! {
"_id.server": server_id,
"roles": {"$in": roles}
})
.session(&mut session)
.batch_size(config.pushd.mass_mention_chunk_size as u32)
.await
.map_err(|_| create_database_error!("find", COL))?;
return Ok(ChunkedServerMembersGenerator::new_mongo(session, cursor));
}
/// Fetch all memberships for a user
async fn fetch_all_memberships<'a>(&self, user_id: &str) -> Result<Vec<Member>> {
Ok(self
.col::<Member>(COL)
.find(
doc! {
"_id.user": user_id
},
None,
)
.find(doc! {
"_id.user": user_id
})
.await
.map_err(|_| create_database_error!("find", COL))?
.filter_map(|s| async {
@@ -79,15 +162,12 @@ impl AbstractServerMembers for MongoDb {
async fn fetch_members<'a>(&self, server_id: &str, ids: &'a [String]) -> Result<Vec<Member>> {
Ok(self
.col::<Member>(COL)
.find(
doc! {
"_id.server": server_id,
"_id.user": {
"$in": ids
}
},
None,
)
.find(doc! {
"_id.server": server_id,
"_id.user": {
"$in": ids
}
})
.await
.map_err(|_| create_database_error!("find", COL))?
.filter_map(|s| async {
@@ -104,12 +184,9 @@ impl AbstractServerMembers for MongoDb {
/// Fetch member count of a server
async fn fetch_member_count(&self, server_id: &str) -> Result<usize> {
self.col::<Member>(COL)
.count_documents(
doc! {
"_id.server": server_id
},
None,
)
.count_documents(doc! {
"_id.server": server_id
})
.await
.map(|c| c as usize)
.map_err(|_| create_database_error!("count_documents", COL))
@@ -118,12 +195,9 @@ impl AbstractServerMembers for MongoDb {
/// Fetch server count of a user
async fn fetch_server_count(&self, user_id: &str) -> Result<usize> {
self.col::<Member>(COL)
.count_documents(
doc! {
"_id.user": user_id
},
None,
)
.count_documents(doc! {
"_id.user": user_id
})
.await
.map(|c| c as usize)
.map_err(|_| create_database_error!("count_documents", COL))

66
crates/core/database/src/models/server_members/ops/reference.rs
View File

@@ -3,7 +3,7 @@ use revolt_result::Result;
use crate::ReferenceDb;
use crate::{FieldsMember, Member, MemberCompositeKey, PartialMember};
use super::AbstractServerMembers;
use super::{AbstractServerMembers, ChunkedServerMembersGenerator};
#[async_trait]
impl AbstractServerMembers for ReferenceDb {
@@ -40,6 +40,70 @@ impl AbstractServerMembers for ReferenceDb {
.collect())
}
/// Fetch all members in a server as an iterator
async fn fetch_all_members_chunked(
&self,
server_id: &str,
) -> Result<ChunkedServerMembersGenerator> {
let server_members = self.server_members.lock().await;
let members = server_members
.clone()
.into_values()
.filter(move |member| member.id.server == server_id)
.collect();
// this is inefficient as shit but its the reference db so its fine
Ok(ChunkedServerMembersGenerator::new_reference(members))
}
/// Fetch all members that have any of the roles given
async fn fetch_all_members_with_roles(
&self,
server_id: &str,
roles: &[String],
) -> Result<Vec<Member>> {
let server_members = self.server_members.lock().await;
Ok(server_members
.clone()
.into_values()
.filter(|member| {
member.id.server == server_id
&& !member
.roles
.iter()
.filter(|p| roles.contains(*p))
.collect::<Vec<&String>>()
.is_empty()
})
.collect())
}
async fn fetch_all_members_with_roles_chunked(
&self,
server_id: &str,
roles: &[String],
) -> Result<ChunkedServerMembersGenerator> {
let server_members = self.server_members.lock().await;
let resp = server_members
.clone()
.into_values()
.filter(|member| {
member.id.server == server_id
&& !member
.roles
.iter()
.filter(|p| roles.contains(*p))
.collect::<Vec<&String>>()
.is_empty()
})
.collect();
return Ok(ChunkedServerMembersGenerator::new_reference(resp));
}
/// Fetch all memberships for a user
async fn fetch_all_memberships<'a>(&self, user_id: &str) -> Result<Vec<Member>> {
let server_members = self.server_members.lock().await;

45
crates/core/database/src/models/servers/ops/mongodb.rs
View File

@@ -25,14 +25,11 @@ impl AbstractServers for MongoDb {
async fn fetch_servers<'a>(&self, ids: &'a [String]) -> Result<Vec<Server>> {
Ok(self
.col::<Server>(COL)
.find(
doc! {
"_id": {
"$in": ids
}
},
None,
)
.find(doc! {
"_id": {
"$in": ids
}
})
.await
.map_err(|_| create_database_error!("find", "servers"))?
.filter_map(|s| async {
@@ -84,7 +81,6 @@ impl AbstractServers for MongoDb {
.map_err(|_| create_database_error!("to_document", "role"))?
}
},
None,
)
.await
.map(|_| ())
@@ -125,7 +121,6 @@ impl AbstractServers for MongoDb {
"roles": &role_id
}
},
None,
)
.await
.map_err(|_| create_database_error!("update_many", "server_members"))?;
@@ -140,7 +135,6 @@ impl AbstractServers for MongoDb {
"role_permissions.".to_owned() + role_id: 1_i32
}
},
None,
)
.await
.map_err(|_| create_database_error!("update_one", "channels"))?;
@@ -155,7 +149,6 @@ impl AbstractServers for MongoDb {
"roles.".to_owned() + role_id: 1_i32
}
},
None,
)
.await
.map(|_| ())
@@ -188,12 +181,9 @@ impl MongoDb {
// Find all channels
let channels: Vec<String> = self
.col::<Document>("channels")
.find(
doc! {
"server": server_id
},
None,
)
.find(doc! {
"server": server_id
})
.await
.map_err(|_| create_database_error!("find", "channels"))?
.filter_map(|s| async {
@@ -225,19 +215,15 @@ impl MongoDb {
}
}
},
None,
)
.await
.map_err(|_| create_database_error!("update_many", "emojis"))?;
// Delete all channels.
self.col::<Document>("channels")
.delete_many(
doc! {
"server": &server_id
},
None,
)
.delete_many(doc! {
"server": &server_id
})
.await
.map_err(|_| create_database_error!("delete_many", "channels"))?;
@@ -248,12 +234,9 @@ impl MongoDb {
// Delete members and bans.
for with in &["server_members", "server_bans"] {
self.col::<Document>(with)
.delete_many(
doc! {
"_id.server": &server_id
},
None,
)
.delete_many(doc! {
"_id.server": &server_id
})
.await
.map_err(|_| create_database_error!("delete_many", with))?;
}

2
crates/core/database/src/models/user_settings/ops/mongodb.rs
View File

@@ -53,8 +53,8 @@ impl AbstractUserSettings for MongoDb {
doc! {
"$set": set
},
UpdateOptions::builder().upsert(true).build(),
)
.with_options(UpdateOptions::builder().upsert(true).build())
.await
.map(|_| ())
.map_err(|_| create_database_error!("update_one", "user_settings"))

142
crates/core/database/src/models/users/ops/mongodb.rs
View File

@@ -49,12 +49,9 @@ impl AbstractUsers for MongoDb {
/// Fetch a session from the database by token
async fn fetch_session_by_token(&self, token: &str) -> Result<Session> {
self.col::<Session>("sessions")
.find_one(
doc! {
"token": token
},
None,
)
.find_one(doc! {
"token": token
})
.await
.map_err(|_| create_database_error!("find_one", "sessions"))?
.ok_or_else(|| create_error!(InvalidSession))
@@ -64,14 +61,11 @@ impl AbstractUsers for MongoDb {
async fn fetch_users<'a>(&self, ids: &'a [String]) -> Result<Vec<User>> {
Ok(self
.col::<User>(COL)
.find(
doc! {
"_id": {
"$in": ids
}
},
None,
)
.find(doc! {
"_id": {
"$in": ids
}
})
.await
.map_err(|_| create_database_error!("find", COL))?
.filter_map(|s| async {
@@ -94,10 +88,10 @@ impl AbstractUsers for MongoDb {
Ok(self
.col::<UserDocument>(COL)
.find(
doc! {
"username": username
},
.find(doc! {
"username": username
})
.with_options(
FindOptions::builder()
.collation(
Collation::builder()
@@ -122,15 +116,13 @@ impl AbstractUsers for MongoDb {
async fn fetch_mutual_user_ids(&self, user_a: &str, user_b: &str) -> Result<Vec<String>> {
Ok(self
.col::<DocumentId>(COL)
.find(
doc! {
"$and": [
{ "relations": { "$elemMatch": { "_id": &user_a, "status": "Friend" } } },
{ "relations": { "$elemMatch": { "_id": &user_b, "status": "Friend" } } }
]
},
FindOptions::builder().projection(doc! { "_id": 1 }).build(),
)
.find(doc! {
"$and": [
{ "relations": { "$elemMatch": { "_id": &user_a, "status": "Friend" } } },
{ "relations": { "$elemMatch": { "_id": &user_b, "status": "Friend" } } }
]
})
.with_options(FindOptions::builder().projection(doc! { "_id": 1 }).build())
.await
.map_err(|_| create_database_error!("find", COL))?
.filter_map(|s| async { s.ok() })
@@ -143,17 +135,15 @@ impl AbstractUsers for MongoDb {
async fn fetch_mutual_channel_ids(&self, user_a: &str, user_b: &str) -> Result<Vec<String>> {
Ok(self
.col::<DocumentId>("channels")
.find(
doc! {
"channel_type": {
"$in": ["Group", "DirectMessage"]
},
"recipients": {
"$all": [ user_a, user_b ]
}
.find(doc! {
"channel_type": {
"$in": ["Group", "DirectMessage"]
},
FindOptions::builder().projection(doc! { "_id": 1 }).build(),
)
"recipients": {
"$all": [ user_a, user_b ]
}
})
.with_options(FindOptions::builder().projection(doc! { "_id": 1 }).build())
.await
.map_err(|_| create_database_error!("find", "channels"))?
.filter_map(|s| async { s.ok() })
@@ -166,49 +156,46 @@ impl AbstractUsers for MongoDb {
async fn fetch_mutual_server_ids(&self, user_a: &str, user_b: &str) -> Result<Vec<String>> {
Ok(self
.col::<DocumentId>("server_members")
.aggregate(
vec![
doc! {
"$match": {
"_id.user": user_a
}
},
doc! {
"$lookup": {
"from": "server_members",
"as": "members",
"let": {
"server": "$_id.server"
},
"pipeline": [
{
"$match": {
"$expr": {
"$and": [
{ "$eq": [ "$_id.user", user_b ] },
{ "$eq": [ "$_id.server", "$$server" ] }
]
}
.aggregate(vec![
doc! {
"$match": {
"_id.user": user_a
}
},
doc! {
"$lookup": {
"from": "server_members",
"as": "members",
"let": {
"server": "$_id.server"
},
"pipeline": [
{
"$match": {
"$expr": {
"$and": [
{ "$eq": [ "$_id.user", user_b ] },
{ "$eq": [ "$_id.server", "$$server" ] }
]
}
}
]
}
},
doc! {
"$match": {
"members": {
"$size": 1_i32
}
]
}
},
doc! {
"$match": {
"members": {
"$size": 1_i32
}
},
doc! {
"$project": {
"_id": "$_id.server"
}
},
],
None,
)
}
},
doc! {
"$project": {
"_id": "$_id.server"
}
},
])
.await
.map_err(|_| create_database_error!("aggregate", "server_members"))?
.filter_map(|s| async { s.ok() })
@@ -284,7 +271,6 @@ impl AbstractUsers for MongoDb {
}
}
}],
None,
)
.await
.map(|_| ())
@@ -305,7 +291,6 @@ impl AbstractUsers for MongoDb {
}
}
},
None,
)
.await
.map(|_| ())
@@ -329,7 +314,6 @@ impl AbstractUsers for MongoDb {
"subscription": 1
}
},
None,
)
.await
.map(|_| ())