From b3494b12fc0657a4ff9074a167acd58eb7c06137 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 11 May 2024 17:42:08 +0100 Subject: [PATCH 001/184] docs: create mdbook --- default.nix | 3 +++ doc/.gitignore | 1 + doc/book.toml | 6 ++++++ doc/src/SUMMARY.md | 7 +++++++ doc/src/hello.md | 5 +++++ 5 files changed, 22 insertions(+) create mode 100644 doc/.gitignore create mode 100644 doc/book.toml create mode 100644 doc/src/SUMMARY.md create mode 100644 doc/src/hello.md diff --git a/default.nix b/default.nix index 74e22096..0ff5a314 100644 --- a/default.nix +++ b/default.nix @@ -31,6 +31,9 @@ in pkgs.mkShell { pkgs.rustfmt pkgs.pkgconfig pkgs.openssl.dev + + # mdbook + pkgs.mdbook ]; RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}"; diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 00000000..7585238e --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1 @@ +book diff --git a/doc/book.toml b/doc/book.toml new file mode 100644 index 00000000..5ef489b5 --- /dev/null +++ b/doc/book.toml @@ -0,0 +1,6 @@ +[book] +authors = [] +language = "en" +multilingual = false +src = "src" +title = "Revolt Backend" diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md new file mode 100644 index 00000000..1b15bc68 --- /dev/null +++ b/doc/src/SUMMARY.md @@ -0,0 +1,7 @@ +# Summary + +- [Introduction](./hello.md) +- [Project Structure]() +- [Testing]() + - [Writing a new database test]() + - [Writing a new API test]() diff --git a/doc/src/hello.md b/doc/src/hello.md new file mode 100644 index 00000000..f8182679 --- /dev/null +++ b/doc/src/hello.md @@ -0,0 +1,5 @@ +# Revolt Backend + +Welcome to the developer documentation for the Revolt backend. + +This is very much incomplete and needs more work! From b69202c2224ec51b42a7c2bb0373edb294e232ac Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 11 May 2024 17:42:50 +0100 Subject: [PATCH 002/184] ci: add deployment action for documentation --- .github/workflows/book.yml | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/book.yml diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml new file mode 100644 index 00000000..168f27a1 --- /dev/null +++ b/.github/workflows/book.yml @@ -0,0 +1,40 @@ +name: Build documentation +on: + push: + branches: + - main + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + contents: write # To push a branch + pages: write # To push to a GitHub Pages site + id-token: write # To update the deployment status + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install latest mdbook + run: | + tag=$(curl 'https://api.github.com/repos/rust-lang/mdbook/releases/latest' | jq -r '.tag_name') + url="https://github.com/rust-lang/mdbook/releases/download/${tag}/mdbook-${tag}-x86_64-unknown-linux-gnu.tar.gz" + mkdir mdbook + curl -sSL $url | tar -xz --directory=./mdbook + echo `pwd`/mdbook >> $GITHUB_PATH + - name: Build Book + run: | + # This assumes your book is in the root of your repository. + # Just add a `cd` here if you need to change to another directory. + cd doc + mdbook build + - name: Setup Pages + uses: actions/configure-pages@v4 + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + # Upload entire repository + path: "book" + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 From 08127218cea41a550515fb1e90035c5194b3a2db Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 11 May 2024 17:50:52 +0100 Subject: [PATCH 003/184] ci: fix path to pages artifact ci: ignore changes to documentation for docker/rust builds --- .github/workflows/book.yml | 5 +---- .github/workflows/docker.yaml | 1 + .github/workflows/rust.yaml | 8 ++++++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 168f27a1..00f7ef71 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -24,8 +24,6 @@ jobs: echo `pwd`/mdbook >> $GITHUB_PATH - name: Build Book run: | - # This assumes your book is in the root of your repository. - # Just add a `cd` here if you need to change to another directory. cd doc mdbook build - name: Setup Pages @@ -33,8 +31,7 @@ jobs: - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: - # Upload entire repository - path: "book" + path: "doc/book" - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v4 diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 4f941cd2..3e919761 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -10,6 +10,7 @@ on: - ".github/**" - "!.github/workflows/docker.yml" - ".vscode/**" + - "doc/**" - ".gitignore" - "LICENSE" - "README" diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 34aa403e..6901b99e 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -2,6 +2,14 @@ name: Rust build, test, and generate specification on: push: + paths-ignore: + - ".github/**" + - "!.github/workflows/rust.yaml" + - ".vscode/**" + - "doc/**" + - ".gitignore" + - "LICENSE" + - "README" pull_request: branches: [main] From 471e0f55e476616f96cfba73211e1453147ac09d Mon Sep 17 00:00:00 2001 From: Zomatree Date: Wed, 15 May 2024 22:48:20 +0100 Subject: [PATCH 004/184] fix: Use correct enum tagging for invites --- .../admin_migrations/ops/mongodb/scripts.rs | 55 ++++++++++++++++++- .../src/models/channel_invites/model.rs | 1 + crates/core/models/src/v0/channel_invites.rs | 1 + 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs index ecc14672..a8eda5ac 100644 --- a/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs +++ b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs @@ -4,8 +4,7 @@ use crate::{ mongodb::{ bson::{doc, from_bson, from_document, to_document, Bson, DateTime, Document}, options::FindOptions, - }, - MongoDb, DISCRIMINATOR_SEARCH_SPACE, + }, Invite, MongoDb, DISCRIMINATOR_SEARCH_SPACE }; use futures::StreamExt; use rand::seq::SliceRandom; @@ -19,7 +18,7 @@ struct MigrationInfo { revision: i32, } -pub const LATEST_REVISION: i32 = 26; +pub const LATEST_REVISION: i32 = 27; pub async fn migrate_database(db: &MongoDb) { let migrations = db.col::("migrations"); @@ -983,6 +982,56 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 { .expect("Failed to create ratelimit_events index."); } + if revision <= 26 { + info!("Running migration [revision 26 / 15-05-2024]: fix invites being incorrectly serialized with wrong enum tagging."); + + auto_derived!( + pub enum OldInvite { + Server { + #[serde(rename = "_id")] + code: String, + server: String, + creator: String, + channel: String, + }, + Group { + #[serde(rename = "_id")] + code: String, + creator: String, + channel: String, + } + } + ); + + let mut invites = db.db() + .collection::("channel_invites") + .find(doc! { + "type": { "$exists": false } + }, None) + .await + .expect("failed to find invites"); + + while let Some(Ok(invite)) = invites.next().await { + let new_invite = match invite { + OldInvite::Server { code, server, creator, channel } => Invite::Server { code, server, creator, channel }, + OldInvite::Group { code, creator, channel } => Invite::Group { code, creator, channel } + }; + + db.db() + .collection("channel_invites") + .replace_one(doc! { "$or": [ + { + "Server._id": new_invite.code() + }, + { + "Group._id": new_invite.code() + } + ]}, new_invite, None) + .await + .unwrap(); + } + } + // Need to migrate fields on attachments, change `user_id`, `object_id`, etc to `parent`. // Reminder to update LATEST_REVISION when adding new migrations. diff --git a/crates/core/database/src/models/channel_invites/model.rs b/crates/core/database/src/models/channel_invites/model.rs index aaaff55e..e3af921a 100644 --- a/crates/core/database/src/models/channel_invites/model.rs +++ b/crates/core/database/src/models/channel_invites/model.rs @@ -10,6 +10,7 @@ static ALPHABET: [char; 54] = [ auto_derived!( /// Invite + #[serde(tag = "type")] pub enum Invite { /// Invite to a specific server channel Server { diff --git a/crates/core/models/src/v0/channel_invites.rs b/crates/core/models/src/v0/channel_invites.rs index f0e889d5..475bf6fc 100644 --- a/crates/core/models/src/v0/channel_invites.rs +++ b/crates/core/models/src/v0/channel_invites.rs @@ -2,6 +2,7 @@ use super::{Channel, File, Server, User}; auto_derived!( /// Invite + #[serde(tag = "type")] pub enum Invite { /// Invite to a specific server channel Server { From 878c1a83ad9ef6517d7664220890116e2bd81faa Mon Sep 17 00:00:00 2001 From: Zomatree Date: Wed, 15 May 2024 22:57:34 +0100 Subject: [PATCH 005/184] fix: continue fix --- .../models/admin_migrations/ops/mongodb/scripts.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs index a8eda5ac..53b20cfe 100644 --- a/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs +++ b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs @@ -6,6 +6,7 @@ use crate::{ options::FindOptions, }, Invite, MongoDb, DISCRIMINATOR_SEARCH_SPACE }; +use bson::oid::ObjectId; use futures::StreamExt; use rand::seq::SliceRandom; use revolt_permissions::DEFAULT_WEBHOOK_PERMISSIONS; @@ -1003,8 +1004,15 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 { } ); + #[derive(serde::Serialize, serde::Deserialize)] + struct Outer { + _id: ObjectId, + #[serde(flatten)] + invite: OldInvite + } + let mut invites = db.db() - .collection::("channel_invites") + .collection::("channel_invites") .find(doc! { "type": { "$exists": false } }, None) @@ -1012,7 +1020,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 { .expect("failed to find invites"); while let Some(Ok(invite)) = invites.next().await { - let new_invite = match invite { + let new_invite = match invite.invite { OldInvite::Server { code, server, creator, channel } => Invite::Server { code, server, creator, channel }, OldInvite::Group { code, creator, channel } => Invite::Group { code, creator, channel } }; From bfcdb13d6d53eed15c95cf96b8202234ce2aa3cf Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Thu, 16 May 2024 15:33:07 +0100 Subject: [PATCH 006/184] chore: remove cla workflow --- .github/workflows/cla.yml | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 .github/workflows/cla.yml diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml deleted file mode 100644 index 8a02c492..00000000 --- a/.github/workflows/cla.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: "CLA Assistant" -on: - issue_comment: - types: [created] - pull_request_target: - types: [opened,closed,synchronize] - -jobs: - CLAssistant: - runs-on: ubuntu-latest - steps: - - name: "CLA Assistant" - if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' - # Beta Release - uses: cla-assistant/github-action@v2.1.3-beta - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # the below token should have repo scope and must be manually added by you in the repository's secret - PERSONAL_ACCESS_TOKEN : ${{ secrets.PAT }} - with: - path-to-signatures: 'signatures/version1/cla.json' - path-to-document: 'https://github.com/revoltchat/cla/blob/master/CLA.md' # e.g. a CLA or a DCO document - # branch should not be protected - branch: 'master' - allowlist: insertish,bot* - - #below are the optional inputs - If the optional inputs are not given, then default values will be taken - remote-organization-name: revoltchat - remote-repository-name: cla - create-file-commit-message: 'cla(create): creating file for storing CLA Signatures' - signed-commit-message: 'cla(sign): $contributorName has signed the CLA in #$pullRequestNo' - #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign' - #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA' - #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.' - #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) - #use-dco-flag: true - If you are using DCO instead of CLA From 19e72babc9af62c943d0c79a90197b3f92c66fa5 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Thu, 16 May 2024 15:43:33 +0100 Subject: [PATCH 007/184] fix: must not use replace_one as it expects no change in _id --- .../admin_migrations/ops/mongodb/scripts.rs | 83 +++++++++++++------ 1 file changed, 56 insertions(+), 27 deletions(-) diff --git a/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs index 53b20cfe..c5336d40 100644 --- a/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs +++ b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs @@ -4,7 +4,8 @@ use crate::{ mongodb::{ bson::{doc, from_bson, from_document, to_document, Bson, DateTime, Document}, options::FindOptions, - }, Invite, MongoDb, DISCRIMINATOR_SEARCH_SPACE + }, + Invite, MongoDb, DISCRIMINATOR_SEARCH_SPACE, }; use bson::oid::ObjectId; use futures::StreamExt; @@ -1000,7 +1001,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 { code: String, creator: String, channel: String, - } + }, } ); @@ -1008,36 +1009,64 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 { struct Outer { _id: ObjectId, #[serde(flatten)] - invite: OldInvite + invite: OldInvite, } - let mut invites = db.db() + let invites = db + .db() .collection::("channel_invites") - .find(doc! { - "type": { "$exists": false } - }, None) + .find( + doc! { + "type": { "$exists": false } + }, + None, + ) + .await + .expect("failed to find invites") + .filter_map(|s| async { s.ok() }) + .collect::>() + .await + .into_iter() + .map(|invite| match invite.invite { + OldInvite::Server { + code, + server, + creator, + channel, + } => Invite::Server { + code, + server, + creator, + channel, + }, + OldInvite::Group { + code, + creator, + channel, + } => Invite::Group { + code, + creator, + channel, + }, + }) + .collect::>(); + + db.db() + .collection("channel_invites") + .insert_many(invites, None) + .await + .expect("failed to insert corrected invite"); + + db.db() + .collection::("channel_invites") + .delete_many( + doc! { + "type": { "$exists": false } + }, + None, + ) .await .expect("failed to find invites"); - - while let Some(Ok(invite)) = invites.next().await { - let new_invite = match invite.invite { - OldInvite::Server { code, server, creator, channel } => Invite::Server { code, server, creator, channel }, - OldInvite::Group { code, creator, channel } => Invite::Group { code, creator, channel } - }; - - db.db() - .collection("channel_invites") - .replace_one(doc! { "$or": [ - { - "Server._id": new_invite.code() - }, - { - "Group._id": new_invite.code() - } - ]}, new_invite, None) - .await - .unwrap(); - } } // Need to migrate fields on attachments, change `user_id`, `object_id`, etc to `parent`. From c59aad43c8dd1f59d31545eaf8b57c5fc9bb633b Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Thu, 16 May 2024 15:44:09 +0100 Subject: [PATCH 008/184] chore: bump version to 0.7.2 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 98fb375c..e47611f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3408,7 +3408,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.1" +version = "0.7.2" dependencies = [ "async-std", "async-tungstenite", @@ -3435,7 +3435,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.1" +version = "0.7.2" dependencies = [ "async-std", "cached", @@ -3451,7 +3451,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.1" +version = "0.7.2" dependencies = [ "async-lock", "async-recursion", @@ -3497,7 +3497,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.1" +version = "0.7.2" dependencies = [ "async-channel", "async-std", @@ -3543,7 +3543,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.1" +version = "0.7.2" dependencies = [ "indexmap", "iso8601-timestamp 0.2.11", @@ -3560,7 +3560,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.1" +version = "0.7.2" dependencies = [ "async-std", "async-trait", @@ -3575,7 +3575,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.1" +version = "0.7.2" dependencies = [ "async-std", "log", @@ -3586,7 +3586,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.1" +version = "0.7.2" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 485ec527..15ee3141 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.1" +version = "0.7.2" license = "AGPL-3.0-or-later" edition = "2021" @@ -38,7 +38,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.1", path = "../core/permissions" } +revolt-permissions = { version = "0.7.2", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index 69986f48..a29cd1a3 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.1" +version = "0.7.2" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 2e6bbcab..b42de3d4 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.1" +version = "0.7.2" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.1", path = "../config" } -revolt-result = { version = "0.7.1", path = "../result" } -revolt-models = { version = "0.7.1", path = "../models", features = [ +revolt-config = { version = "0.7.2", path = "../config" } +revolt-result = { version = "0.7.2", path = "../result" } +revolt-models = { version = "0.7.2", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.1", path = "../presence" } -revolt-permissions = { version = "0.7.1", path = "../permissions", features = [ +revolt-presence = { version = "0.7.2", path = "../presence" } +revolt-permissions = { version = "0.7.2", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 5b41b771..80f77e31 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.1" +version = "0.7.2" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.1", path = "../config" } -revolt-permissions = { version = "0.7.1", path = "../permissions" } +revolt-config = { version = "0.7.2", path = "../config" } +revolt-permissions = { version = "0.7.2", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index 7ee44d2c..c6f7f027 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.1" +version = "0.7.2" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.1", path = "../result" } +revolt-result = { version = "0.7.2", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index c09d5010..c92d49df 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.1" +version = "0.7.2" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index c7f1ec04..a53de068 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.1" +version = "0.7.2" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index cc4d0d39..53ef358e 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.1" +version = "0.7.2" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From a5d0cdf0dddaca45877a76ab335382c59374c9ae Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 4 Jun 2024 18:46:11 +0100 Subject: [PATCH 009/184] fix(core/database): skip insertion / deletion if no invites need to be corrected --- .../admin_migrations/ops/mongodb/scripts.rs | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs index c5336d40..acde3021 100644 --- a/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs +++ b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs @@ -1051,22 +1051,24 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 { }) .collect::>(); - db.db() - .collection("channel_invites") - .insert_many(invites, None) - .await - .expect("failed to insert corrected invite"); + if !invites.is_empty() { + db.db() + .collection("channel_invites") + .insert_many(invites, None) + .await + .expect("failed to insert corrected invite"); - db.db() - .collection::("channel_invites") - .delete_many( - doc! { - "type": { "$exists": false } - }, - None, - ) - .await - .expect("failed to find invites"); + db.db() + .collection::("channel_invites") + .delete_many( + doc! { + "type": { "$exists": false } + }, + None, + ) + .await + .expect("failed to find invites"); + } } // Need to migrate fields on attachments, change `user_id`, `object_id`, etc to `parent`. From 506968634ed6449fcf32110837a2088bc33369d9 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 9 Jun 2024 13:41:51 +0100 Subject: [PATCH 010/184] fix(core/database): set do not set online status if presence is Invisible closes #34 --- crates/core/database/src/util/bridge/v0.rs | 29 +++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/core/database/src/util/bridge/v0.rs b/crates/core/database/src/util/bridge/v0.rs index d8e184db..2ac317a6 100644 --- a/crates/core/database/src/util/bridge/v0.rs +++ b/crates/core/database/src/util/bridge/v0.rs @@ -995,6 +995,15 @@ impl crate::User { vec![] }, badges: self.badges.unwrap_or_default() as u32, + online: can_see_profile + && revolt_presence::is_online(&self.id).await + && !matches!( + self.status, + Some(crate::UserStatus { + presence: Some(crate::Presence::Invisible), + .. + }) + ), status: if can_see_profile { self.status.map(|status| status.into()) } else { @@ -1009,7 +1018,6 @@ impl crate::User { privileged: self.privileged, bot: self.bot.map(|bot| bot.into()), relationship, - online: can_see_profile && revolt_presence::is_online(&self.id).await, id: self.id, } } @@ -1064,6 +1072,15 @@ impl crate::User { vec![] }, badges: self.badges.unwrap_or_default() as u32, + online: can_see_profile + && is_online + && !matches!( + self.status, + Some(crate::UserStatus { + presence: Some(crate::Presence::Invisible), + .. + }) + ), status: if can_see_profile { self.status.map(|status| status.into()) } else { @@ -1078,7 +1095,6 @@ impl crate::User { privileged: self.privileged, bot: self.bot.map(|bot| bot.into()), relationship, - online: can_see_profile && is_online, id: self.id, } } @@ -1099,13 +1115,20 @@ impl crate::User { }) .unwrap_or_default(), badges: self.badges.unwrap_or_default() as u32, + online: revolt_presence::is_online(&self.id).await + && !matches!( + self.status, + Some(crate::UserStatus { + presence: Some(crate::Presence::Invisible), + .. + }) + ), status: self.status.map(|status| status.into()), profile: self.profile.map(|profile| profile.into()), flags: self.flags.unwrap_or_default() as u32, privileged: self.privileged, bot: self.bot.map(|bot| bot.into()), relationship: RelationshipStatus::User, - online: revolt_presence::is_online(&self.id).await, id: self.id, } } From da9a91e05f739a3330bff1d443ba4c55182126b8 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 9 Jun 2024 13:42:12 +0100 Subject: [PATCH 011/184] fix(bonfire): send InvalidSession error if token is not sent successfully closes #289 --- crates/bonfire/src/websocket.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/bonfire/src/websocket.rs b/crates/bonfire/src/websocket.rs index e5517356..ce1b6b75 100644 --- a/crates/bonfire/src/websocket.rs +++ b/crates/bonfire/src/websocket.rs @@ -19,6 +19,7 @@ use revolt_database::{ use revolt_presence::{create_session, delete_session}; use async_std::{net::TcpStream, sync::Mutex}; +use revolt_result::create_error; use crate::config::{ProtocolConfiguration, WebsocketHandshakeCallback}; use crate::events::state::{State, SubscriptionStateChange}; @@ -69,6 +70,7 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) // Try to authenticate the user. let Some(token) = config.get_session_token().as_ref() else { + write.send(config.encode(&create_error!(InvalidSession))).await.ok(); return; }; let user = match User::from_token(db, token, UserHint::Any).await { From 4c6e78e1a522a02148c816d4541bf2c82049f341 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 9 Jun 2024 13:47:22 +0100 Subject: [PATCH 012/184] chore: relicense some core crates to MIT chore: update/create LICENSE files closes #278 closes #277 --- LICENSE | 5 ++++- crates/core/config/Cargo.toml | 2 +- crates/core/config/LICENSE | 9 +++++++++ crates/core/models/Cargo.toml | 2 +- crates/core/models/LICENSE | 9 +++++++++ crates/core/permissions/Cargo.toml | 2 +- crates/core/permissions/LICENSE | 9 +++++++++ crates/core/result/Cargo.toml | 2 +- crates/core/result/LICENSE | 9 +++++++++ 9 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 crates/core/config/LICENSE create mode 100644 crates/core/models/LICENSE create mode 100644 crates/core/permissions/LICENSE create mode 100644 crates/core/result/LICENSE diff --git a/LICENSE b/LICENSE index eaeac67c..dad4e57d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,7 @@ - GNU AFFERO GENERAL PUBLIC LICENSE +With the exception of crates that specify their own LICENSE file, +the following license applies to the source code of this project. + +GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 Copyright (C) 2007 Free Software Foundation, Inc. diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index a29cd1a3..eb7a1480 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -2,7 +2,7 @@ name = "revolt-config" version = "0.7.2" edition = "2021" -license = "AGPL-3.0-or-later" +license = "MIT" authors = ["Paul Makles "] description = "Revolt Backend: Configuration" diff --git a/crates/core/config/LICENSE b/crates/core/config/LICENSE new file mode 100644 index 00000000..7c2815b9 --- /dev/null +++ b/crates/core/config/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2024 Pawel Makles + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 80f77e31..a9f76ed6 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -2,7 +2,7 @@ name = "revolt-models" version = "0.7.2" edition = "2021" -license = "AGPL-3.0-or-later" +license = "MIT" authors = ["Paul Makles "] description = "Revolt Backend: API Models" diff --git a/crates/core/models/LICENSE b/crates/core/models/LICENSE new file mode 100644 index 00000000..7c2815b9 --- /dev/null +++ b/crates/core/models/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2024 Pawel Makles + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index c6f7f027..ab630ee8 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -2,7 +2,7 @@ name = "revolt-permissions" version = "0.7.2" edition = "2021" -license = "AGPL-3.0-or-later" +license = "MIT" authors = ["Paul Makles "] description = "Revolt Backend: Permission Logic" diff --git a/crates/core/permissions/LICENSE b/crates/core/permissions/LICENSE new file mode 100644 index 00000000..7c2815b9 --- /dev/null +++ b/crates/core/permissions/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2024 Pawel Makles + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index a53de068..87a65866 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -2,7 +2,7 @@ name = "revolt-result" version = "0.7.2" edition = "2021" -license = "AGPL-3.0-or-later" +license = "MIT" authors = ["Paul Makles "] description = "Revolt Backend: Result and Error types" diff --git a/crates/core/result/LICENSE b/crates/core/result/LICENSE new file mode 100644 index 00000000..7c2815b9 --- /dev/null +++ b/crates/core/result/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2024 Pawel Makles + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From cb4435d7009b5b8d694f348a32895082a3804fc8 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 9 Jun 2024 13:49:38 +0100 Subject: [PATCH 013/184] chore: bump version to 0.7.3 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e47611f1..4b3288ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3408,7 +3408,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.2" +version = "0.7.3" dependencies = [ "async-std", "async-tungstenite", @@ -3435,7 +3435,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.2" +version = "0.7.3" dependencies = [ "async-std", "cached", @@ -3451,7 +3451,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.2" +version = "0.7.3" dependencies = [ "async-lock", "async-recursion", @@ -3497,7 +3497,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.2" +version = "0.7.3" dependencies = [ "async-channel", "async-std", @@ -3543,7 +3543,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.2" +version = "0.7.3" dependencies = [ "indexmap", "iso8601-timestamp 0.2.11", @@ -3560,7 +3560,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.2" +version = "0.7.3" dependencies = [ "async-std", "async-trait", @@ -3575,7 +3575,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.2" +version = "0.7.3" dependencies = [ "async-std", "log", @@ -3586,7 +3586,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.2" +version = "0.7.3" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 15ee3141..7d59d1e2 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.2" +version = "0.7.3" license = "AGPL-3.0-or-later" edition = "2021" @@ -38,7 +38,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.2", path = "../core/permissions" } +revolt-permissions = { version = "0.7.3", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index eb7a1480..d8feb2a9 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.2" +version = "0.7.3" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index b42de3d4..eb4fe9a3 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.2" +version = "0.7.3" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.2", path = "../config" } -revolt-result = { version = "0.7.2", path = "../result" } -revolt-models = { version = "0.7.2", path = "../models", features = [ +revolt-config = { version = "0.7.3", path = "../config" } +revolt-result = { version = "0.7.3", path = "../result" } +revolt-models = { version = "0.7.3", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.2", path = "../presence" } -revolt-permissions = { version = "0.7.2", path = "../permissions", features = [ +revolt-presence = { version = "0.7.3", path = "../presence" } +revolt-permissions = { version = "0.7.3", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index a9f76ed6..756a6f31 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.2" +version = "0.7.3" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.2", path = "../config" } -revolt-permissions = { version = "0.7.2", path = "../permissions" } +revolt-config = { version = "0.7.3", path = "../config" } +revolt-permissions = { version = "0.7.3", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index ab630ee8..a2938d0f 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.2" +version = "0.7.3" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.2", path = "../result" } +revolt-result = { version = "0.7.3", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index c92d49df..ab3f4026 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.2" +version = "0.7.3" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index 87a65866..e46bbc73 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.2" +version = "0.7.3" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 53ef358e..368458a5 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.2" +version = "0.7.3" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From 92e948aabce79f0eb303d0e669b46a1230573f57 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 9 Jun 2024 14:11:30 +0100 Subject: [PATCH 014/184] chore: update README [skip ci] --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index ee942fa7..71d8a4ed 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,21 @@ Tag and push a new release by running: just release ``` +## Testing + +First, start the required services: + +```sh +docker compose -f docker-compose.db.yml up -d +``` + +Now run tests for whichever database: + +```sh +TEST_DB=REFERENCE cargo nextest run +TEST_DB=MONGOBD cargo nextest run +``` + ## License The Revolt backend is generally licensed under the [GNU Affero General Public License v3.0](https://github.com/revoltchat/backend/blob/master/LICENSE). From 2cb20618da33e2a7fe4fb704c7c47ee69606c563 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 11 Jun 2024 12:38:32 +0100 Subject: [PATCH 015/184] feat(bonfire): add disconnection mechanism feat(bonfire): handle session deletion and logout events feat(core): trigger logout on bot token reset --- Cargo.lock | 1 + crates/bonfire/Cargo.toml | 1 + crates/bonfire/src/events/state.rs | 4 +- crates/bonfire/src/websocket.rs | 40 ++++++++++++++++--- crates/core/database/src/events/client.rs | 2 + crates/core/database/src/models/bots/model.rs | 6 ++- .../core/database/src/models/users/model.rs | 18 +++++---- crates/core/database/src/models/users/ops.rs | 5 ++- .../database/src/models/users/ops/mongodb.rs | 11 ++--- .../src/models/users/ops/reference.rs | 5 ++- 10 files changed, 68 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b3288ca..1be2fe33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3412,6 +3412,7 @@ version = "0.7.3" dependencies = [ "async-std", "async-tungstenite", + "authifier", "bincode", "fred", "futures", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 7d59d1e2..b9a751d5 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -34,6 +34,7 @@ async-std = { version = "1.8.0", features = [ ] } # core +authifier = { version = "1.0.8" } revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } diff --git a/crates/bonfire/src/events/state.rs b/crates/bonfire/src/events/state.rs index ae2e8904..878eb8bb 100644 --- a/crates/bonfire/src/events/state.rs +++ b/crates/bonfire/src/events/state.rs @@ -58,6 +58,7 @@ impl Default for Cache { pub struct State { pub cache: Cache, + pub session_id: String, pub private_topic: String, subscribed: HashSet, state: SubscriptionStateChange, @@ -65,7 +66,7 @@ pub struct State { impl State { /// Create state from User - pub fn from(user: User) -> State { + pub fn from(user: User, session_id: String) -> State { let mut subscribed = HashSet::new(); let private_topic = format!("{}!", user.id); subscribed.insert(private_topic.clone()); @@ -81,6 +82,7 @@ impl State { State { cache, subscribed, + session_id, private_topic, state: SubscriptionStateChange::Reset, } diff --git a/crates/bonfire/src/websocket.rs b/crates/bonfire/src/websocket.rs index ce1b6b75..0bfa666b 100644 --- a/crates/bonfire/src/websocket.rs +++ b/crates/bonfire/src/websocket.rs @@ -1,6 +1,7 @@ use std::net::SocketAddr; use async_tungstenite::WebSocketStream; +use authifier::AuthifierEvent; use fred::{ interfaces::{ClientLike, EventInterface, PubsubInterface}, types::RedisConfig, @@ -73,17 +74,19 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) write.send(config.encode(&create_error!(InvalidSession))).await.ok(); return; }; - let user = match User::from_token(db, token, UserHint::Any).await { + + let (user, session_id) = match User::from_token(db, token, UserHint::Any).await { Ok(user) => user, Err(err) => { write.send(config.encode(&err)).await.ok(); return; } }; + info!("User {addr:?} authenticated as @{}", user.username); // Create local state. - let mut state = State::from(user); + let mut state = State::from(user, session_id); let user_id = state.cache.user_id.clone(); // Notify socket we have authenticated. @@ -189,6 +192,7 @@ async fn listener( }) else { return; }; + let event = match *REDIS_PAYLOAD_TYPE { PayloadType::Json => message .value @@ -203,13 +207,34 @@ async fn listener( .as_bytes() .and_then(|b| bincode::deserialize::(b).ok()), }; + let Some(mut event) = event else { warn!("Failed to deserialise an event for {}!", message.channel); return; }; - let should_send = state.handle_incoming_event_v1(db, &mut event).await; - if !should_send { - continue; + + if let EventV1::Auth(auth) = &event { + if let AuthifierEvent::DeleteSession { session_id, .. } = auth { + if &state.session_id == session_id { + event = EventV1::Logout; + } + } else if let AuthifierEvent::DeleteAllSessions { + exclude_session_id, .. + } = auth + { + if let Some(excluded) = exclude_session_id { + if &state.session_id != excluded { + event = EventV1::Logout; + } + } else { + event = EventV1::Logout; + } + } + } else { + let should_send = state.handle_incoming_event_v1(db, &mut event).await; + if !should_send { + continue; + } } let result = write.lock().await.send(config.encode(&event)).await; @@ -218,6 +243,11 @@ async fn listener( if !matches!(e, Error::AlreadyClosed | Error::ConnectionClosed) { warn!("Error while sending an event to {addr:?}: {e:?}"); } + + return; + } + + if let EventV1::Logout = event { return; } } diff --git a/crates/core/database/src/events/client.rs b/crates/core/database/src/events/client.rs index 4d3b6c0d..68802af7 100644 --- a/crates/core/database/src/events/client.rs +++ b/crates/core/database/src/events/client.rs @@ -48,6 +48,8 @@ pub enum EventV1 { /// Successfully authenticated Authenticated, + /// Logged out + Logout, /// Basic data to cache Ready { users: Vec, diff --git a/crates/core/database/src/models/bots/model.rs b/crates/core/database/src/models/bots/model.rs index cf36bedc..6b213432 100644 --- a/crates/core/database/src/models/bots/model.rs +++ b/crates/core/database/src/models/bots/model.rs @@ -2,7 +2,7 @@ use revolt_config::config; use revolt_result::Result; use ulid::Ulid; -use crate::{BotInformation, Database, PartialUser, User}; +use crate::{events::client::EventV1, BotInformation, Database, PartialUser, User}; auto_derived_partial!( /// Bot @@ -142,6 +142,10 @@ impl Bot { db.update_bot(&self.id, &partial, remove).await?; + if partial.token.is_some() { + EventV1::Logout.private(self.id.clone()).await; + } + self.apply_options(partial); Ok(()) } diff --git a/crates/core/database/src/models/users/model.rs b/crates/core/database/src/models/users/model.rs index f09449d7..63749941 100644 --- a/crates/core/database/src/models/users/model.rs +++ b/crates/core/database/src/models/users/model.rs @@ -276,23 +276,27 @@ impl User { Ok(username) } - /// Find a user from a given token and hint + /// Find a user and session ID from a given token and hint #[async_recursion] - pub async fn from_token(db: &Database, token: &str, hint: UserHint) -> Result { + pub async fn from_token(db: &Database, token: &str, hint: UserHint) -> Result<(User, String)> { match hint { - UserHint::Bot => { + UserHint::Bot => Ok(( db.fetch_user( &db.fetch_bot_by_token(token) .await .map_err(|_| create_error!(InvalidSession))? .id, ) - .await + .await?, + String::new(), + )), + UserHint::User => { + let session = db.fetch_session_by_token(token).await?; + Ok((db.fetch_user(&session.user_id).await?, session.id)) } - UserHint::User => db.fetch_user_by_token(token).await, UserHint::Any => { - if let Ok(user) = User::from_token(db, token, UserHint::User).await { - Ok(user) + if let Ok(result) = User::from_token(db, token, UserHint::User).await { + Ok(result) } else { User::from_token(db, token, UserHint::Bot).await } diff --git a/crates/core/database/src/models/users/ops.rs b/crates/core/database/src/models/users/ops.rs index 58247d26..62c7fd7c 100644 --- a/crates/core/database/src/models/users/ops.rs +++ b/crates/core/database/src/models/users/ops.rs @@ -1,3 +1,4 @@ +use authifier::models::Session; use revolt_result::Result; use crate::{FieldsUser, PartialUser, RelationshipStatus, User}; @@ -16,8 +17,8 @@ pub trait AbstractUsers: Sync + Send { /// Fetch a user from the database by their username async fn fetch_user_by_username(&self, username: &str, discriminator: &str) -> Result; - /// Fetch a user from the database by their session token - async fn fetch_user_by_token(&self, token: &str) -> Result; + /// Fetch a session from the database by token + async fn fetch_session_by_token(&self, token: &str) -> Result; /// Fetch multiple users by their ids async fn fetch_users<'a>(&self, ids: &'a [String]) -> Result>; diff --git a/crates/core/database/src/models/users/ops/mongodb.rs b/crates/core/database/src/models/users/ops/mongodb.rs index 44729959..99b02094 100644 --- a/crates/core/database/src/models/users/ops/mongodb.rs +++ b/crates/core/database/src/models/users/ops/mongodb.rs @@ -46,10 +46,9 @@ impl AbstractUsers for MongoDb { .ok_or_else(|| create_error!(NotFound)) } - /// Fetch a user from the database by their session token - async fn fetch_user_by_token(&self, token: &str) -> Result { - let session = self - .col::("sessions") + /// Fetch a session from the database by token + async fn fetch_session_by_token(&self, token: &str) -> Result { + self.col::("sessions") .find_one( doc! { "token": token @@ -58,9 +57,7 @@ impl AbstractUsers for MongoDb { ) .await .map_err(|_| create_database_error!("find_one", "sessions"))? - .ok_or_else(|| create_error!(InvalidSession))?; - - self.fetch_user(&session.user_id).await + .ok_or_else(|| create_error!(InvalidSession)) } /// Fetch multiple users by their ids diff --git a/crates/core/database/src/models/users/ops/reference.rs b/crates/core/database/src/models/users/ops/reference.rs index 4c2d4e7d..47028276 100644 --- a/crates/core/database/src/models/users/ops/reference.rs +++ b/crates/core/database/src/models/users/ops/reference.rs @@ -1,3 +1,4 @@ +use authifier::models::Session; use revolt_result::Result; use crate::{FieldsUser, PartialUser, RelationshipStatus, User}; @@ -40,8 +41,8 @@ impl AbstractUsers for ReferenceDb { .ok_or_else(|| create_error!(NotFound)) } - /// Fetch a user from the database by their session token - async fn fetch_user_by_token(&self, _token: &str) -> Result { + /// Fetch a session from the database by token + async fn fetch_session_by_token(&self, _token: &str) -> Result { todo!() } From 96fb0eecca7143ad2fc59d9c552c035b539ecb7e Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 11 Jun 2024 12:51:52 +0100 Subject: [PATCH 016/184] fix: don't allow sending typing notification to unknown channel closes #177 --- crates/bonfire/src/events/state.rs | 42 ++++++++++++++++-------------- crates/bonfire/src/websocket.rs | 29 +++++++++++++++++---- 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/crates/bonfire/src/events/state.rs b/crates/bonfire/src/events/state.rs index 878eb8bb..988d5143 100644 --- a/crates/bonfire/src/events/state.rs +++ b/crates/bonfire/src/events/state.rs @@ -1,5 +1,9 @@ -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use async_std::sync::RwLock; use lru::LruCache; use revolt_database::{Channel, Member, Server, User}; @@ -60,8 +64,8 @@ pub struct State { pub session_id: String, pub private_topic: String, - subscribed: HashSet, - state: SubscriptionStateChange, + pub state: SubscriptionStateChange, + pub subscribed: Arc>>, } impl State { @@ -81,7 +85,7 @@ impl State { State { cache, - subscribed, + subscribed: Arc::new(RwLock::new(subscribed)), session_id, private_topic, state: SubscriptionStateChange::Reset, @@ -89,15 +93,16 @@ impl State { } /// Apply currently queued state - pub fn apply_state(&mut self) -> SubscriptionStateChange { + pub async fn apply_state(&mut self) -> SubscriptionStateChange { let state = std::mem::replace(&mut self.state, SubscriptionStateChange::None); + let mut subscribed = self.subscribed.write().await; if let SubscriptionStateChange::Change { add, remove } = &state { for id in add { - self.subscribed.insert(id.clone()); + subscribed.insert(id.clone()); } for id in remove { - self.subscribed.remove(id); + subscribed.remove(id); } } @@ -109,20 +114,16 @@ impl State { self.cache.users.get(&self.cache.user_id).unwrap().clone() } - /// Iterate through all subscriptions - pub fn iter_subscriptions(&self) -> std::collections::hash_set::Iter<'_, std::string::String> { - self.subscribed.iter() - } - /// Reset the current state - pub fn reset_state(&mut self) { + pub async fn reset_state(&mut self) { self.state = SubscriptionStateChange::Reset; - self.subscribed.clear(); + self.subscribed.write().await.clear(); } /// Add a new subscription - pub fn insert_subscription(&mut self, subscription: String) { - if self.subscribed.contains(&subscription) { + pub async fn insert_subscription(&mut self, subscription: String) { + let mut subscribed = self.subscribed.write().await; + if subscribed.contains(&subscription) { return; } @@ -139,12 +140,13 @@ impl State { SubscriptionStateChange::Reset => {} } - self.subscribed.insert(subscription); + subscribed.insert(subscription); } /// Remove existing subscription - pub fn remove_subscription(&mut self, subscription: &str) { - if !self.subscribed.contains(&subscription.to_string()) { + pub async fn remove_subscription(&mut self, subscription: &str) { + let mut subscribed = self.subscribed.write().await; + if !subscribed.contains(&subscription.to_string()) { return; } @@ -161,6 +163,6 @@ impl State { SubscriptionStateChange::Reset => panic!("Should not remove during a reset!"), } - self.subscribed.remove(subscription); + subscribed.remove(subscription); } } diff --git a/crates/bonfire/src/websocket.rs b/crates/bonfire/src/websocket.rs index 0bfa666b..6745cfe9 100644 --- a/crates/bonfire/src/websocket.rs +++ b/crates/bonfire/src/websocket.rs @@ -1,4 +1,4 @@ -use std::net::SocketAddr; +use std::{collections::HashSet, net::SocketAddr, sync::Arc}; use async_tungstenite::WebSocketStream; use authifier::AuthifierEvent; @@ -19,7 +19,10 @@ use revolt_database::{ }; use revolt_presence::{create_session, delete_session}; -use async_std::{net::TcpStream, sync::Mutex}; +use async_std::{ + net::TcpStream, + sync::{Mutex, RwLock}, +}; use revolt_result::create_error; use crate::config::{ProtocolConfiguration, WebsocketHandshakeCallback}; @@ -44,6 +47,7 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) else { return; }; + // Verify we've received a valid config, otherwise we should just drop the connection. let Ok(mut config) = receiver.await else { return; @@ -102,6 +106,7 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) let Ok(ready_payload) = state.generate_ready_payload(db).await else { return; }; + if write.send(config.encode(&ready_payload)).await.is_err() { return; } @@ -116,10 +121,12 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) { let write = Mutex::new(write); + let subscribed = state.subscribed.clone(); + // Create a PubSub connection to poll on. let listener = listener(db, &mut state, addr, &config, &write).fuse(); // Read from WebSocket stream. - let worker = worker(addr, user_id.clone(), &config, read, &write).fuse(); + let worker = worker(addr, subscribed, user_id.clone(), &config, read, &write).fuse(); // Pin both tasks. pin_mut!(listener, worker); @@ -157,10 +164,12 @@ async fn listener( let mut message_rx = subscriber.message_rx(); loop { // Check for state changes for subscriptions. - match state.apply_state() { + match state.apply_state().await { SubscriptionStateChange::Reset => { subscriber.unsubscribe_all().await.unwrap(); - for id in state.iter_subscriptions() { + + let subscribed = state.subscribed.read().await; + for id in subscribed.iter() { subscriber.subscribe(id).await.unwrap(); } @@ -255,6 +264,7 @@ async fn listener( async fn worker( addr: SocketAddr, + subscribed: Arc>>, user_id: String, config: &ProtocolConfiguration, mut read: WsReader, @@ -277,8 +287,13 @@ async fn worker( let Ok(payload) = config.decode(&msg) else { continue; }; + match payload { ClientMessage::BeginTyping { channel } => { + if !subscribed.read().await.contains(&channel) { + break; + } + EventV1::ChannelStartTyping { id: channel.clone(), user: user_id.clone(), @@ -287,6 +302,10 @@ async fn worker( .await; } ClientMessage::EndTyping { channel } => { + if !subscribed.read().await.contains(&channel) { + break; + } + EventV1::ChannelStopTyping { id: channel.clone(), user: user_id.clone(), From 4868205df4a998e522a767a5dc06f62a4ac2ed11 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 11 Jun 2024 12:52:45 +0100 Subject: [PATCH 017/184] chore: bump version to 0.7.4 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1be2fe33..a5182724 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3408,7 +3408,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.3" +version = "0.7.4" dependencies = [ "async-std", "async-tungstenite", @@ -3436,7 +3436,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.3" +version = "0.7.4" dependencies = [ "async-std", "cached", @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.3" +version = "0.7.4" dependencies = [ "async-lock", "async-recursion", @@ -3498,7 +3498,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.3" +version = "0.7.4" dependencies = [ "async-channel", "async-std", @@ -3544,7 +3544,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.3" +version = "0.7.4" dependencies = [ "indexmap", "iso8601-timestamp 0.2.11", @@ -3561,7 +3561,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.3" +version = "0.7.4" dependencies = [ "async-std", "async-trait", @@ -3576,7 +3576,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.3" +version = "0.7.4" dependencies = [ "async-std", "log", @@ -3587,7 +3587,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.3" +version = "0.7.4" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index b9a751d5..3fb96b37 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.3" +version = "0.7.4" license = "AGPL-3.0-or-later" edition = "2021" @@ -39,7 +39,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.3", path = "../core/permissions" } +revolt-permissions = { version = "0.7.4", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index d8feb2a9..277f4dba 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.3" +version = "0.7.4" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index eb4fe9a3..5004d8c8 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.3" +version = "0.7.4" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.3", path = "../config" } -revolt-result = { version = "0.7.3", path = "../result" } -revolt-models = { version = "0.7.3", path = "../models", features = [ +revolt-config = { version = "0.7.4", path = "../config" } +revolt-result = { version = "0.7.4", path = "../result" } +revolt-models = { version = "0.7.4", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.3", path = "../presence" } -revolt-permissions = { version = "0.7.3", path = "../permissions", features = [ +revolt-presence = { version = "0.7.4", path = "../presence" } +revolt-permissions = { version = "0.7.4", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 756a6f31..bbfb78e9 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.3" +version = "0.7.4" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.3", path = "../config" } -revolt-permissions = { version = "0.7.3", path = "../permissions" } +revolt-config = { version = "0.7.4", path = "../config" } +revolt-permissions = { version = "0.7.4", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index a2938d0f..28400a2f 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.3" +version = "0.7.4" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.3", path = "../result" } +revolt-result = { version = "0.7.4", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index ab3f4026..c49408ac 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.3" +version = "0.7.4" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index e46bbc73..1288712d 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.3" +version = "0.7.4" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 368458a5..bbd9e138 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.3" +version = "0.7.4" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From 8e7dd21bce185822de1991f7ccbab83a3075619d Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 11 Jun 2024 13:08:24 +0100 Subject: [PATCH 018/184] docs: add note about creating GitHub release [skip ci] --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 71d8a4ed..4be1b150 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,8 @@ Tag and push a new release by running: just release ``` +If you have bumped the crate versions, proceed to [GitHub releases](https://github.com/revoltchat/backend/releases/new) to create a changelog. + ## Testing First, start the required services: From 5c40f66010ee0cda81249ad138928a56a3eebaf4 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 11 Jun 2024 14:01:59 +0100 Subject: [PATCH 019/184] fix: wrong match keyword causing disconnects on typing event --- crates/bonfire/src/websocket.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/bonfire/src/websocket.rs b/crates/bonfire/src/websocket.rs index 6745cfe9..b3282a2e 100644 --- a/crates/bonfire/src/websocket.rs +++ b/crates/bonfire/src/websocket.rs @@ -291,7 +291,7 @@ async fn worker( match payload { ClientMessage::BeginTyping { channel } => { if !subscribed.read().await.contains(&channel) { - break; + continue; } EventV1::ChannelStartTyping { @@ -303,7 +303,7 @@ async fn worker( } ClientMessage::EndTyping { channel } => { if !subscribed.read().await.contains(&channel) { - break; + continue; } EventV1::ChannelStopTyping { From 8099310f89091092c0b25f2fd5e288a69238a232 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 11 Jun 2024 14:16:35 +0100 Subject: [PATCH 020/184] fix: wait on unwaited futures --- crates/bonfire/src/events/impl.rs | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/crates/bonfire/src/events/impl.rs b/crates/bonfire/src/events/impl.rs index 4880edf0..45d9a262 100644 --- a/crates/bonfire/src/events/impl.rs +++ b/crates/bonfire/src/events/impl.rs @@ -21,11 +21,11 @@ impl Cache { let server = self.servers.get(server); let mut query = DatabasePermissionQuery::new(db, self.users.get(&self.user_id).unwrap()) - .channel(&channel); + .channel(channel); // let mut perms = perms(self.users.get(&self.user_id).unwrap()).channel(channel); if let Some(member) = member { - query = query.member(&member); + query = query.member(member); } if let Some(server) = server { @@ -182,19 +182,19 @@ impl State { users.push(user.into_self().await); // Set subscription state internally. - self.reset_state(); - self.insert_subscription(self.private_topic.clone()); + self.reset_state().await; + self.insert_subscription(self.private_topic.clone()).await; for user in &users { - self.insert_subscription(user.id.clone()); + self.insert_subscription(user.id.clone()).await; } for server in &servers { - self.insert_subscription(server.id.clone()); + self.insert_subscription(server.id.clone()).await; } for channel in &channels { - self.insert_subscription(channel.id().to_string()); + self.insert_subscription(channel.id().to_string()).await; } Ok(EventV1::Ready { @@ -236,11 +236,11 @@ impl State { let mut bulk_events = vec![]; for id in added_channels { - self.insert_subscription(id); + self.insert_subscription(id).await; } for id in removed_channels { - self.remove_subscription(&id); + self.remove_subscription(&id).await; self.cache.channels.remove(&id); bulk_events.push(EventV1::ChannelDelete { id }); @@ -263,7 +263,7 @@ impl State { .channels .insert(channel.id().to_string(), channel.clone()); - self.insert_subscription(channel.id().to_string()); + self.insert_subscription(channel.id().to_string()).await; bulk_events.push(EventV1::ChannelCreate(channel.into())); } } @@ -336,7 +336,7 @@ impl State { match event { EventV1::ChannelCreate(channel) => { let id = channel.id().to_string(); - self.insert_subscription(id.clone()); + self.insert_subscription(id.clone()).await; self.cache.channels.insert(id, channel.clone().into()); } EventV1::ChannelUpdate { @@ -376,17 +376,17 @@ impl State { } } EventV1::ChannelDelete { id } => { - self.remove_subscription(id); + self.remove_subscription(id).await; self.cache.channels.remove(id); } EventV1::ChannelGroupJoin { user, .. } => { - self.insert_subscription(user.clone()); + self.insert_subscription(user.clone()).await; } EventV1::ChannelGroupLeave { id, user, .. } => { if user == &self.cache.user_id { - self.remove_subscription(id); + self.remove_subscription(id).await; } else if !self.cache.can_subscribe_to_user(user) { - self.remove_subscription(user); + self.remove_subscription(user).await; } } @@ -396,7 +396,7 @@ impl State { channels, emojis: _, } => { - self.insert_subscription(id.clone()); + self.insert_subscription(id.clone()).await; self.cache.servers.insert(id.clone(), server.clone().into()); let member = Member { id: MemberCompositeKey { @@ -435,11 +435,11 @@ impl State { } EventV1::ServerMemberLeave { id, user } => { if user == &self.cache.user_id { - self.remove_subscription(id); + self.remove_subscription(id).await; if let Some(server) = self.cache.servers.remove(id) { for channel in &server.channels { - self.remove_subscription(channel); + self.remove_subscription(channel).await; self.cache.channels.remove(channel); } } @@ -447,11 +447,11 @@ impl State { } } EventV1::ServerDelete { id } => { - self.remove_subscription(id); + self.remove_subscription(id).await; if let Some(server) = self.cache.servers.remove(id) { for channel in &server.channels { - self.remove_subscription(channel); + self.remove_subscription(channel).await; self.cache.channels.remove(channel); } } @@ -524,9 +524,9 @@ impl State { self.cache.users.insert(id.clone(), user.clone().into()); if self.cache.can_subscribe_to_user(id) { - self.insert_subscription(id.clone()); + self.insert_subscription(id.clone()).await; } else { - self.remove_subscription(id); + self.remove_subscription(id).await; } } @@ -540,11 +540,11 @@ impl State { // Sub / unsub accordingly. if let Some(id) = queue_add { - self.insert_subscription(id); + self.insert_subscription(id).await; } if let Some(id) = queue_remove { - self.remove_subscription(&id); + self.remove_subscription(&id).await; } true From d677716f935bf192111f8f81fd49e504c9295a62 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 11 Jun 2024 14:16:44 +0100 Subject: [PATCH 021/184] docs: update development guide --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 4be1b150..65069e67 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,29 @@ cp .env.example .env # (optionally) copy the default configuration file cp crates/core/config/Revolt.toml Revolt.toml # configure as necessary... +``` +You may want to copy the following configuration: + +```toml +# Revolt.toml +[database] +mongodb = "mongodb://localhost" +redis = "redis://localhost" + +[hosts] +app = "http://local.revolt.chat" +api = "http://local.revolt.chat:8000" +events = "ws://local.revolt.chat:9000" +autumn = "http://local.revolt.chat:3000" +january = "http://local.revolt.chat:7000" +voso_legacy = "" +voso_legacy_ws = "" +``` + +Then continue: + +```bash # start other necessary services docker compose up -d From 962c7d62c7eddbb4250671cbaea3bb973ea743d3 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 11 Jun 2024 18:46:07 +0100 Subject: [PATCH 022/184] fix: allow fetching discoverable servers as invites --- crates/core/database/src/util/reference.rs | 24 ++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/crates/core/database/src/util/reference.rs b/crates/core/database/src/util/reference.rs index 4c220654..9282d39d 100644 --- a/crates/core/database/src/util/reference.rs +++ b/crates/core/database/src/util/reference.rs @@ -1,3 +1,5 @@ +use std::str::FromStr; + use revolt_result::Result; #[cfg(feature = "rocket-impl")] use rocket::request::FromParam; @@ -44,9 +46,27 @@ impl Reference { db.fetch_channel(&self.id).await } - /// Fetch invite from Ref + /// Fetch invite from Ref or create invite to server if discoverable pub async fn as_invite(&self, db: &Database) -> Result { - db.fetch_invite(&self.id).await + if ulid::Ulid::from_str(&self.id).is_ok() { + let server = self.as_server(db).await?; + if !server.discoverable { + return Err(create_error!(NotFound)); + } + + Ok(Invite::Server { + code: self.id.to_string(), + server: server.id, + creator: server.owner, + channel: server + .channels + .into_iter() + .next() + .ok_or(create_error!(NotFound))?, + }) + } else { + db.fetch_invite(&self.id).await + } } /// Fetch message from Ref From b12e7285148ff4a2bf58c8ea8a4ce90231f6b861 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 15 Jun 2024 10:50:22 +0100 Subject: [PATCH 023/184] feat(core/result): implement std::error::Error for Error --- crates/core/result/src/lib.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/core/result/src/lib.rs b/crates/core/result/src/lib.rs index 5608509d..1f40be19 100644 --- a/crates/core/result/src/lib.rs +++ b/crates/core/result/src/lib.rs @@ -1,3 +1,5 @@ +use std::fmt::Display; + #[cfg(feature = "serde")] #[macro_use] extern crate serde; @@ -28,6 +30,14 @@ pub struct Error { pub location: String, } +impl Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?} occurred in {}", self.error_type, self.location) + } +} + +impl std::error::Error for Error {} + /// Possible error types #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] From a31a3e2e65d45d82449139edd9180df01547f7dc Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 15 Jun 2024 10:50:43 +0100 Subject: [PATCH 024/184] refactor: remove profile from v0 user, added unintentionally --- crates/core/database/src/util/bridge/v0.rs | 14 +------------- crates/core/models/src/v0/users.rs | 3 --- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/crates/core/database/src/util/bridge/v0.rs b/crates/core/database/src/util/bridge/v0.rs index 2ac317a6..5c1291f4 100644 --- a/crates/core/database/src/util/bridge/v0.rs +++ b/crates/core/database/src/util/bridge/v0.rs @@ -1009,11 +1009,6 @@ impl crate::User { } else { None }, - profile: if can_see_profile { - self.profile.map(|profile| profile.into()) - } else { - None - }, flags: self.flags.unwrap_or_default() as u32, privileged: self.privileged, bot: self.bot.map(|bot| bot.into()), @@ -1086,11 +1081,6 @@ impl crate::User { } else { None }, - profile: if can_see_profile { - self.profile.map(|profile| profile.into()) - } else { - None - }, flags: self.flags.unwrap_or_default() as u32, privileged: self.privileged, bot: self.bot.map(|bot| bot.into()), @@ -1124,7 +1114,6 @@ impl crate::User { }) ), status: self.status.map(|status| status.into()), - profile: self.profile.map(|profile| profile.into()), flags: self.flags.unwrap_or_default() as u32, privileged: self.privileged, bot: self.bot.map(|bot| bot.into()), @@ -1152,7 +1141,7 @@ impl From for crate::User { relations: None, badges: Some(value.badges as i32), status: value.status.map(Into::into), - profile: value.profile.map(Into::into), + profile: None, flags: Some(value.flags as i32), privileged: value.privileged, bot: value.bot.map(Into::into), @@ -1175,7 +1164,6 @@ impl From for PartialUser { }), badges: value.badges.map(|badges| badges as u32), status: value.status.map(|status| status.into()), - profile: value.profile.map(|profile| profile.into()), flags: value.flags.map(|flags| flags as u32), privileged: value.privileged, bot: value.bot.map(|bot| bot.into()), diff --git a/crates/core/models/src/v0/users.rs b/crates/core/models/src/v0/users.rs index 1966e35c..ac3a8d1d 100644 --- a/crates/core/models/src/v0/users.rs +++ b/crates/core/models/src/v0/users.rs @@ -50,9 +50,6 @@ auto_derived_partial!( /// User's current status #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] pub status: Option, - /// User's profile page - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub profile: Option, /// Enum of user flags #[cfg_attr( From fc9ceb754008f7c53909eb609e4211bf62002dc1 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 15 Jun 2024 10:51:00 +0100 Subject: [PATCH 025/184] refactor(bonfire): add additional error handling --- crates/bonfire/src/websocket.rs | 117 +++++++++++++++++++++++--------- 1 file changed, 86 insertions(+), 31 deletions(-) diff --git a/crates/bonfire/src/websocket.rs b/crates/bonfire/src/websocket.rs index b3282a2e..5727ad75 100644 --- a/crates/bonfire/src/websocket.rs +++ b/crates/bonfire/src/websocket.rs @@ -24,6 +24,7 @@ use async_std::{ sync::{Mutex, RwLock}, }; use revolt_result::create_error; +use sentry::Level; use crate::config::{ProtocolConfiguration, WebsocketHandshakeCallback}; use crate::events::state::{State, SubscriptionStateChange}; @@ -52,6 +53,7 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) let Ok(mut config) = receiver.await else { return; }; + info!( "User {addr:?} provided protocol configuration (version = {}, format = {:?})", config.get_protocol_version(), @@ -63,10 +65,8 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) // If the user has not provided authentication, request information. if config.get_session_token().is_none() { - while let Ok(message) = read.try_next().await { - if let Ok(ClientMessage::Authenticate { token }) = - config.decode(message.as_ref().unwrap()) - { + while let Ok(Some(message)) = read.try_next().await { + if let Ok(ClientMessage::Authenticate { token }) = config.decode(&message) { config.set_session_token(token); break; } @@ -75,7 +75,10 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) // Try to authenticate the user. let Some(token) = config.get_session_token().as_ref() else { - write.send(config.encode(&create_error!(InvalidSession))).await.ok(); + write + .send(config.encode(&create_error!(InvalidSession))) + .await + .ok(); return; }; @@ -94,20 +97,24 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) let user_id = state.cache.user_id.clone(); // Notify socket we have authenticated. - if write - .send(config.encode(&EventV1::Authenticated)) - .await - .is_err() - { + if let Err(err) = write.send(config.encode(&EventV1::Authenticated)).await { + error!("Failed to write: {err:?}"); + sentry::capture_error(&err); return; } // Download required data to local cache and send Ready payload. - let Ok(ready_payload) = state.generate_ready_payload(db).await else { - return; + let ready_payload = match state.generate_ready_payload(db).await { + Ok(ready_payload) => ready_payload, + Err(err) => { + sentry::capture_error(&err); + return; + } }; - if write.send(config.encode(&ready_payload)).await.is_err() { + if let Err(err) = write.send(config.encode(&ready_payload)).await { + error!("Failed to write: {err:?}"); + sentry::capture_error(&err); return; } @@ -154,23 +161,41 @@ async fn listener( write: &Mutex, ) { let redis_config = RedisConfig::from_url(&REDIS_URI).unwrap(); - let Ok(subscriber) = fred::types::Builder::from_config(redis_config).build_subscriber_client() - else { - return; - }; - if subscriber.init().await.is_err() { + + let subscriber = match fred::types::Builder::from_config(redis_config).build_subscriber_client() + { + Ok(subscriber) => subscriber, + Err(err) => { + error!("Failed to build a subscriber: {err:?}"); + sentry::capture_error(&err); + return; + } + }; + + if let Err(err) = subscriber.init().await { + error!("Failed to init subscriber: {err:?}"); + sentry::capture_error(&err); return; }; + let mut message_rx = subscriber.message_rx(); loop { // Check for state changes for subscriptions. match state.apply_state().await { SubscriptionStateChange::Reset => { - subscriber.unsubscribe_all().await.unwrap(); + if let Err(err) = subscriber.unsubscribe_all().await { + error!("Unsubscribe all failed: {err:?}"); + sentry::capture_error(&err); + return; + } let subscribed = state.subscribed.read().await; for id in subscribed.iter() { - subscriber.subscribe(id).await.unwrap(); + if let Err(err) = subscriber.subscribe(id).await { + error!("Subscribe failed: {err:?}"); + sentry::capture_error(&err); + return; + } } #[cfg(debug_assertions)] @@ -181,25 +206,35 @@ async fn listener( #[cfg(debug_assertions)] info!("{addr:?} unsubscribing from {id}"); - subscriber.unsubscribe(id).await.unwrap(); + if let Err(err) = subscriber.unsubscribe(id).await { + error!("Unsubscribe failed: {err:?}"); + sentry::capture_error(&err); + return; + } } for id in add { #[cfg(debug_assertions)] info!("{addr:?} subscribing to {id}"); - subscriber.subscribe(id).await.unwrap(); + if let Err(err) = subscriber.subscribe(id).await { + error!("Subscribe failed: {err:?}"); + sentry::capture_error(&err); + return; + } } } SubscriptionStateChange::None => {} } // Handle incoming events. - let Ok(message) = message_rx.recv().await.map_err(|e| { - warn!("Error while consuming pub/sub messages: {e:?}"); - sentry::capture_error(&e); - }) else { - return; + let message = match message_rx.recv().await { + Ok(message) => message, + Err(e) => { + error!("Error while consuming pub/sub messages: {e:?}"); + sentry::capture_error(&e); + return; + } }; let event = match *REDIS_PAYLOAD_TYPE { @@ -218,7 +253,17 @@ async fn listener( }; let Some(mut event) = event else { - warn!("Failed to deserialise an event for {}!", message.channel); + let err = format!( + "Failed to deserialise an event for {}! Introspection: `{:?}`", + message.channel, + message + .value + .as_string() + .map(|x| x.chars().take(32).collect::()) + ); + + error!("{}", err); + sentry::capture_message(&err, Level::Error); return; }; @@ -250,13 +295,16 @@ async fn listener( if let Err(e) = result { use async_tungstenite::tungstenite::Error; if !matches!(e, Error::AlreadyClosed | Error::ConnectionClosed) { - warn!("Error while sending an event to {addr:?}: {e:?}"); + let err = format!("Error while sending an event to {addr:?}: {e:?}"); + warn!("{}", err); + sentry::capture_message(&err, Level::Warning); } return; } if let EventV1::Logout = event { + info!("User {addr:?} received log out event!"); return; } } @@ -274,12 +322,19 @@ async fn worker( let result = read.try_next().await; let msg = match result { Ok(Some(msg)) => msg, - Ok(None) => return, + Ok(None) => { + warn!("Received a None message!"); + sentry::capture_message("Received a None message!", Level::Warning); + return; + } Err(e) => { use async_tungstenite::tungstenite::Error; if !matches!(e, Error::AlreadyClosed | Error::ConnectionClosed) { - warn!("Error while reading an event from {addr:?}: {e:?}"); + let err = format!("Error while reading an event from {addr:?}: {e:?}"); + warn!("{}", err); + sentry::capture_message(&err, Level::Warning); } + return; } }; From d179d1e6957e924f1b1ae6a633d05f72b5c8f1cc Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 15 Jun 2024 11:01:03 +0100 Subject: [PATCH 026/184] fix(delta): don't specify the other member in permission check supersedes / closes #323 --- crates/delta/src/routes/servers/member_edit.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/delta/src/routes/servers/member_edit.rs b/crates/delta/src/routes/servers/member_edit.rs index 8d478812..21c22730 100644 --- a/crates/delta/src/routes/servers/member_edit.rs +++ b/crates/delta/src/routes/servers/member_edit.rs @@ -30,12 +30,12 @@ pub async fn edit( }) })?; - // Fetch server, target member and current permissions + // Fetch server and target member let mut server = server.as_server(db).await?; let mut member = target.as_member(db, &server.id).await?; - let mut query = DatabasePermissionQuery::new(db, &user) - .server(&server) - .member(&member); + + // Fetch our currrent permissions + let mut query = DatabasePermissionQuery::new(db, &user).server(&server); let permissions = calculate_server_permissions(&mut query).await; // Check permissions in server From 4055363cff8a892b983ba7afde3245e88a2b41ce Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 15 Jun 2024 11:02:09 +0100 Subject: [PATCH 027/184] fix(core/database): fetch users mentioned in system messages --- .../database/src/models/messages/model.rs | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/core/database/src/models/messages/model.rs b/crates/core/database/src/models/messages/model.rs index 373cf222..b23ef488 100644 --- a/crates/core/database/src/models/messages/model.rs +++ b/crates/core/database/src/models/messages/model.rs @@ -504,7 +504,35 @@ impl Message { if let Some(true) = include_users { let user_ids = messages .iter() - .map(|m| m.author.clone()) + .flat_map(|m| { + let mut users = vec![m.author.clone()]; + if let Some(system) = &m.system { + match system { + v0::SystemMessage::ChannelDescriptionChanged { by } => { + users.push(by.clone()) + } + v0::SystemMessage::ChannelIconChanged { by } => users.push(by.clone()), + v0::SystemMessage::ChannelOwnershipChanged { from, to, .. } => { + users.push(from.clone()); + users.push(to.clone()) + } + v0::SystemMessage::ChannelRenamed { by, .. } => users.push(by.clone()), + v0::SystemMessage::UserAdded { by, id, .. } + | v0::SystemMessage::UserRemove { by, id, .. } => { + users.push(by.clone()); + users.push(id.clone()); + } + v0::SystemMessage::UserBanned { id, .. } + | v0::SystemMessage::UserKicked { id, .. } + | v0::SystemMessage::UserJoined { id, .. } + | v0::SystemMessage::UserLeft { id, .. } => { + users.push(id.clone()); + } + v0::SystemMessage::Text { .. } => {} + } + } + users + }) .collect::>() .into_iter() .collect::>(); From ba411fef430f355ecd41783496993ba36b477b44 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 15 Jun 2024 11:02:30 +0100 Subject: [PATCH 028/184] chore: bump version to 0.7.5 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5182724..ff5ccdb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3408,7 +3408,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.4" +version = "0.7.5" dependencies = [ "async-std", "async-tungstenite", @@ -3436,7 +3436,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.4" +version = "0.7.5" dependencies = [ "async-std", "cached", @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.4" +version = "0.7.5" dependencies = [ "async-lock", "async-recursion", @@ -3498,7 +3498,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.4" +version = "0.7.5" dependencies = [ "async-channel", "async-std", @@ -3544,7 +3544,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.4" +version = "0.7.5" dependencies = [ "indexmap", "iso8601-timestamp 0.2.11", @@ -3561,7 +3561,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.4" +version = "0.7.5" dependencies = [ "async-std", "async-trait", @@ -3576,7 +3576,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.4" +version = "0.7.5" dependencies = [ "async-std", "log", @@ -3587,7 +3587,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.4" +version = "0.7.5" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 3fb96b37..bc06f367 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.4" +version = "0.7.5" license = "AGPL-3.0-or-later" edition = "2021" @@ -39,7 +39,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.4", path = "../core/permissions" } +revolt-permissions = { version = "0.7.5", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index 277f4dba..902650a7 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.4" +version = "0.7.5" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 5004d8c8..ad6e5a29 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.4" +version = "0.7.5" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.4", path = "../config" } -revolt-result = { version = "0.7.4", path = "../result" } -revolt-models = { version = "0.7.4", path = "../models", features = [ +revolt-config = { version = "0.7.5", path = "../config" } +revolt-result = { version = "0.7.5", path = "../result" } +revolt-models = { version = "0.7.5", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.4", path = "../presence" } -revolt-permissions = { version = "0.7.4", path = "../permissions", features = [ +revolt-presence = { version = "0.7.5", path = "../presence" } +revolt-permissions = { version = "0.7.5", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index bbfb78e9..2ee0e91f 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.4" +version = "0.7.5" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.4", path = "../config" } -revolt-permissions = { version = "0.7.4", path = "../permissions" } +revolt-config = { version = "0.7.5", path = "../config" } +revolt-permissions = { version = "0.7.5", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index 28400a2f..b5c85c65 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.4" +version = "0.7.5" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.4", path = "../result" } +revolt-result = { version = "0.7.5", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index c49408ac..9117646b 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.4" +version = "0.7.5" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index 1288712d..60574bcd 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.4" +version = "0.7.5" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index bbd9e138..d6a95857 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.4" +version = "0.7.5" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From 36ecb48c7b054a1c2a7c449fed9143324c7d3d52 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 19 Jun 2024 18:24:51 +0100 Subject: [PATCH 029/184] feat(bonfire): don't fan out UserUpdate (server) by default --- Cargo.lock | 84 +++++++--- crates/bonfire/Cargo.toml | 2 + crates/bonfire/src/events/impl.rs | 10 ++ crates/bonfire/src/events/state.rs | 45 +++++- crates/bonfire/src/websocket.rs | 182 +++++++++++++--------- crates/core/database/src/events/client.rs | 7 +- crates/core/database/src/events/server.rs | 1 + 7 files changed, 237 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff5ccdb2..947600e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,11 +138,23 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" dependencies = [ - "concurrent-queue", - "event-listener", + "concurrent-queue 1.2.2", + "event-listener 2.5.2", "futures-core", ] +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue 2.5.0", + "event-listener-strategy", + "futures-core", + "pin-project-lite 0.2.13", +] + [[package]] name = "async-executor" version = "1.4.1" @@ -150,7 +162,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" dependencies = [ "async-task", - "concurrent-queue", + "concurrent-queue 1.2.2", "fastrand", "futures-lite", "once_cell", @@ -163,7 +175,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd8b508d585e01084059b60f06ade4cb7415cd2e4084b71dd1cb44e7d3fb9880" dependencies = [ - "async-channel", + "async-channel 1.6.1", "async-executor", "async-io", "async-lock", @@ -180,7 +192,7 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5e18f61464ae81cde0a23e713ae8fd299580c54d697a35820cfd0625b8b0e07" dependencies = [ - "concurrent-queue", + "concurrent-queue 1.2.2", "futures-lite", "libc", "log", @@ -199,7 +211,7 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" dependencies = [ - "event-listener", + "event-listener 2.5.2", ] [[package]] @@ -211,7 +223,7 @@ dependencies = [ "async-io", "blocking", "cfg-if", - "event-listener", + "event-listener 2.5.2", "futures-lite", "libc", "once_cell", @@ -237,7 +249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" dependencies = [ "async-attributes", - "async-channel", + "async-channel 1.6.1", "async-global-executor", "async-io", "async-lock", @@ -531,7 +543,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc" dependencies = [ - "async-channel", + "async-channel 1.6.1", "async-task", "atomic-waker", "fastrand", @@ -729,6 +741,15 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "config" version = "0.13.3" @@ -833,13 +854,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.8" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" -dependencies = [ - "cfg-if", - "lazy_static", -] +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crypto-bigint" @@ -1295,6 +1312,27 @@ version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" +[[package]] +name = "event-listener" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +dependencies = [ + "concurrent-queue 2.5.0", + "parking", + "pin-project-lite 0.2.13", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +dependencies = [ + "event-listener 5.3.1", + "pin-project-lite 0.2.13", +] + [[package]] name = "fake-simd" version = "0.1.2" @@ -1976,13 +2014,13 @@ version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "334e04b4d781f436dc315cb1e7515bd96826426345d498149e4bde36b67f8ee9" dependencies = [ - "async-channel", + "async-channel 1.6.1", "castaway", "crossbeam-utils", "curl", "curl-sys", "encoding_rs", - "event-listener", + "event-listener 2.5.2", "futures-lite", "http", "log", @@ -2266,6 +2304,12 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lru_time_cache" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" + [[package]] name = "maplit" version = "1.0.2" @@ -3410,6 +3454,7 @@ dependencies = [ name = "revolt-bonfire" version = "0.7.5" dependencies = [ + "async-channel 2.3.1", "async-std", "async-tungstenite", "authifier", @@ -3418,6 +3463,7 @@ dependencies = [ "futures", "log", "lru 0.7.6", + "lru_time_cache", "once_cell", "querystring", "redis-kiss", @@ -3500,7 +3546,7 @@ dependencies = [ name = "revolt-delta" version = "0.7.5" dependencies = [ - "async-channel", + "async-channel 1.6.1", "async-std", "authifier", "bitfield", @@ -4362,7 +4408,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d7400c0eff44aa2fcb5e31a5f24ba9716ed90138769e4977a2ba6014ae63eb5" dependencies = [ - "async-channel", + "async-channel 1.6.1", "futures-core", "futures-io", ] diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index bc06f367..41d6ec28 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -14,6 +14,8 @@ lru = "0.7.6" ulid = "0.5.0" once_cell = "1.9.0" redis-kiss = "0.1.4" +lru_time_cache = "0.11.11" +async-channel = "2.3.1" # parsing querystring = "1.1.0" diff --git a/crates/bonfire/src/events/impl.rs b/crates/bonfire/src/events/impl.rs index 45d9a262..46c854b8 100644 --- a/crates/bonfire/src/events/impl.rs +++ b/crates/bonfire/src/events/impl.rs @@ -92,6 +92,7 @@ impl State { /// Generate a Ready packet for the current user pub async fn generate_ready_payload(&mut self, db: &Database) -> Result { let user = self.clone_user(); + self.cache.is_bot = user.bot.is_some(); // Find all relationships to the user. let mut user_ids: HashSet = user @@ -191,6 +192,10 @@ impl State { for server in &servers { self.insert_subscription(server.id.clone()).await; + + if self.cache.is_bot { + self.insert_subscription(format!("{}u", server.id)).await; + } } for channel in &channels { @@ -397,6 +402,11 @@ impl State { emojis: _, } => { self.insert_subscription(id.clone()).await; + + if self.cache.is_bot { + self.insert_subscription(format!("{}u", id)).await; + } + self.cache.servers.insert(id.clone(), server.clone().into()); let member = Member { id: MemberCompositeKey { diff --git a/crates/bonfire/src/events/state.rs b/crates/bonfire/src/events/state.rs index 988d5143..eae39801 100644 --- a/crates/bonfire/src/events/state.rs +++ b/crates/bonfire/src/events/state.rs @@ -1,10 +1,12 @@ use std::{ collections::{HashMap, HashSet}, sync::Arc, + time::Duration, }; -use async_std::sync::RwLock; +use async_std::sync::{Mutex, RwLock}; use lru::LruCache; +use lru_time_cache::{LruCache as LruTimeCache, TimedEntry}; use revolt_database::{Channel, Member, Server, User}; /// Enumeration representing some change in subscriptions @@ -34,6 +36,7 @@ pub enum SubscriptionStateChange { #[derive(Debug)] pub struct Cache { pub user_id: String, + pub is_bot: bool, pub users: HashMap, pub channels: HashMap, @@ -47,6 +50,7 @@ impl Default for Cache { fn default() -> Self { Cache { user_id: Default::default(), + is_bot: false, users: Default::default(), channels: Default::default(), @@ -65,7 +69,9 @@ pub struct State { pub session_id: String, pub private_topic: String, pub state: SubscriptionStateChange, + pub subscribed: Arc>>, + pub active_servers: Arc>>, } impl State { @@ -86,6 +92,10 @@ impl State { State { cache, subscribed: Arc::new(RwLock::new(subscribed)), + active_servers: Arc::new(Mutex::new(LruTimeCache::with_expiry_duration_and_capacity( + Duration::from_secs(900), + 5, + ))), session_id, private_topic, state: SubscriptionStateChange::Reset, @@ -94,6 +104,39 @@ impl State { /// Apply currently queued state pub async fn apply_state(&mut self) -> SubscriptionStateChange { + // Check if we need to change subscriptions to member event topics + if !self.cache.is_bot { + enum Server { + Subscribe(String), + Unsubscribe(String), + } + + let active_server_changes: Vec = { + let mut active_servers = self.active_servers.lock().await; + active_servers + .notify_iter() + .map(|e| match e { + TimedEntry::Valid(k, _) => Server::Subscribe(format!("{}u", k)), + TimedEntry::Expired(k, _) => Server::Unsubscribe(format!("{}u", k)), + }) + .collect() + // It is bad practice to open more than one Mutex at once and could + // lead to a deadlock, so instead we choose to collect the changes. + }; + + for entry in active_server_changes { + match entry { + Server::Subscribe(k) => { + self.insert_subscription(k).await; + } + Server::Unsubscribe(k) => { + self.remove_subscription(&k).await; + } + } + } + } + + // Flush changes to subscriptions let state = std::mem::replace(&mut self.state, SubscriptionStateChange::None); let mut subscribed = self.subscribed.write().await; if let SubscriptionStateChange::Change { add, remove } = &state { diff --git a/crates/bonfire/src/websocket.rs b/crates/bonfire/src/websocket.rs index 5727ad75..5e9f0aac 100644 --- a/crates/bonfire/src/websocket.rs +++ b/crates/bonfire/src/websocket.rs @@ -129,11 +129,23 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) { let write = Mutex::new(write); let subscribed = state.subscribed.clone(); + let active_servers = state.active_servers.clone(); + let (signal_s, signal_r) = async_channel::unbounded(); // Create a PubSub connection to poll on. - let listener = listener(db, &mut state, addr, &config, &write).fuse(); + let listener = listener(db, &mut state, addr, &config, signal_r, &write).fuse(); // Read from WebSocket stream. - let worker = worker(addr, subscribed, user_id.clone(), &config, read, &write).fuse(); + let worker = worker( + addr, + subscribed, + active_servers, + user_id.clone(), + &config, + signal_s, + read, + &write, + ) + .fuse(); // Pin both tasks. pin_mut!(listener, worker); @@ -158,6 +170,7 @@ async fn listener( state: &mut State, addr: SocketAddr, config: &ProtocolConfiguration, + signal_r: async_channel::Receiver<()>, write: &Mutex, ) { let redis_config = RedisConfig::from_url(&REDIS_URI).unwrap(); @@ -227,94 +240,107 @@ async fn listener( SubscriptionStateChange::None => {} } - // Handle incoming events. - let message = match message_rx.recv().await { - Ok(message) => message, - Err(e) => { - error!("Error while consuming pub/sub messages: {e:?}"); - sentry::capture_error(&e); - return; - } - }; + let t1 = message_rx.recv().fuse(); + let t2 = signal_r.recv().fuse(); - let event = match *REDIS_PAYLOAD_TYPE { - PayloadType::Json => message - .value - .as_str() - .and_then(|s| serde_json::from_str::(s.as_ref()).ok()), - PayloadType::Msgpack => message - .value - .as_bytes() - .and_then(|b| rmp_serde::from_slice::(b).ok()), - PayloadType::Bincode => message - .value - .as_bytes() - .and_then(|b| bincode::deserialize::(b).ok()), - }; + pin_mut!(t1, t2); - let Some(mut event) = event else { - let err = format!( - "Failed to deserialise an event for {}! Introspection: `{:?}`", - message.channel, - message - .value - .as_string() - .map(|x| x.chars().take(32).collect::()) - ); + select! { + _ = t2 => {}, + message = t1 => { + // Handle incoming events. + let message = match message { + Ok(message) => message, + Err(e) => { + error!("Error while consuming pub/sub messages: {e:?}"); + sentry::capture_error(&e); + return; + } + }; - error!("{}", err); - sentry::capture_message(&err, Level::Error); - return; - }; + let event = match *REDIS_PAYLOAD_TYPE { + PayloadType::Json => message + .value + .as_str() + .and_then(|s| serde_json::from_str::(s.as_ref()).ok()), + PayloadType::Msgpack => message + .value + .as_bytes() + .and_then(|b| rmp_serde::from_slice::(b).ok()), + PayloadType::Bincode => message + .value + .as_bytes() + .and_then(|b| bincode::deserialize::(b).ok()), + }; - if let EventV1::Auth(auth) = &event { - if let AuthifierEvent::DeleteSession { session_id, .. } = auth { - if &state.session_id == session_id { - event = EventV1::Logout; - } - } else if let AuthifierEvent::DeleteAllSessions { - exclude_session_id, .. - } = auth - { - if let Some(excluded) = exclude_session_id { - if &state.session_id != excluded { - event = EventV1::Logout; + let Some(mut event) = event else { + let err = format!( + "Failed to deserialise an event for {}! Introspection: `{:?}`", + message.channel, + message + .value + .as_string() + .map(|x| x.chars().take(32).collect::()) + ); + + error!("{}", err); + sentry::capture_message(&err, Level::Error); + return; + }; + + if let EventV1::Auth(auth) = &event { + if let AuthifierEvent::DeleteSession { session_id, .. } = auth { + if &state.session_id == session_id { + event = EventV1::Logout; + } + } else if let AuthifierEvent::DeleteAllSessions { + exclude_session_id, .. + } = auth + { + if let Some(excluded) = exclude_session_id { + if &state.session_id != excluded { + event = EventV1::Logout; + } + } else { + event = EventV1::Logout; + } } } else { - event = EventV1::Logout; + let should_send = state.handle_incoming_event_v1(db, &mut event).await; + if !should_send { + continue; + } + } + + let result = write.lock().await.send(config.encode(&event)).await; + if let Err(e) = result { + use async_tungstenite::tungstenite::Error; + if !matches!(e, Error::AlreadyClosed | Error::ConnectionClosed) { + let err = format!("Error while sending an event to {addr:?}: {e:?}"); + warn!("{}", err); + sentry::capture_message(&err, Level::Warning); + } + + return; + } + + if let EventV1::Logout = event { + info!("User {addr:?} received log out event!"); + return; } } - } else { - let should_send = state.handle_incoming_event_v1(db, &mut event).await; - if !should_send { - continue; - } - } - - let result = write.lock().await.send(config.encode(&event)).await; - if let Err(e) = result { - use async_tungstenite::tungstenite::Error; - if !matches!(e, Error::AlreadyClosed | Error::ConnectionClosed) { - let err = format!("Error while sending an event to {addr:?}: {e:?}"); - warn!("{}", err); - sentry::capture_message(&err, Level::Warning); - } - - return; - } - - if let EventV1::Logout = event { - info!("User {addr:?} received log out event!"); - return; } } } +#[allow(clippy::too_many_arguments)] async fn worker( addr: SocketAddr, subscribed: Arc>>, + active_servers: Arc>>, user_id: String, config: &ProtocolConfiguration, + signal_s: async_channel::Sender<()>, mut read: WsReader, write: &Mutex, ) { @@ -368,6 +394,16 @@ async fn worker( .p(channel.clone()) .await; } + ClientMessage::Subscribe { server_id } => { + let mut servers = active_servers.lock().await; + let has_item = servers.contains_key(&server_id); + servers.insert(server_id, ()); + + if !has_item { + // Poke the listener to adjust subscriptions + signal_s.send(()).await.ok(); + } + } ClientMessage::Ping { data, responded } => { if responded.is_none() { write diff --git a/crates/core/database/src/events/client.rs b/crates/core/database/src/events/client.rs index 68802af7..dd20dd4f 100644 --- a/crates/core/database/src/events/client.rs +++ b/crates/core/database/src/events/client.rs @@ -249,7 +249,7 @@ impl EventV1 { // TODO: this should be captured by member list in the future and not immediately fanned out to users if let Ok(members) = db.fetch_all_memberships(&id).await { for member in members { - self.clone().p(member.id.server).await; + self.clone().server(member.id.server).await; } } } @@ -259,6 +259,11 @@ impl EventV1 { self.p(format!("{id}!")).await; } + /// Publish server member event + pub async fn server(self, id: String) { + self.p(format!("{id}u")).await; + } + /// Publish internal global event pub async fn global(self) { self.p("global".to_string()).await; diff --git a/crates/core/database/src/events/server.rs b/crates/core/database/src/events/server.rs index 883b92c3..cdaf6418 100644 --- a/crates/core/database/src/events/server.rs +++ b/crates/core/database/src/events/server.rs @@ -8,5 +8,6 @@ pub enum ClientMessage { Authenticate { token: String }, BeginTyping { channel: String }, EndTyping { channel: String }, + Subscribe { server_id: String }, Ping { data: Ping, responded: Option<()> }, } From cf0b4274c8f25202b00b8bd6a8d452de1d5cb585 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 19 Jun 2024 18:25:11 +0100 Subject: [PATCH 030/184] chore: bump version to 0.7.6 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 947600e4..e4bcf91b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.5" +version = "0.7.6" dependencies = [ "async-channel 2.3.1", "async-std", @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.5" +version = "0.7.6" dependencies = [ "async-std", "cached", @@ -3498,7 +3498,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.5" +version = "0.7.6" dependencies = [ "async-lock", "async-recursion", @@ -3544,7 +3544,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.5" +version = "0.7.6" dependencies = [ "async-channel 1.6.1", "async-std", @@ -3590,7 +3590,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.5" +version = "0.7.6" dependencies = [ "indexmap", "iso8601-timestamp 0.2.11", @@ -3607,7 +3607,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.5" +version = "0.7.6" dependencies = [ "async-std", "async-trait", @@ -3622,7 +3622,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.5" +version = "0.7.6" dependencies = [ "async-std", "log", @@ -3633,7 +3633,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.5" +version = "0.7.6" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 41d6ec28..9e36ada7 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.5" +version = "0.7.6" license = "AGPL-3.0-or-later" edition = "2021" @@ -41,7 +41,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.5", path = "../core/permissions" } +revolt-permissions = { version = "0.7.6", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index 902650a7..05544a99 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.5" +version = "0.7.6" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index ad6e5a29..4401fec8 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.5" +version = "0.7.6" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.5", path = "../config" } -revolt-result = { version = "0.7.5", path = "../result" } -revolt-models = { version = "0.7.5", path = "../models", features = [ +revolt-config = { version = "0.7.6", path = "../config" } +revolt-result = { version = "0.7.6", path = "../result" } +revolt-models = { version = "0.7.6", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.5", path = "../presence" } -revolt-permissions = { version = "0.7.5", path = "../permissions", features = [ +revolt-presence = { version = "0.7.6", path = "../presence" } +revolt-permissions = { version = "0.7.6", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 2ee0e91f..526d971a 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.5" +version = "0.7.6" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.5", path = "../config" } -revolt-permissions = { version = "0.7.5", path = "../permissions" } +revolt-config = { version = "0.7.6", path = "../config" } +revolt-permissions = { version = "0.7.6", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index b5c85c65..92c02465 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.5" +version = "0.7.6" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.5", path = "../result" } +revolt-result = { version = "0.7.6", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index 9117646b..7d37e94b 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.5" +version = "0.7.6" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index 60574bcd..e091d325 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.5" +version = "0.7.6" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index d6a95857..8c8a1b37 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.5" +version = "0.7.6" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From 8248a4a5b8b2831e0f21e72fe4687ce148a59b4d Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Thu, 20 Jun 2024 21:30:47 +0100 Subject: [PATCH 031/184] fix(bonfire): clean up Redis connection after disconnect --- crates/bonfire/src/websocket.rs | 265 +++++++++++++++++++++----------- 1 file changed, 175 insertions(+), 90 deletions(-) diff --git a/crates/bonfire/src/websocket.rs b/crates/bonfire/src/websocket.rs index 5e9f0aac..f5d29c74 100644 --- a/crates/bonfire/src/websocket.rs +++ b/crates/bonfire/src/websocket.rs @@ -8,7 +8,7 @@ use fred::{ }; use futures::{ channel::oneshot, - pin_mut, select, + join, pin_mut, select, stream::{SplitSink, SplitStream}, FutureExt, SinkExt, StreamExt, TryStreamExt, }; @@ -127,34 +127,44 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) } { + // Setup channels and mutexes let write = Mutex::new(write); let subscribed = state.subscribed.clone(); let active_servers = state.active_servers.clone(); - let (signal_s, signal_r) = async_channel::unbounded(); + let (topic_signal_s, topic_signal_r) = async_channel::unbounded(); + + // TODO: this needs to be rewritten + // Create channels through which the tasks can signal to each other they need to clean up + let (kill_signal_1_s, kill_signal_1_r) = async_channel::bounded(1); + let (kill_signal_2_s, kill_signal_2_r) = async_channel::bounded(1); // Create a PubSub connection to poll on. - let listener = listener(db, &mut state, addr, &config, signal_r, &write).fuse(); + let listener = listener_with_kill_signal( + db, + &mut state, + addr, + &config, + topic_signal_r, + kill_signal_1_r, + &write, + kill_signal_2_s, + ); + // Read from WebSocket stream. - let worker = worker( + let worker = worker_with_kill_signal( addr, subscribed, active_servers, user_id.clone(), &config, - signal_s, + topic_signal_s, + kill_signal_2_r, read, &write, - ) - .fuse(); - - // Pin both tasks. - pin_mut!(listener, worker); - - // Wait for either disconnect or for listener to die. - select!( - () = listener => {}, - () = worker => {} + kill_signal_1_s, ); + + join!(listener, worker); } // Clean up presence session. let last_session = delete_session(&user_id, session_id).await; @@ -165,16 +175,40 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr) } } +#[allow(clippy::too_many_arguments)] +async fn listener_with_kill_signal( + db: &'static Database, + state: &mut State, + addr: SocketAddr, + config: &ProtocolConfiguration, + topic_signal_r: async_channel::Receiver<()>, + kill_signal_r: async_channel::Receiver<()>, + write: &Mutex, + kill_signal_s: async_channel::Sender<()>, +) { + listener( + db, + state, + addr, + config, + topic_signal_r, + kill_signal_r, + write, + ) + .await; + kill_signal_s.send(()).await.ok(); +} + async fn listener( db: &'static Database, state: &mut State, addr: SocketAddr, config: &ProtocolConfiguration, - signal_r: async_channel::Receiver<()>, + topic_signal_r: async_channel::Receiver<()>, + kill_signal_r: async_channel::Receiver<()>, write: &Mutex, ) { let redis_config = RedisConfig::from_url(&REDIS_URI).unwrap(); - let subscriber = match fred::types::Builder::from_config(redis_config).build_subscriber_client() { Ok(subscriber) => subscriber, @@ -191,15 +225,17 @@ async fn listener( return; }; + // TODO: subscriber.on_error(func) + let mut message_rx = subscriber.message_rx(); - loop { + 'out: loop { // Check for state changes for subscriptions. match state.apply_state().await { SubscriptionStateChange::Reset => { if let Err(err) = subscriber.unsubscribe_all().await { error!("Unsubscribe all failed: {err:?}"); sentry::capture_error(&err); - return; + break 'out; } let subscribed = state.subscribed.read().await; @@ -207,7 +243,7 @@ async fn listener( if let Err(err) = subscriber.subscribe(id).await { error!("Subscribe failed: {err:?}"); sentry::capture_error(&err); - return; + break 'out; } } @@ -222,7 +258,7 @@ async fn listener( if let Err(err) = subscriber.unsubscribe(id).await { error!("Unsubscribe failed: {err:?}"); sentry::capture_error(&err); - return; + break 'out; } } @@ -233,7 +269,7 @@ async fn listener( if let Err(err) = subscriber.subscribe(id).await { error!("Subscribe failed: {err:?}"); sentry::capture_error(&err); - return; + break 'out; } } } @@ -241,11 +277,15 @@ async fn listener( } let t1 = message_rx.recv().fuse(); - let t2 = signal_r.recv().fuse(); + let t2 = topic_signal_r.recv().fuse(); + let t3 = kill_signal_r.recv().fuse(); - pin_mut!(t1, t2); + pin_mut!(t1, t2, t3); select! { + _ = t3 => { + break 'out; + }, _ = t2 => {}, message = t1 => { // Handle incoming events. @@ -254,7 +294,7 @@ async fn listener( Err(e) => { error!("Error while consuming pub/sub messages: {e:?}"); sentry::capture_error(&e); - return; + break 'out; } }; @@ -285,7 +325,7 @@ async fn listener( error!("{}", err); sentry::capture_message(&err, Level::Error); - return; + break 'out; }; if let EventV1::Auth(auth) = &event { @@ -321,16 +361,49 @@ async fn listener( sentry::capture_message(&err, Level::Warning); } - return; + break 'out; } if let EventV1::Logout = event { info!("User {addr:?} received log out event!"); - return; + break 'out; } } } } + + if let Err(err) = subscriber.quit().await { + error!("{}", err); + sentry::capture_error(&err); + } +} + +#[allow(clippy::too_many_arguments)] +async fn worker_with_kill_signal( + addr: SocketAddr, + subscribed: Arc>>, + active_servers: Arc>>, + user_id: String, + config: &ProtocolConfiguration, + topic_signal_s: async_channel::Sender<()>, + kill_signal_r: async_channel::Receiver<()>, + read: WsReader, + write: &Mutex, + kill_signal_s: async_channel::Sender<()>, +) { + worker( + addr, + subscribed, + active_servers, + user_id, + config, + topic_signal_s, + kill_signal_r, + read, + write, + ) + .await; + kill_signal_s.send(()).await.ok(); } #[allow(clippy::too_many_arguments)] @@ -340,81 +413,93 @@ async fn worker( active_servers: Arc>>, user_id: String, config: &ProtocolConfiguration, - signal_s: async_channel::Sender<()>, + topic_signal_s: async_channel::Sender<()>, + kill_signal_r: async_channel::Receiver<()>, mut read: WsReader, write: &Mutex, ) { loop { - let result = read.try_next().await; - let msg = match result { - Ok(Some(msg)) => msg, - Ok(None) => { - warn!("Received a None message!"); - sentry::capture_message("Received a None message!", Level::Warning); + let t1 = read.try_next().fuse(); + let t2 = kill_signal_r.recv().fuse(); + + pin_mut!(t1, t2); + + select! { + _ = t2 => { return; - } - Err(e) => { - use async_tungstenite::tungstenite::Error; - if !matches!(e, Error::AlreadyClosed | Error::ConnectionClosed) { - let err = format!("Error while reading an event from {addr:?}: {e:?}"); - warn!("{}", err); - sentry::capture_message(&err, Level::Warning); - } + }, + result = t1 => { + let msg = match result { + Ok(Some(msg)) => msg, + Ok(None) => { + warn!("Received a None message!"); + sentry::capture_message("Received a None message!", Level::Warning); + return; + } + Err(e) => { + use async_tungstenite::tungstenite::Error; + if !matches!(e, Error::AlreadyClosed | Error::ConnectionClosed) { + let err = format!("Error while reading an event from {addr:?}: {e:?}"); + warn!("{}", err); + sentry::capture_message(&err, Level::Warning); + } - return; - } - }; + return; + } + }; - let Ok(payload) = config.decode(&msg) else { - continue; - }; - - match payload { - ClientMessage::BeginTyping { channel } => { - if !subscribed.read().await.contains(&channel) { + let Ok(payload) = config.decode(&msg) else { continue; - } + }; - EventV1::ChannelStartTyping { - id: channel.clone(), - user: user_id.clone(), - } - .p(channel.clone()) - .await; - } - ClientMessage::EndTyping { channel } => { - if !subscribed.read().await.contains(&channel) { - continue; - } + match payload { + ClientMessage::BeginTyping { channel } => { + if !subscribed.read().await.contains(&channel) { + continue; + } - EventV1::ChannelStopTyping { - id: channel.clone(), - user: user_id.clone(), - } - .p(channel.clone()) - .await; - } - ClientMessage::Subscribe { server_id } => { - let mut servers = active_servers.lock().await; - let has_item = servers.contains_key(&server_id); - servers.insert(server_id, ()); + EventV1::ChannelStartTyping { + id: channel.clone(), + user: user_id.clone(), + } + .p(channel.clone()) + .await; + } + ClientMessage::EndTyping { channel } => { + if !subscribed.read().await.contains(&channel) { + continue; + } - if !has_item { - // Poke the listener to adjust subscriptions - signal_s.send(()).await.ok(); + EventV1::ChannelStopTyping { + id: channel.clone(), + user: user_id.clone(), + } + .p(channel.clone()) + .await; + } + ClientMessage::Subscribe { server_id } => { + let mut servers = active_servers.lock().await; + let has_item = servers.contains_key(&server_id); + servers.insert(server_id, ()); + + if !has_item { + // Poke the listener to adjust subscriptions + topic_signal_s.send(()).await.ok(); + } + } + ClientMessage::Ping { data, responded } => { + if responded.is_none() { + write + .lock() + .await + .send(config.encode(&EventV1::Pong { data })) + .await + .ok(); + } + } + _ => {} } } - ClientMessage::Ping { data, responded } => { - if responded.is_none() { - write - .lock() - .await - .send(config.encode(&EventV1::Pong { data })) - .await - .ok(); - } - } - _ => {} } } } From f16e72e3291688db319ceb8514655c61394c2380 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Thu, 20 Jun 2024 21:40:44 +0100 Subject: [PATCH 032/184] fix(bonfire): drop connections if Redis disconnects --- crates/bonfire/src/websocket.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/crates/bonfire/src/websocket.rs b/crates/bonfire/src/websocket.rs index f5d29c74..a44e6df2 100644 --- a/crates/bonfire/src/websocket.rs +++ b/crates/bonfire/src/websocket.rs @@ -22,6 +22,7 @@ use revolt_presence::{create_session, delete_session}; use async_std::{ net::TcpStream, sync::{Mutex, RwLock}, + task::spawn, }; use revolt_result::create_error; use sentry::Level; @@ -225,7 +226,16 @@ async fn listener( return; }; - // TODO: subscriber.on_error(func) + // Handle Redis connection dropping + let (clean_up_s, clean_up_r) = async_channel::bounded(1); + let clean_up_s = Arc::new(Mutex::new(clean_up_s)); + subscriber.on_error(move |_| { + let clean_up_s = clean_up_s.clone(); + spawn(async move { + clean_up_s.lock().await.send(()).await.ok(); + }); + Ok(()) + }); let mut message_rx = subscriber.message_rx(); 'out: loop { @@ -279,10 +289,14 @@ async fn listener( let t1 = message_rx.recv().fuse(); let t2 = topic_signal_r.recv().fuse(); let t3 = kill_signal_r.recv().fuse(); + let t4 = clean_up_r.recv().fuse(); - pin_mut!(t1, t2, t3); + pin_mut!(t1, t2, t3, t4); select! { + _ = t4 => { + break 'out; + }, _ = t3 => { break 'out; }, From 090d8cba715b6645468cf7d706618737bac9b982 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Thu, 20 Jun 2024 21:41:12 +0100 Subject: [PATCH 033/184] chore: bump version to 0.7.7 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4bcf91b..114b1117 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.6" +version = "0.7.7" dependencies = [ "async-channel 2.3.1", "async-std", @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.6" +version = "0.7.7" dependencies = [ "async-std", "cached", @@ -3498,7 +3498,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.6" +version = "0.7.7" dependencies = [ "async-lock", "async-recursion", @@ -3544,7 +3544,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.6" +version = "0.7.7" dependencies = [ "async-channel 1.6.1", "async-std", @@ -3590,7 +3590,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.6" +version = "0.7.7" dependencies = [ "indexmap", "iso8601-timestamp 0.2.11", @@ -3607,7 +3607,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.6" +version = "0.7.7" dependencies = [ "async-std", "async-trait", @@ -3622,7 +3622,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.6" +version = "0.7.7" dependencies = [ "async-std", "log", @@ -3633,7 +3633,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.6" +version = "0.7.7" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 9e36ada7..2d8821cc 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.6" +version = "0.7.7" license = "AGPL-3.0-or-later" edition = "2021" @@ -41,7 +41,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.6", path = "../core/permissions" } +revolt-permissions = { version = "0.7.7", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index 05544a99..bed48ee0 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.6" +version = "0.7.7" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 4401fec8..d3e03516 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.6" +version = "0.7.7" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.6", path = "../config" } -revolt-result = { version = "0.7.6", path = "../result" } -revolt-models = { version = "0.7.6", path = "../models", features = [ +revolt-config = { version = "0.7.7", path = "../config" } +revolt-result = { version = "0.7.7", path = "../result" } +revolt-models = { version = "0.7.7", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.6", path = "../presence" } -revolt-permissions = { version = "0.7.6", path = "../permissions", features = [ +revolt-presence = { version = "0.7.7", path = "../presence" } +revolt-permissions = { version = "0.7.7", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 526d971a..26eba01f 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.6" +version = "0.7.7" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.6", path = "../config" } -revolt-permissions = { version = "0.7.6", path = "../permissions" } +revolt-config = { version = "0.7.7", path = "../config" } +revolt-permissions = { version = "0.7.7", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index 92c02465..93a6d828 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.6" +version = "0.7.7" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.6", path = "../result" } +revolt-result = { version = "0.7.7", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index 7d37e94b..f64b4b7a 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.6" +version = "0.7.7" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index e091d325..f2cbedad 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.6" +version = "0.7.7" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 8c8a1b37..1a6eb03d 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.6" +version = "0.7.7" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From 93e05e9f18e3790a227cbf0bf46bc478735967c5 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Thu, 20 Jun 2024 22:12:00 +0100 Subject: [PATCH 034/184] fix(bonfire): ignore all Redis errors but `Canceled` --- crates/bonfire/src/websocket.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/bonfire/src/websocket.rs b/crates/bonfire/src/websocket.rs index a44e6df2..d84824b8 100644 --- a/crates/bonfire/src/websocket.rs +++ b/crates/bonfire/src/websocket.rs @@ -3,6 +3,7 @@ use std::{collections::HashSet, net::SocketAddr, sync::Arc}; use async_tungstenite::WebSocketStream; use authifier::AuthifierEvent; use fred::{ + error::{RedisError, RedisErrorKind}, interfaces::{ClientLike, EventInterface, PubsubInterface}, types::RedisConfig, }; @@ -229,11 +230,14 @@ async fn listener( // Handle Redis connection dropping let (clean_up_s, clean_up_r) = async_channel::bounded(1); let clean_up_s = Arc::new(Mutex::new(clean_up_s)); - subscriber.on_error(move |_| { - let clean_up_s = clean_up_s.clone(); - spawn(async move { - clean_up_s.lock().await.send(()).await.ok(); - }); + subscriber.on_error(move |err| { + if let RedisErrorKind::Canceled = err.kind() { + let clean_up_s = clean_up_s.clone(); + spawn(async move { + clean_up_s.lock().await.send(()).await.ok(); + }); + } + Ok(()) }); From eda36436a862bcab92f7ac2cf1c2dd88c41e52a4 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Fri, 21 Jun 2024 16:13:02 +0100 Subject: [PATCH 035/184] fix: specify configuration --- crates/delta/src/main.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/delta/src/main.rs b/crates/delta/src/main.rs index c5ec08a2..844bcc24 100644 --- a/crates/delta/src/main.rs +++ b/crates/delta/src/main.rs @@ -47,8 +47,7 @@ pub async fn web() -> Rocket { authifier::database::MongoDb(client.database("revolt")), ), }, - config: Default::default(), - // config: authifier_config().await, + config: authifier_config().await, event_channel: Some(sender), }; From e8e9613169ac334319beeaa9c42650f89cb751c6 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Fri, 21 Jun 2024 16:13:16 +0100 Subject: [PATCH 036/184] chore: bump version to 0.7.8 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 114b1117..46c770f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-channel 2.3.1", "async-std", @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-std", "cached", @@ -3498,7 +3498,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-lock", "async-recursion", @@ -3544,7 +3544,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-channel 1.6.1", "async-std", @@ -3590,7 +3590,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.7" +version = "0.7.8" dependencies = [ "indexmap", "iso8601-timestamp 0.2.11", @@ -3607,7 +3607,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-std", "async-trait", @@ -3622,7 +3622,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.7" +version = "0.7.8" dependencies = [ "async-std", "log", @@ -3633,7 +3633,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.7" +version = "0.7.8" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 2d8821cc..57f25d83 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.7" +version = "0.7.8" license = "AGPL-3.0-or-later" edition = "2021" @@ -41,7 +41,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.7", path = "../core/permissions" } +revolt-permissions = { version = "0.7.8", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index bed48ee0..0e7135b8 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.7" +version = "0.7.8" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index d3e03516..8047b767 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.7" +version = "0.7.8" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.7", path = "../config" } -revolt-result = { version = "0.7.7", path = "../result" } -revolt-models = { version = "0.7.7", path = "../models", features = [ +revolt-config = { version = "0.7.8", path = "../config" } +revolt-result = { version = "0.7.8", path = "../result" } +revolt-models = { version = "0.7.8", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.7", path = "../presence" } -revolt-permissions = { version = "0.7.7", path = "../permissions", features = [ +revolt-presence = { version = "0.7.8", path = "../presence" } +revolt-permissions = { version = "0.7.8", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 26eba01f..09b15842 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.7" +version = "0.7.8" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.7", path = "../config" } -revolt-permissions = { version = "0.7.7", path = "../permissions" } +revolt-config = { version = "0.7.8", path = "../config" } +revolt-permissions = { version = "0.7.8", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index 93a6d828..c54b0f16 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.7" +version = "0.7.8" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.7", path = "../result" } +revolt-result = { version = "0.7.8", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index f64b4b7a..09902a15 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.7" +version = "0.7.8" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index f2cbedad..7e15b34b 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.7" +version = "0.7.8" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 1a6eb03d..2417d0bf 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.7" +version = "0.7.8" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From 80666848cc91824d82aae78f0e7ca92e09d1e4e5 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Jun 2024 19:39:18 +0100 Subject: [PATCH 037/184] feat(core): add a limit to no. of outgoing pending friend requests --- crates/core/config/Revolt.toml | 2 ++ crates/core/config/src/lib.rs | 2 ++ crates/core/database/src/models/users/model.rs | 18 ++++++++++++++++++ crates/core/result/src/lib.rs | 3 +++ crates/core/result/src/rocket.rs | 1 + 5 files changed, 26 insertions(+) diff --git a/crates/core/config/Revolt.toml b/crates/core/config/Revolt.toml index 85823e63..2d8953fd 100644 --- a/crates/core/config/Revolt.toml +++ b/crates/core/config/Revolt.toml @@ -47,6 +47,8 @@ webhooks_enabled = false [features.limits] [features.limits.default] +outgoing_friend_requests = 10 + group_size = 100 bots = 5 message_length = 2000 diff --git a/crates/core/config/src/lib.rs b/crates/core/config/src/lib.rs index 60fe88fb..13d2ffa5 100644 --- a/crates/core/config/src/lib.rs +++ b/crates/core/config/src/lib.rs @@ -106,6 +106,8 @@ pub struct Api { #[derive(Deserialize, Debug, Clone)] pub struct FeaturesLimits { + pub outgoing_friend_requests: usize, + pub group_size: usize, pub bots: usize, pub message_length: usize, diff --git a/crates/core/database/src/models/users/model.rs b/crates/core/database/src/models/users/model.rs index 63749941..30581791 100644 --- a/crates/core/database/src/models/users/model.rs +++ b/crates/core/database/src/models/users/model.rs @@ -470,6 +470,24 @@ impl User { /// Add another user as a friend pub async fn add_friend(&mut self, db: &Database, target: &mut User) -> Result<()> { + let count = self + .relations + .as_ref() + .map(|relations| { + relations + .iter() + .filter(|r| matches!(r.status, RelationshipStatus::Outgoing)) + .count() + }) + .unwrap_or_default(); + + let config = config().await; + if count >= config.features.limits.default.outgoing_friend_requests { + return Err(create_error!(TooManyPendingFriendRequests { + max: config.features.limits.default.outgoing_friend_requests + })); + } + match self.relationship_with(&target.id) { RelationshipStatus::User => Err(create_error!(NoEffect)), RelationshipStatus::Friend => Err(create_error!(AlreadyFriends)), diff --git a/crates/core/result/src/lib.rs b/crates/core/result/src/lib.rs index 1f40be19..b444d66b 100644 --- a/crates/core/result/src/lib.rs +++ b/crates/core/result/src/lib.rs @@ -60,6 +60,9 @@ pub enum ErrorType { Blocked, BlockedByOther, NotFriends, + TooManyPendingFriendRequests { + max: usize, + }, // ? Channel related errors UnknownChannel, diff --git a/crates/core/result/src/rocket.rs b/crates/core/result/src/rocket.rs index d5ca3493..272cfc42 100644 --- a/crates/core/result/src/rocket.rs +++ b/crates/core/result/src/rocket.rs @@ -25,6 +25,7 @@ impl<'r> Responder<'r, 'static> for Error { ErrorType::Blocked => Status::Conflict, ErrorType::BlockedByOther => Status::Forbidden, ErrorType::NotFriends => Status::Forbidden, + ErrorType::TooManyPendingFriendRequests { .. } => Status::BadRequest, ErrorType::UnknownChannel => Status::NotFound, ErrorType::UnknownMessage => Status::NotFound, From 1ec8f46c1d6225947e075848ef58e426b7ef56f9 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Jun 2024 19:39:32 +0100 Subject: [PATCH 038/184] chore: bump version to 0.7.9 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46c770f5..f84412df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.8" +version = "0.7.9" dependencies = [ "async-channel 2.3.1", "async-std", @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.8" +version = "0.7.9" dependencies = [ "async-std", "cached", @@ -3498,7 +3498,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.8" +version = "0.7.9" dependencies = [ "async-lock", "async-recursion", @@ -3544,7 +3544,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.8" +version = "0.7.9" dependencies = [ "async-channel 1.6.1", "async-std", @@ -3590,7 +3590,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.8" +version = "0.7.9" dependencies = [ "indexmap", "iso8601-timestamp 0.2.11", @@ -3607,7 +3607,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.8" +version = "0.7.9" dependencies = [ "async-std", "async-trait", @@ -3622,7 +3622,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.8" +version = "0.7.9" dependencies = [ "async-std", "log", @@ -3633,7 +3633,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.8" +version = "0.7.9" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 57f25d83..7181bbe3 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.8" +version = "0.7.9" license = "AGPL-3.0-or-later" edition = "2021" @@ -41,7 +41,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.8", path = "../core/permissions" } +revolt-permissions = { version = "0.7.9", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index 0e7135b8..242c78f8 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.8" +version = "0.7.9" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 8047b767..26e649d6 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.8" +version = "0.7.9" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.8", path = "../config" } -revolt-result = { version = "0.7.8", path = "../result" } -revolt-models = { version = "0.7.8", path = "../models", features = [ +revolt-config = { version = "0.7.9", path = "../config" } +revolt-result = { version = "0.7.9", path = "../result" } +revolt-models = { version = "0.7.9", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.8", path = "../presence" } -revolt-permissions = { version = "0.7.8", path = "../permissions", features = [ +revolt-presence = { version = "0.7.9", path = "../presence" } +revolt-permissions = { version = "0.7.9", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 09b15842..1cb0ea83 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.8" +version = "0.7.9" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.8", path = "../config" } -revolt-permissions = { version = "0.7.8", path = "../permissions" } +revolt-config = { version = "0.7.9", path = "../config" } +revolt-permissions = { version = "0.7.9", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index c54b0f16..48fae107 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.8" +version = "0.7.9" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.8", path = "../result" } +revolt-result = { version = "0.7.9", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index 09902a15..ff99110d 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.8" +version = "0.7.9" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index 7e15b34b..3d34ea5c 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.8" +version = "0.7.9" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 2417d0bf..5e94af96 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.8" +version = "0.7.9" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From de5add09d05f95721672678c4c5c2f0512700f32 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 25 Jun 2024 18:07:16 +0100 Subject: [PATCH 039/184] fix(core/database): only run outgoing friend checks if we are creating new request closes #327 --- .../core/database/src/models/users/model.rs | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/crates/core/database/src/models/users/model.rs b/crates/core/database/src/models/users/model.rs index 30581791..1a3b9820 100644 --- a/crates/core/database/src/models/users/model.rs +++ b/crates/core/database/src/models/users/model.rs @@ -470,24 +470,6 @@ impl User { /// Add another user as a friend pub async fn add_friend(&mut self, db: &Database, target: &mut User) -> Result<()> { - let count = self - .relations - .as_ref() - .map(|relations| { - relations - .iter() - .filter(|r| matches!(r.status, RelationshipStatus::Outgoing)) - .count() - }) - .unwrap_or_default(); - - let config = config().await; - if count >= config.features.limits.default.outgoing_friend_requests { - return Err(create_error!(TooManyPendingFriendRequests { - max: config.features.limits.default.outgoing_friend_requests - })); - } - match self.relationship_with(&target.id) { RelationshipStatus::User => Err(create_error!(NoEffect)), RelationshipStatus::Friend => Err(create_error!(AlreadyFriends)), @@ -495,6 +477,7 @@ impl User { RelationshipStatus::Blocked => Err(create_error!(Blocked)), RelationshipStatus::BlockedOther => Err(create_error!(BlockedByOther)), RelationshipStatus::Incoming => { + // Accept incoming friend request self.apply_relationship( db, target, @@ -504,6 +487,27 @@ impl User { .await } RelationshipStatus::None => { + // Get this user's current count of outgoing friend requests + let count = self + .relations + .as_ref() + .map(|relations| { + relations + .iter() + .filter(|r| matches!(r.status, RelationshipStatus::Outgoing)) + .count() + }) + .unwrap_or_default(); + + // If we're over the limit, don't allow creating more requests + let config = config().await; + if count >= config.features.limits.default.outgoing_friend_requests { + return Err(create_error!(TooManyPendingFriendRequests { + max: config.features.limits.default.outgoing_friend_requests + })); + } + + // Send the friend request self.apply_relationship( db, target, From e5eea267cf5c251929f07a7b2c3cb0727678af88 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 25 Jun 2024 18:10:23 +0100 Subject: [PATCH 040/184] refactor(core/config): move limits to 'global' key --- crates/core/config/Revolt.toml | 36 ++++++++++++++----- crates/core/config/src/lib.rs | 25 ++++++++----- .../database/src/models/channels/model.rs | 12 +++---- .../database/src/models/messages/model.rs | 8 ++--- .../src/routes/customisation/emoji_create.rs | 4 +-- .../delta/src/routes/servers/roles_create.rs | 4 +-- 6 files changed, 59 insertions(+), 30 deletions(-) diff --git a/crates/core/config/Revolt.toml b/crates/core/config/Revolt.toml index 2d8953fd..9f5c0c26 100644 --- a/crates/core/config/Revolt.toml +++ b/crates/core/config/Revolt.toml @@ -46,21 +46,41 @@ webhooks_enabled = false [features.limits] -[features.limits.default] -outgoing_friend_requests = 10 - +[features.limits.global] group_size = 100 -bots = 5 -message_length = 2000 -message_embeds = 5 message_replies = 5 -message_attachments = 5 message_reactions = 20 -servers = 100 server_emoji = 100 server_roles = 200 server_channels = 200 +new_user_days = 3 + +[features.limits.new_user] +outgoing_friend_requests = 5 + +bots = 2 +message_length = 2000 +message_embeds = 5 +message_attachments = 5 +servers = 100 + +attachment_size = 20000000 +avatar_size = 4000000 +background_size = 6000000 +icon_size = 2500000 +banner_size = 6000000 +emoji_size = 500000 + +[features.limits.default] +outgoing_friend_requests = 10 + +bots = 5 +message_length = 2000 +message_embeds = 5 +message_attachments = 5 +servers = 100 + attachment_size = 20000000 avatar_size = 4000000 background_size = 6000000 diff --git a/crates/core/config/src/lib.rs b/crates/core/config/src/lib.rs index 13d2ffa5..02d238dd 100644 --- a/crates/core/config/src/lib.rs +++ b/crates/core/config/src/lib.rs @@ -105,21 +105,27 @@ pub struct Api { } #[derive(Deserialize, Debug, Clone)] -pub struct FeaturesLimits { - pub outgoing_friend_requests: usize, - +pub struct GlobalLimits { pub group_size: usize, - pub bots: usize, - pub message_length: usize, pub message_replies: usize, - pub message_attachments: usize, - pub message_embeds: usize, pub message_reactions: usize, - pub servers: usize, pub server_emoji: usize, pub server_roles: usize, pub server_channels: usize, + pub new_user_days: usize, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct FeaturesLimits { + pub outgoing_friend_requests: usize, + + pub bots: usize, + pub message_length: usize, + pub message_attachments: usize, + pub message_embeds: usize, + pub servers: usize, + pub attachment_size: usize, pub avatar_size: usize, pub background_size: usize, @@ -130,6 +136,9 @@ pub struct FeaturesLimits { #[derive(Deserialize, Debug, Clone)] pub struct FeaturesLimitsCollection { + pub global: GlobalLimits, + + pub new_user: FeaturesLimits, pub default: FeaturesLimits, #[serde(flatten)] diff --git a/crates/core/database/src/models/channels/model.rs b/crates/core/database/src/models/channels/model.rs index eafd88a6..4a879c8c 100644 --- a/crates/core/database/src/models/channels/model.rs +++ b/crates/core/database/src/models/channels/model.rs @@ -201,9 +201,9 @@ impl Channel { update_server: bool, ) -> Result { let config = config().await; - if server.channels.len() > config.features.limits.default.server_channels { + if server.channels.len() > config.features.limits.global.server_channels { return Err(create_error!(TooManyChannels { - max: config.features.limits.default.server_channels, + max: config.features.limits.global.server_channels, })); }; @@ -263,9 +263,9 @@ impl Channel { data.users.insert(owner_id.to_string()); let config = config().await; - if data.users.len() > config.features.limits.default.group_size { + if data.users.len() > config.features.limits.global.group_size { return Err(create_error!(GroupTooLarge { - max: config.features.limits.default.group_size, + max: config.features.limits.global.group_size, })); } @@ -346,9 +346,9 @@ impl Channel { } let config = config().await; - if recipients.len() >= config.features.limits.default.group_size { + if recipients.len() >= config.features.limits.global.group_size { return Err(create_error!(GroupTooLarge { - max: config.features.limits.default.group_size + max: config.features.limits.global.group_size })); } diff --git a/crates/core/database/src/models/messages/model.rs b/crates/core/database/src/models/messages/model.rs index b23ef488..afafff91 100644 --- a/crates/core/database/src/models/messages/model.rs +++ b/crates/core/database/src/models/messages/model.rs @@ -288,9 +288,9 @@ impl Message { // Verify replies are valid. let mut replies = HashSet::new(); if let Some(entries) = data.replies { - if entries.len() > config.features.limits.default.message_replies { + if entries.len() > config.features.limits.global.message_replies { return Err(create_error!(TooManyReplies { - max: config.features.limits.default.message_replies, + max: config.features.limits.global.message_replies, })); } @@ -616,7 +616,7 @@ impl Message { pub async fn add_reaction(&self, db: &Database, user: &User, emoji: &str) -> Result<()> { // Check how many reactions are already on the message let config = config().await; - if self.reactions.len() >= config.features.limits.default.message_reactions + if self.reactions.len() >= config.features.limits.global.message_reactions && !self.reactions.contains_key(emoji) { return Err(create_error!(InvalidOperation)); @@ -781,7 +781,7 @@ impl Interactions { if let Some(reactions) = &self.reactions { permissions.throw_if_lacking_channel_permission(ChannelPermission::React)?; - if reactions.len() > config.features.limits.default.message_reactions { + if reactions.len() > config.features.limits.global.message_reactions { return Err(create_error!(InvalidOperation)); } diff --git a/crates/delta/src/routes/customisation/emoji_create.rs b/crates/delta/src/routes/customisation/emoji_create.rs index a8a44412..f4bd12df 100644 --- a/crates/delta/src/routes/customisation/emoji_create.rs +++ b/crates/delta/src/routes/customisation/emoji_create.rs @@ -45,9 +45,9 @@ pub async fn create_emoji( // Check that we haven't hit the emoji limit let emojis = db.fetch_emoji_by_parent_id(&server.id).await?; - if emojis.len() >= config.features.limits.default.server_emoji { + if emojis.len() >= config.features.limits.global.server_emoji { return Err(create_error!(TooManyEmoji { - max: config.features.limits.default.server_emoji, + max: config.features.limits.global.server_emoji, })); } } diff --git a/crates/delta/src/routes/servers/roles_create.rs b/crates/delta/src/routes/servers/roles_create.rs index dae18baf..c7c67f09 100644 --- a/crates/delta/src/routes/servers/roles_create.rs +++ b/crates/delta/src/routes/servers/roles_create.rs @@ -34,9 +34,9 @@ pub async fn create( .throw_if_lacking_channel_permission(ChannelPermission::ManageRole)?; let config = config().await; - if server.roles.len() >= config.features.limits.default.server_roles { + if server.roles.len() >= config.features.limits.global.server_roles { return Err(create_error!(TooManyRoles { - max: config.features.limits.default.server_roles, + max: config.features.limits.global.server_roles, })); }; From 6ec8007e4ee9a98e553c1f5c2721bb2d3ee3df11 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 25 Jun 2024 19:20:34 +0100 Subject: [PATCH 041/184] feat(core): separate limits for new user accounts --- crates/core/config/Revolt.toml | 3 +- crates/core/config/src/lib.rs | 2 +- crates/core/database/src/models/bots/model.rs | 3 +- .../database/src/models/messages/model.rs | 14 +++++---- .../core/database/src/models/users/model.rs | 30 ++++++++++++++----- .../core/database/src/tasks/process_embeds.rs | 2 +- .../delta/src/routes/channels/message_edit.rs | 3 +- .../delta/src/routes/channels/message_send.rs | 1 + .../src/routes/webhooks/webhook_execute.rs | 2 ++ 9 files changed, 38 insertions(+), 22 deletions(-) diff --git a/crates/core/config/Revolt.toml b/crates/core/config/Revolt.toml index 9f5c0c26..3fa1fe80 100644 --- a/crates/core/config/Revolt.toml +++ b/crates/core/config/Revolt.toml @@ -48,6 +48,7 @@ webhooks_enabled = false [features.limits.global] group_size = 100 +message_embeds = 5 message_replies = 5 message_reactions = 20 server_emoji = 100 @@ -61,7 +62,6 @@ outgoing_friend_requests = 5 bots = 2 message_length = 2000 -message_embeds = 5 message_attachments = 5 servers = 100 @@ -77,7 +77,6 @@ outgoing_friend_requests = 10 bots = 5 message_length = 2000 -message_embeds = 5 message_attachments = 5 servers = 100 diff --git a/crates/core/config/src/lib.rs b/crates/core/config/src/lib.rs index 02d238dd..701fc14d 100644 --- a/crates/core/config/src/lib.rs +++ b/crates/core/config/src/lib.rs @@ -107,6 +107,7 @@ pub struct Api { #[derive(Deserialize, Debug, Clone)] pub struct GlobalLimits { pub group_size: usize, + pub message_embeds: usize, pub message_replies: usize, pub message_reactions: usize, pub server_emoji: usize, @@ -123,7 +124,6 @@ pub struct FeaturesLimits { pub bots: usize, pub message_length: usize, pub message_attachments: usize, - pub message_embeds: usize, pub servers: usize, pub attachment_size: usize, diff --git a/crates/core/database/src/models/bots/model.rs b/crates/core/database/src/models/bots/model.rs index 6b213432..cb36d008 100644 --- a/crates/core/database/src/models/bots/model.rs +++ b/crates/core/database/src/models/bots/model.rs @@ -80,8 +80,7 @@ impl Bot { return Err(create_error!(IsBot)); } - let config = config().await; - if db.get_number_of_bots_by_user(&owner.id).await? >= config.features.limits.default.bots { + if db.get_number_of_bots_by_user(&owner.id).await? >= owner.limits().await.bots { return Err(create_error!(ReachedMaximumBots)); } diff --git a/crates/core/database/src/models/messages/model.rs b/crates/core/database/src/models/messages/model.rs index afafff91..b46e73c7 100644 --- a/crates/core/database/src/models/messages/model.rs +++ b/crates/core/database/src/models/messages/model.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use indexmap::{IndexMap, IndexSet}; use iso8601_timestamp::Timestamp; -use revolt_config::config; +use revolt_config::{config, FeaturesLimits}; use revolt_models::v0::{ self, BulkMessageResponse, DataMessageSend, Embed, MessageAuthor, MessageSort, MessageWebhook, PushNotification, ReplyIntent, SendableEmbed, Text, RE_MENTION, @@ -207,11 +207,13 @@ impl Default for Message { #[allow(clippy::disallowed_methods)] impl Message { /// Create message from API data + #[allow(clippy::too_many_arguments)] pub async fn create_from_api( db: &Database, channel: Channel, data: DataMessageSend, author: MessageAuthor<'_>, + limits: FeaturesLimits, mut idempotency: IdempotencyKey, generate_embeds: bool, allow_mentions: bool, @@ -221,7 +223,7 @@ impl Message { Message::validate_sum( &data.content, data.embeds.as_deref().unwrap_or_default(), - config.features.limits.default.message_length, + limits.message_length, )?; idempotency @@ -320,20 +322,20 @@ impl Message { if data .attachments .as_ref() - .is_some_and(|v| v.len() > config.features.limits.default.message_attachments) + .is_some_and(|v| v.len() > limits.message_attachments) { return Err(create_error!(TooManyAttachments { - max: config.features.limits.default.message_attachments, + max: limits.message_attachments, })); } if data .embeds .as_ref() - .is_some_and(|v| v.len() > config.features.limits.default.message_embeds) + .is_some_and(|v| v.len() > config.features.limits.global.message_embeds) { return Err(create_error!(TooManyEmbeds { - max: config.features.limits.default.message_embeds, + max: config.features.limits.global.message_embeds, })); } diff --git a/crates/core/database/src/models/users/model.rs b/crates/core/database/src/models/users/model.rs index 1a3b9820..c434291d 100644 --- a/crates/core/database/src/models/users/model.rs +++ b/crates/core/database/src/models/users/model.rs @@ -1,10 +1,10 @@ -use std::{collections::HashSet, time::Duration}; +use std::{collections::HashSet, str::FromStr, time::Duration}; use crate::{events::client::EventV1, Database, File, RatelimitEvent}; use once_cell::sync::Lazy; use rand::seq::SliceRandom; -use revolt_config::config; +use revolt_config::{config, FeaturesLimits}; use revolt_models::v0; use revolt_presence::filter_online; use revolt_result::{create_error, Result}; @@ -197,6 +197,22 @@ impl User { Ok(user) } + /// Get limits for this user + pub async fn limits(&self) -> FeaturesLimits { + let config = config().await; + if ulid::Ulid::from_str(&self.id) + .expect("`ulid`") + .datetime() + .elapsed() + .expect("time went backwards") + <= Duration::from_secs(86400u64 * config.features.limits.global.new_user_days as u64) + { + config.features.limits.new_user + } else { + config.features.limits.default + } + } + /// Get the relationship with another user pub fn relationship_with(&self, user_b: &str) -> RelationshipStatus { if self.id == user_b { @@ -235,12 +251,11 @@ impl User { /// Check if this user can acquire another server pub async fn can_acquire_server(&self, db: &Database) -> Result<()> { - let config = config().await; - if db.fetch_server_count(&self.id).await? <= config.features.limits.default.servers { + if db.fetch_server_count(&self.id).await? <= self.limits().await.servers { Ok(()) } else { Err(create_error!(TooManyServers { - max: config.features.limits.default.servers + max: self.limits().await.servers })) } } @@ -500,10 +515,9 @@ impl User { .unwrap_or_default(); // If we're over the limit, don't allow creating more requests - let config = config().await; - if count >= config.features.limits.default.outgoing_friend_requests { + if count >= self.limits().await.outgoing_friend_requests { return Err(create_error!(TooManyPendingFriendRequests { - max: config.features.limits.default.outgoing_friend_requests + max: self.limits().await.outgoing_friend_requests })); } diff --git a/crates/core/database/src/tasks/process_embeds.rs b/crates/core/database/src/tasks/process_embeds.rs index 459b8b3a..d9d61d75 100644 --- a/crates/core/database/src/tasks/process_embeds.rs +++ b/crates/core/database/src/tasks/process_embeds.rs @@ -56,7 +56,7 @@ pub async fn worker(db: Database) { let embeds = generate( task.content, &config.hosts.january, - config.features.limits.default.message_embeds, + config.features.limits.global.message_embeds, semaphore, ) .await; diff --git a/crates/delta/src/routes/channels/message_edit.rs b/crates/delta/src/routes/channels/message_edit.rs index 4c1a78b9..41958e34 100644 --- a/crates/delta/src/routes/channels/message_edit.rs +++ b/crates/delta/src/routes/channels/message_edit.rs @@ -30,11 +30,10 @@ pub async fn edit( }) })?; - let config = config().await; Message::validate_sum( &edit.content, edit.embeds.as_deref().unwrap_or_default(), - config.features.limits.default.message_length, + user.limits().await.message_length, )?; // Ensure we have permissions to send a message diff --git a/crates/delta/src/routes/channels/message_send.rs b/crates/delta/src/routes/channels/message_send.rs index ab83be95..28b90651 100644 --- a/crates/delta/src/routes/channels/message_send.rs +++ b/crates/delta/src/routes/channels/message_send.rs @@ -81,6 +81,7 @@ pub async fn message_send( channel, data, v0::MessageAuthor::User(&author), + user.limits().await, idempotency, permissions.has_channel_permission(ChannelPermission::SendEmbeds), allow_mentions, diff --git a/crates/delta/src/routes/webhooks/webhook_execute.rs b/crates/delta/src/routes/webhooks/webhook_execute.rs index f370b186..6a6eebc4 100644 --- a/crates/delta/src/routes/webhooks/webhook_execute.rs +++ b/crates/delta/src/routes/webhooks/webhook_execute.rs @@ -1,3 +1,4 @@ +use revolt_config::config; use revolt_database::{ util::{idempotency::IdempotencyKey, reference::Reference}, Database, Message, @@ -58,6 +59,7 @@ pub async fn webhook_execute( channel, data, v0::MessageAuthor::Webhook(&webhook.into()), + config().await.features.limits.default, idempotency, true, true, From ac20b6bc990b36fd1f86a8b6ee15e013149928d9 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 25 Jun 2024 19:24:28 +0100 Subject: [PATCH 042/184] chore: bump version to 0.7.10 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f84412df..7bb67a67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.9" +version = "0.7.10" dependencies = [ "async-channel 2.3.1", "async-std", @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.9" +version = "0.7.10" dependencies = [ "async-std", "cached", @@ -3498,7 +3498,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.9" +version = "0.7.10" dependencies = [ "async-lock", "async-recursion", @@ -3544,7 +3544,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.9" +version = "0.7.10" dependencies = [ "async-channel 1.6.1", "async-std", @@ -3590,7 +3590,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.9" +version = "0.7.10" dependencies = [ "indexmap", "iso8601-timestamp 0.2.11", @@ -3607,7 +3607,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.9" +version = "0.7.10" dependencies = [ "async-std", "async-trait", @@ -3622,7 +3622,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.9" +version = "0.7.10" dependencies = [ "async-std", "log", @@ -3633,7 +3633,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.9" +version = "0.7.10" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 7181bbe3..74228288 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.9" +version = "0.7.10" license = "AGPL-3.0-or-later" edition = "2021" @@ -41,7 +41,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.9", path = "../core/permissions" } +revolt-permissions = { version = "0.7.10", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index 242c78f8..6805eab4 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.9" +version = "0.7.10" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 26e649d6..32263118 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.9" +version = "0.7.10" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.9", path = "../config" } -revolt-result = { version = "0.7.9", path = "../result" } -revolt-models = { version = "0.7.9", path = "../models", features = [ +revolt-config = { version = "0.7.10", path = "../config" } +revolt-result = { version = "0.7.10", path = "../result" } +revolt-models = { version = "0.7.10", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.9", path = "../presence" } -revolt-permissions = { version = "0.7.9", path = "../permissions", features = [ +revolt-presence = { version = "0.7.10", path = "../presence" } +revolt-permissions = { version = "0.7.10", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 1cb0ea83..b9b64e6d 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.9" +version = "0.7.10" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.9", path = "../config" } -revolt-permissions = { version = "0.7.9", path = "../permissions" } +revolt-config = { version = "0.7.10", path = "../config" } +revolt-permissions = { version = "0.7.10", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index 48fae107..99cba14a 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.9" +version = "0.7.10" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.9", path = "../result" } +revolt-result = { version = "0.7.10", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index ff99110d..e252908a 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.9" +version = "0.7.10" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index 3d34ea5c..27b02bc3 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.9" +version = "0.7.10" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 5e94af96..6450d16a 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.9" +version = "0.7.10" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From 1d5dae47518261f6f904bab1bccbf1c77a3622af Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 25 Jun 2024 21:01:45 +0100 Subject: [PATCH 043/184] feat(core): include user and member on Message events --- .../database/src/models/channels/model.rs | 6 ++ .../database/src/models/messages/model.rs | 23 +++++-- .../src/models/server_members/model.rs | 4 +- crates/core/database/src/util/bridge/v0.rs | 68 +++++++++++++------ crates/core/models/src/v0/messages.rs | 6 ++ .../delta/src/routes/channels/channel_edit.rs | 8 +-- .../delta/src/routes/channels/message_edit.rs | 2 +- .../src/routes/channels/message_fetch.rs | 2 +- .../delta/src/routes/channels/message_send.rs | 19 +++++- .../src/routes/webhooks/webhook_execute.rs | 4 +- .../routes/webhooks/webhook_execute_github.rs | 9 ++- 11 files changed, 117 insertions(+), 34 deletions(-) diff --git a/crates/core/database/src/models/channels/model.rs b/crates/core/database/src/models/channels/model.rs index 4a879c8c..d48d7aad 100644 --- a/crates/core/database/src/models/channels/model.rs +++ b/crates/core/database/src/models/channels/model.rs @@ -377,6 +377,8 @@ impl Channel { username: &user.username, avatar: user.avatar.as_ref().map(|file| file.id.as_ref()), }, + None, + None, self, false, ) @@ -688,6 +690,8 @@ impl Channel { username: name, avatar: None, }, + None, + None, self, false, ) @@ -723,6 +727,8 @@ impl Channel { username: &user.username, avatar: user.avatar.as_ref().map(|file| file.id.as_ref()), }, + None, + None, self, false, ) diff --git a/crates/core/database/src/models/messages/model.rs b/crates/core/database/src/models/messages/model.rs index b46e73c7..ea79c016 100644 --- a/crates/core/database/src/models/messages/model.rs +++ b/crates/core/database/src/models/messages/model.rs @@ -213,6 +213,8 @@ impl Message { channel: Channel, data: DataMessageSend, author: MessageAuthor<'_>, + user: Option, + member: Option, limits: FeaturesLimits, mut idempotency: IdempotencyKey, generate_embeds: bool, @@ -362,7 +364,9 @@ impl Message { message.nonce = Some(idempotency.into_key()); // Send the message - message.send(db, author, &channel, generate_embeds).await?; + message + .send(db, author, user, member, &channel, generate_embeds) + .await?; Ok(message) } @@ -371,13 +375,15 @@ impl Message { pub async fn send_without_notifications( &mut self, db: &Database, + user: Option, + member: Option, is_dm: bool, generate_embeds: bool, ) -> Result<()> { db.insert_message(self).await?; // Fan out events - EventV1::Message(self.clone().into()) + EventV1::Message(self.clone().into_model(user, member)) .p(self.channel.to_string()) .await; @@ -418,11 +424,15 @@ impl Message { &mut self, db: &Database, author: MessageAuthor<'_>, + user: Option, + member: Option, channel: &Channel, generate_embeds: bool, ) -> Result<()> { self.send_without_notifications( db, + user, + member, matches!(channel, Channel::DirectMessage { .. }), generate_embeds, ) @@ -438,7 +448,12 @@ impl Message { _ => vec![], } }, - PushNotification::from(self.clone().into(), Some(author), &channel.id()).await, + PushNotification::from( + self.clone().into_model(None, None), + Some(author), + &channel.id(), + ) + .await, ) .await; @@ -500,7 +515,7 @@ impl Message { .fetch_messages(query) .await? .into_iter() - .map(Into::into) + .map(|msg| msg.into_model(None, None)) .collect(); if let Some(true) = include_users { diff --git a/crates/core/database/src/models/server_members/model.rs b/crates/core/database/src/models/server_members/model.rs index 63a35ec3..2a38b8bb 100644 --- a/crates/core/database/src/models/server_members/model.rs +++ b/crates/core/database/src/models/server_members/model.rs @@ -150,7 +150,7 @@ impl Member { id: user.id.clone(), } .into_message(id.to_string()) - .send_without_notifications(db, false, false) + .send_without_notifications(db, None, None, false, false) .await .ok(); } @@ -250,7 +250,7 @@ impl Member { } .into_message(id.to_string()) // TODO: support notifications here in the future? - .send_without_notifications(db, false, false) + .send_without_notifications(db, None, None, false, false) .await .ok(); } diff --git a/crates/core/database/src/util/bridge/v0.rs b/crates/core/database/src/util/bridge/v0.rs index 5c1291f4..b0319059 100644 --- a/crates/core/database/src/util/bridge/v0.rs +++ b/crates/core/database/src/util/bridge/v0.rs @@ -459,26 +459,28 @@ impl From for crate::Metadata { } } -impl From for Message { - fn from(value: crate::Message) -> Self { +impl crate::Message { + pub fn into_model(self, user: Option, member: Option) -> Message { Message { - id: value.id, - nonce: value.nonce, - channel: value.channel, - author: value.author, - webhook: value.webhook, - content: value.content, - system: value.system.map(|system| system.into()), - attachments: value + id: self.id, + nonce: self.nonce, + channel: self.channel, + author: self.author, + user, + member, + webhook: self.webhook, + content: self.content, + system: self.system.map(|system| system.into()), + attachments: self .attachments .map(|v| v.into_iter().map(|f| f.into()).collect()), - edited: value.edited, - embeds: value.embeds, - mentions: value.mentions, - replies: value.replies, - reactions: value.reactions, - interactions: value.interactions.into(), - masquerade: value.masquerade.map(|masq| masq.into()), + edited: self.edited, + embeds: self.embeds, + mentions: self.mentions, + replies: self.replies, + reactions: self.reactions, + interactions: self.interactions.into(), + masquerade: self.masquerade.map(|masq| masq.into()), } } } @@ -490,6 +492,8 @@ impl From for PartialMessage { nonce: value.nonce, channel: value.channel, author: value.author, + user: None, + member: None, webhook: value.webhook, content: value.content, system: value.system.map(|system| system.into()), @@ -1089,7 +1093,33 @@ impl crate::User { } } - pub async fn into_self(self) -> User { + /// Convert user object into user model without presence information + pub fn into_known_static<'a>(self, is_online: bool) -> User { + User { + username: self.username, + discriminator: self.discriminator, + display_name: self.display_name, + avatar: self.avatar.map(|file| file.into()), + relations: vec![], + badges: self.badges.unwrap_or_default() as u32, + online: is_online + && !matches!( + self.status, + Some(crate::UserStatus { + presence: Some(crate::Presence::Invisible), + .. + }) + ), + status: self.status.map(|status| status.into()), + flags: self.flags.unwrap_or_default() as u32, + privileged: self.privileged, + bot: self.bot.map(|bot| bot.into()), + relationship: RelationshipStatus::None, // events client will populate this from cache + id: self.id, + } + } + + pub async fn into_self(self, force_online: bool) -> User { User { username: self.username, discriminator: self.discriminator, @@ -1105,7 +1135,7 @@ impl crate::User { }) .unwrap_or_default(), badges: self.badges.unwrap_or_default() as u32, - online: revolt_presence::is_online(&self.id).await + online: (force_online || revolt_presence::is_online(&self.id).await) && !matches!( self.status, Some(crate::UserStatus { diff --git a/crates/core/models/src/v0/messages.rs b/crates/core/models/src/v0/messages.rs index 885abaed..ce2e3cf0 100644 --- a/crates/core/models/src/v0/messages.rs +++ b/crates/core/models/src/v0/messages.rs @@ -31,6 +31,12 @@ auto_derived_partial!( pub channel: String, /// Id of the user or webhook that sent this message pub author: String, + /// The user that sent this message + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + /// The member that sent this message + #[serde(skip_serializing_if = "Option::is_none")] + pub member: Option, /// The webhook that sent this message #[serde(skip_serializing_if = "Option::is_none")] pub webhook: Option, diff --git a/crates/delta/src/routes/channels/channel_edit.rs b/crates/delta/src/routes/channels/channel_edit.rs index 3ee2774b..a26a7b34 100644 --- a/crates/delta/src/routes/channels/channel_edit.rs +++ b/crates/delta/src/routes/channels/channel_edit.rs @@ -73,7 +73,7 @@ pub async fn edit( return Err(create_error!(InvalidOperation)); } .into_message(channel.id().to_string()) - .send(db, user.as_author_for_system(), &channel, false) + .send(db, user.as_author_for_system(), None, None, &channel, false) .await .ok(); } @@ -151,7 +151,7 @@ pub async fn edit( by: user.id.clone(), } .into_message(channel.id().to_string()) - .send(db, user.as_author_for_system(), &channel, false) + .send(db, user.as_author_for_system(), None, None, &channel, false) .await .ok(); } @@ -161,7 +161,7 @@ pub async fn edit( by: user.id.clone(), } .into_message(channel.id().to_string()) - .send(db, user.as_author_for_system(), &channel, false) + .send(db, user.as_author_for_system(), None, None, &channel, false) .await .ok(); } @@ -171,7 +171,7 @@ pub async fn edit( by: user.id.clone(), } .into_message(channel.id().to_string()) - .send(db, user.as_author_for_system(), &channel, false) + .send(db, user.as_author_for_system(), None, None, &channel, false) .await .ok(); } diff --git a/crates/delta/src/routes/channels/message_edit.rs b/crates/delta/src/routes/channels/message_edit.rs index 41958e34..507ce03f 100644 --- a/crates/delta/src/routes/channels/message_edit.rs +++ b/crates/delta/src/routes/channels/message_edit.rs @@ -97,5 +97,5 @@ pub async fn edit( } } - Ok(Json(message.into())) + Ok(Json(message.into_model(None, None))) } diff --git a/crates/delta/src/routes/channels/message_fetch.rs b/crates/delta/src/routes/channels/message_fetch.rs index e2d4f94a..b5e89685 100644 --- a/crates/delta/src/routes/channels/message_fetch.rs +++ b/crates/delta/src/routes/channels/message_fetch.rs @@ -29,5 +29,5 @@ pub async fn fetch( return Err(create_error!(NotFound)); } - Ok(Json(message.into())) + Ok(Json(message.into_model(None, None))) } diff --git a/crates/delta/src/routes/channels/message_send.rs b/crates/delta/src/routes/channels/message_send.rs index 28b90651..93a29057 100644 --- a/crates/delta/src/routes/channels/message_send.rs +++ b/crates/delta/src/routes/channels/message_send.rs @@ -5,6 +5,7 @@ use revolt_database::{ }; use revolt_database::{Interactions, Message}; use revolt_models::v0; +use revolt_permissions::PermissionQuery; use revolt_permissions::{calculate_channel_permissions, ChannelPermission}; use revolt_result::{create_error, Result}; use rocket::serde::json::Json; @@ -75,18 +76,34 @@ pub async fn message_send( // Create the message let author: v0::User = user.clone().into(db, Some(&user)).await; + + // Make sure we have server member (edge case if server owner) + query.are_we_a_member().await; + + // Create model user / members + let model_user = user + .clone() + .into_known_static(revolt_presence::is_online(&user.id).await); + + let model_member: Option = query + .member_ref() + .as_ref() + .map(|member| member.clone().into_owned().into()); + Ok(Json( Message::create_from_api( db, channel, data, v0::MessageAuthor::User(&author), + Some(model_user.clone()), + model_member.clone(), user.limits().await, idempotency, permissions.has_channel_permission(ChannelPermission::SendEmbeds), allow_mentions, ) .await? - .into(), + .into_model(Some(model_user), model_member), )) } diff --git a/crates/delta/src/routes/webhooks/webhook_execute.rs b/crates/delta/src/routes/webhooks/webhook_execute.rs index 6a6eebc4..29b33dde 100644 --- a/crates/delta/src/routes/webhooks/webhook_execute.rs +++ b/crates/delta/src/routes/webhooks/webhook_execute.rs @@ -59,12 +59,14 @@ pub async fn webhook_execute( channel, data, v0::MessageAuthor::Webhook(&webhook.into()), + None, + None, config().await.features.limits.default, idempotency, true, true, ) .await? - .into(), + .into_model(None, None), )) } diff --git a/crates/delta/src/routes/webhooks/webhook_execute_github.rs b/crates/delta/src/routes/webhooks/webhook_execute_github.rs index 16142bef..85bf2e42 100644 --- a/crates/delta/src/routes/webhooks/webhook_execute_github.rs +++ b/crates/delta/src/routes/webhooks/webhook_execute_github.rs @@ -1072,6 +1072,13 @@ pub async fn webhook_execute_github( #[allow(clippy::disallowed_methods)] message.attach_sendable_embed(db, sendable_embed).await?; message - .send(db, MessageAuthor::Webhook(&webhook.into()), &channel, false) + .send( + db, + MessageAuthor::Webhook(&webhook.into()), + None, + None, + &channel, + false, + ) .await } From c5f4b94aa5ca4e313aa25cb49af87570843c4949 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 25 Jun 2024 21:02:27 +0100 Subject: [PATCH 044/184] fix(bonfire): Ready event should include user with online status true (if applicable) --- Cargo.lock | 1 + crates/bonfire/src/events/impl.rs | 16 +++++++++++++++- crates/delta/Cargo.toml | 1 + crates/delta/src/routes/bots/fetch_owned.rs | 2 +- crates/delta/src/routes/onboard/complete.rs | 2 +- crates/delta/src/routes/servers/ban_list.rs | 2 +- crates/delta/src/routes/users/edit_user.rs | 4 ++-- crates/delta/src/routes/users/fetch_self.rs | 2 +- crates/delta/src/routes/users/fetch_user.rs | 2 +- 9 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7bb67a67..f1fd1ecb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3572,6 +3572,7 @@ dependencies = [ "revolt-database", "revolt-models", "revolt-permissions", + "revolt-presence", "revolt-result", "revolt_rocket_okapi", "rocket", diff --git a/crates/bonfire/src/events/impl.rs b/crates/bonfire/src/events/impl.rs index 46c854b8..325267f4 100644 --- a/crates/bonfire/src/events/impl.rs +++ b/crates/bonfire/src/events/impl.rs @@ -180,7 +180,7 @@ impl State { .collect(); // Make sure we see our own user correctly. - users.push(user.into_self().await); + users.push(user.into_self(true).await); // Set subscription state internally. self.reset_state().await; @@ -540,6 +540,20 @@ impl State { } } + EventV1::Message(message) => { + // Since Message events are fanned out to many clients, + // we must reconstruct the relationship value at this end. + if let Some(user) = &mut message.user { + user.relationship = self + .cache + .users + .get(&self.cache.user_id) + .expect("missing self?") + .relationship_with(&message.author) + .into(); + } + } + _ => {} } diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 6450d16a..3d15fc2c 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -77,6 +77,7 @@ revolt-models = { path = "../core/models", features = [ "validator", "rocket", ] } +revolt-presence = { path = "../core/presence" } revolt-result = { path = "../core/result", features = ["rocket", "okapi"] } revolt-permissions = { path = "../core/permissions", features = ["schemas"] } diff --git a/crates/delta/src/routes/bots/fetch_owned.rs b/crates/delta/src/routes/bots/fetch_owned.rs index 8a40b228..db4017ce 100644 --- a/crates/delta/src/routes/bots/fetch_owned.rs +++ b/crates/delta/src/routes/bots/fetch_owned.rs @@ -24,7 +24,7 @@ pub async fn fetch_owned_bots(db: &State, user: User) -> Result Result> { - Ok(Json(user.into_self().await)) + Ok(Json(user.into_self(false).await)) } diff --git a/crates/delta/src/routes/users/fetch_user.rs b/crates/delta/src/routes/users/fetch_user.rs index e8886ca9..68be67b9 100644 --- a/crates/delta/src/routes/users/fetch_user.rs +++ b/crates/delta/src/routes/users/fetch_user.rs @@ -15,7 +15,7 @@ use rocket::{serde::json::Json, State}; #[get("/")] pub async fn fetch(db: &State, user: User, target: Reference) -> Result> { if user.id == target.id { - return Ok(Json(user.into_self().await)); + return Ok(Json(user.into_self(false).await)); } let target = target.as_user(db).await?; From f903f716e3a7cdcd1d67f2aa7bc1990825c325b4 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 25 Jun 2024 21:03:00 +0100 Subject: [PATCH 045/184] chore: bump version to 0.7.11 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1fd1ecb..a6bc715a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.10" +version = "0.7.11" dependencies = [ "async-channel 2.3.1", "async-std", @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.10" +version = "0.7.11" dependencies = [ "async-std", "cached", @@ -3498,7 +3498,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.10" +version = "0.7.11" dependencies = [ "async-lock", "async-recursion", @@ -3544,7 +3544,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.10" +version = "0.7.11" dependencies = [ "async-channel 1.6.1", "async-std", @@ -3591,7 +3591,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.10" +version = "0.7.11" dependencies = [ "indexmap", "iso8601-timestamp 0.2.11", @@ -3608,7 +3608,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.10" +version = "0.7.11" dependencies = [ "async-std", "async-trait", @@ -3623,7 +3623,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.10" +version = "0.7.11" dependencies = [ "async-std", "log", @@ -3634,7 +3634,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.10" +version = "0.7.11" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 74228288..aadf82da 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.10" +version = "0.7.11" license = "AGPL-3.0-or-later" edition = "2021" @@ -41,7 +41,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.10", path = "../core/permissions" } +revolt-permissions = { version = "0.7.11", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index 6805eab4..79ce9846 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.10" +version = "0.7.11" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 32263118..44ed346b 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.10" +version = "0.7.11" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.10", path = "../config" } -revolt-result = { version = "0.7.10", path = "../result" } -revolt-models = { version = "0.7.10", path = "../models", features = [ +revolt-config = { version = "0.7.11", path = "../config" } +revolt-result = { version = "0.7.11", path = "../result" } +revolt-models = { version = "0.7.11", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.10", path = "../presence" } -revolt-permissions = { version = "0.7.10", path = "../permissions", features = [ +revolt-presence = { version = "0.7.11", path = "../presence" } +revolt-permissions = { version = "0.7.11", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index b9b64e6d..6ba82f04 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.10" +version = "0.7.11" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.10", path = "../config" } -revolt-permissions = { version = "0.7.10", path = "../permissions" } +revolt-config = { version = "0.7.11", path = "../config" } +revolt-permissions = { version = "0.7.11", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index 99cba14a..eacd0184 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.10" +version = "0.7.11" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.10", path = "../result" } +revolt-result = { version = "0.7.11", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index e252908a..361bcd32 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.10" +version = "0.7.11" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index 27b02bc3..16582817 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.10" +version = "0.7.11" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 3d15fc2c..000accd7 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.10" +version = "0.7.11" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From ea6ba598411a15eb54f0a350df077c4f75ccb279 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 26 Jun 2024 14:54:06 +0100 Subject: [PATCH 046/184] fix: add `DisplayName` back to user fields (prevented removing name) --- crates/core/database/src/models/users/model.rs | 2 ++ crates/core/database/src/models/users/ops/mongodb.rs | 1 + crates/core/database/src/util/bridge/v0.rs | 2 ++ crates/core/models/src/v0/users.rs | 1 + 4 files changed, 6 insertions(+) diff --git a/crates/core/database/src/models/users/model.rs b/crates/core/database/src/models/users/model.rs index c434291d..2639504a 100644 --- a/crates/core/database/src/models/users/model.rs +++ b/crates/core/database/src/models/users/model.rs @@ -61,6 +61,7 @@ auto_derived!( StatusPresence, ProfileContent, ProfileBackground, + DisplayName, } /// User's relationship with another user (or themselves) @@ -657,6 +658,7 @@ impl User { x.background = None; } } + FieldsUser::DisplayName => self.display_name = None, } } diff --git a/crates/core/database/src/models/users/ops/mongodb.rs b/crates/core/database/src/models/users/ops/mongodb.rs index 99b02094..4f812a15 100644 --- a/crates/core/database/src/models/users/ops/mongodb.rs +++ b/crates/core/database/src/models/users/ops/mongodb.rs @@ -326,6 +326,7 @@ impl IntoDocumentPath for FieldsUser { FieldsUser::ProfileContent => "profile.content", FieldsUser::StatusPresence => "status.presence", FieldsUser::StatusText => "status.text", + FieldsUser::DisplayName => "display_name", }) } } diff --git a/crates/core/database/src/util/bridge/v0.rs b/crates/core/database/src/util/bridge/v0.rs index b0319059..b185c9c1 100644 --- a/crates/core/database/src/util/bridge/v0.rs +++ b/crates/core/database/src/util/bridge/v0.rs @@ -1212,6 +1212,7 @@ impl From for crate::FieldsUser { FieldsUser::ProfileContent => crate::FieldsUser::ProfileContent, FieldsUser::StatusPresence => crate::FieldsUser::StatusPresence, FieldsUser::StatusText => crate::FieldsUser::StatusText, + FieldsUser::DisplayName => crate::FieldsUser::DisplayName, } } } @@ -1224,6 +1225,7 @@ impl From for FieldsUser { crate::FieldsUser::ProfileContent => FieldsUser::ProfileContent, crate::FieldsUser::StatusPresence => FieldsUser::StatusPresence, crate::FieldsUser::StatusText => FieldsUser::StatusText, + crate::FieldsUser::DisplayName => FieldsUser::DisplayName, } } } diff --git a/crates/core/models/src/v0/users.rs b/crates/core/models/src/v0/users.rs index ac3a8d1d..8d72eb18 100644 --- a/crates/core/models/src/v0/users.rs +++ b/crates/core/models/src/v0/users.rs @@ -83,6 +83,7 @@ auto_derived!( StatusPresence, ProfileContent, ProfileBackground, + DisplayName, } /// User's relationship with another user (or themselves) From ec578ab0ef9df5e596d87b61250c91a69a1c64db Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 26 Jun 2024 15:07:19 +0100 Subject: [PATCH 047/184] chore: lessen restrictions on interacting with bot users as owner --- crates/core/database/src/util/permissions.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/core/database/src/util/permissions.rs b/crates/core/database/src/util/permissions.rs index 3adb2ec3..e80b0da1 100644 --- a/crates/core/database/src/util/permissions.rs +++ b/crates/core/database/src/util/permissions.rs @@ -53,6 +53,12 @@ impl PermissionQuery for DatabasePermissionQuery<'_> { if let Some(other_user) = &self.user { if self.perspective.id == other_user.id { return RelationshipStatus::User; + } else if let Some(bot) = &other_user.bot { + // For the purposes of permissions checks, + // assume owner is the same as bot + if self.perspective.id == bot.owner { + return RelationshipStatus::User; + } } if let Some(relations) = &self.perspective.relations { From ac4e2cb10b40930b7a08709e2393be2165c8e7bf Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 26 Jun 2024 15:08:12 +0100 Subject: [PATCH 048/184] ci: disable image builds if they're not tagged [skip ci] (seems like a waste of CPU) --- .github/workflows/docker.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 3e919761..ccd1c1a7 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -2,8 +2,8 @@ name: Docker Test & Publish on: push: - branches: - - "main" + # branches: + # - "main" tags: - "*" paths-ignore: From fa598dd6f885c34ab73a608dcf5b44aa09837d41 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 26 Jun 2024 17:18:10 +0100 Subject: [PATCH 049/184] docs: update README.md Signed-off-by: Paul Makles --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 65069e67..c47a625a 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ Rust 1.70 or higher. ## Development Guide +Before contributing, make yourself familiar with [our contribution guidelines](https://developers.revolt.chat/contrib.html) and the [technical documentation for this project](https://revoltchat.github.io/backend/). + Before getting started, you'll want to install: - Rust toolchain (rustup recommended) From 00e881799f255fecfb302eb08b82dd50b1de784e Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 26 Jun 2024 18:24:38 +0100 Subject: [PATCH 050/184] fix: replace user with target user when editing someone else --- crates/delta/src/routes/users/edit_user.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/delta/src/routes/users/edit_user.rs b/crates/delta/src/routes/users/edit_user.rs index b6aab49e..d98c01ee 100644 --- a/crates/delta/src/routes/users/edit_user.rs +++ b/crates/delta/src/routes/users/edit_user.rs @@ -24,23 +24,26 @@ pub async fn edit( }) })?; + // Filter out invalid edit fields + if !user.privileged && (data.badges.is_some() || data.flags.is_some()) { + return Err(create_error!(NotPrivileged)); + } + // If we want to edit a different user than self, ensure we have // permissions and subsequently replace the user in question if target.id != "@me" && target.id != user.id { let target_user = target.as_user(db).await?; let is_bot_owner = target_user .bot + .as_ref() .map(|bot| bot.owner == user.id) .unwrap_or_default(); if !is_bot_owner && !user.privileged { return Err(create_error!(NotPrivileged)); } - } - // Otherwise, filter out invalid edit fields - if !user.privileged && (data.badges.is_some() || data.flags.is_some()) { - return Err(create_error!(NotPrivileged)); + user = target_user; } // Exit out early if nothing is changed From 64a07d09f433227da70e84e6a4131563f734d17b Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 26 Jun 2024 18:46:05 +0100 Subject: [PATCH 051/184] feat: silent messages and message flags --- .../database/src/models/messages/model.rs | 63 +++++++++++++------ crates/core/database/src/util/bridge/v0.rs | 12 ++-- crates/core/models/src/v0/messages.rs | 21 +++++++ crates/core/models/src/v0/users.rs | 4 ++ 4 files changed, 76 insertions(+), 24 deletions(-) diff --git a/crates/core/database/src/models/messages/model.rs b/crates/core/database/src/models/messages/model.rs index ea79c016..abb0597b 100644 --- a/crates/core/database/src/models/messages/model.rs +++ b/crates/core/database/src/models/messages/model.rs @@ -4,8 +4,8 @@ use indexmap::{IndexMap, IndexSet}; use iso8601_timestamp::Timestamp; use revolt_config::{config, FeaturesLimits}; use revolt_models::v0::{ - self, BulkMessageResponse, DataMessageSend, Embed, MessageAuthor, MessageSort, MessageWebhook, - PushNotification, ReplyIntent, SendableEmbed, Text, RE_MENTION, + self, BulkMessageResponse, DataMessageSend, Embed, MessageAuthor, MessageFlags, MessageSort, + MessageWebhook, PushNotification, ReplyIntent, SendableEmbed, Text, RE_MENTION, }; use revolt_permissions::{ChannelPermission, PermissionValue}; use revolt_result::Result; @@ -65,6 +65,10 @@ auto_derived_partial!( /// Name and / or avatar overrides for this message #[serde(skip_serializing_if = "Option::is_none")] pub masquerade: Option, + + /// Bitfield of message flags + #[serde(skip_serializing_if = "Option::is_none")] + pub flags: Option, }, "PartialMessage" ); @@ -200,6 +204,7 @@ impl Default for Message { reactions: Default::default(), interactions: Default::default(), masquerade: None, + flags: None, } } } @@ -241,6 +246,13 @@ impl Message { return Err(create_error!(EmptyMessage)); } + // Ensure flags are either not set or have permissible values + if let Some(flags) = &data.flags { + if flags != &0 && flags != &1 { + return Err(create_error!(InvalidProperty)); + } + } + // Ensure restrict_reactions is not specified without reactions list if let Some(interactions) = &data.interactions { if interactions.restrict_reactions { @@ -274,6 +286,7 @@ impl Message { .unwrap_or_default(), author: author_id, webhook: webhook.map(|w| w.into()), + flags: data.flags.map(|v| v as i32), ..Default::default() }; @@ -438,24 +451,26 @@ impl Message { ) .await?; - // Push out Web Push notifications - crate::tasks::web_push::queue( - { - match channel { - Channel::DirectMessage { recipients, .. } - | Channel::Group { recipients, .. } => recipients.clone(), - Channel::TextChannel { .. } => self.mentions.clone().unwrap_or_default(), - _ => vec![], - } - }, - PushNotification::from( - self.clone().into_model(None, None), - Some(author), - &channel.id(), + if !self.has_suppressed_notifications() { + // Push out Web Push notifications + crate::tasks::web_push::queue( + { + match channel { + Channel::DirectMessage { recipients, .. } + | Channel::Group { recipients, .. } => recipients.clone(), + Channel::TextChannel { .. } => self.mentions.clone().unwrap_or_default(), + _ => vec![], + } + }, + PushNotification::from( + self.clone().into_model(None, None), + Some(author), + &channel.id(), + ) + .await, ) - .await, - ) - .await; + .await; + } Ok(()) } @@ -487,6 +502,16 @@ impl Message { })) } + /// Whether this message has suppressed notifications + pub fn has_suppressed_notifications(&self) -> bool { + if let Some(flags) = self.flags { + flags & MessageFlags::SupressNotifications as i32 + == MessageFlags::SupressNotifications as i32 + } else { + false + } + } + /// Update message data pub async fn update(&mut self, db: &Database, partial: PartialMessage) -> Result<()> { self.apply_options(partial.clone()); diff --git a/crates/core/database/src/util/bridge/v0.rs b/crates/core/database/src/util/bridge/v0.rs index b185c9c1..747857cd 100644 --- a/crates/core/database/src/util/bridge/v0.rs +++ b/crates/core/database/src/util/bridge/v0.rs @@ -470,7 +470,7 @@ impl crate::Message { member, webhook: self.webhook, content: self.content, - system: self.system.map(|system| system.into()), + system: self.system.map(Into::into), attachments: self .attachments .map(|v| v.into_iter().map(|f| f.into()).collect()), @@ -480,7 +480,8 @@ impl crate::Message { replies: self.replies, reactions: self.reactions, interactions: self.interactions.into(), - masquerade: self.masquerade.map(|masq| masq.into()), + masquerade: self.masquerade.map(Into::into), + flags: self.flags.map(|flags| flags as u32).unwrap_or_default(), } } } @@ -496,7 +497,7 @@ impl From for PartialMessage { member: None, webhook: value.webhook, content: value.content, - system: value.system.map(|system| system.into()), + system: value.system.map(Into::into), attachments: value .attachments .map(|v| v.into_iter().map(|f| f.into()).collect()), @@ -505,8 +506,9 @@ impl From for PartialMessage { mentions: value.mentions, replies: value.replies, reactions: value.reactions, - interactions: value.interactions.map(|interactions| interactions.into()), - masquerade: value.masquerade.map(|masq| masq.into()), + interactions: value.interactions.map(Into::into), + masquerade: value.masquerade.map(Into::into), + flags: value.flags.map(|flags| flags as u32), } } } diff --git a/crates/core/models/src/v0/messages.rs b/crates/core/models/src/v0/messages.rs index ce2e3cf0..8d0d601c 100644 --- a/crates/core/models/src/v0/messages.rs +++ b/crates/core/models/src/v0/messages.rs @@ -70,6 +70,15 @@ auto_derived_partial!( /// Name and / or avatar overrides for this message #[serde(skip_serializing_if = "Option::is_none")] pub masquerade: Option, + + /// Bitfield of message flags + /// + /// https://docs.rs/revolt-models/latest/revolt_models/v0/enum.MessageFlags.html + #[cfg_attr( + feature = "serde", + serde(skip_serializing_if = "crate::if_zero_u32", default) + )] + pub flags: u32, }, "PartialMessage" ); @@ -247,6 +256,11 @@ auto_derived!( pub masquerade: Option, /// Information about how this message should be interacted with pub interactions: Option, + + /// Bitfield of message flags + /// + /// https://docs.rs/revolt-models/latest/revolt_models/v0/enum.MessageFlags.html + pub flags: Option, } /// Options for querying messages @@ -334,6 +348,13 @@ auto_derived!( /// Remove all reactions pub remove_all: Option, } + + /// Message flag bitfield + #[repr(u32)] + pub enum MessageFlags { + /// Message will not send push / desktop notifications + SupressNotifications = 1, + } ); /// Message Author Abstraction diff --git a/crates/core/models/src/v0/users.rs b/crates/core/models/src/v0/users.rs index 8d72eb18..58a7f2c4 100644 --- a/crates/core/models/src/v0/users.rs +++ b/crates/core/models/src/v0/users.rs @@ -42,6 +42,8 @@ auto_derived_partial!( pub relations: Vec, /// Bitfield of user badges + /// + /// https://docs.rs/revolt-models/latest/revolt_models/v0/enum.UserBadges.html #[cfg_attr( feature = "serde", serde(skip_serializing_if = "crate::if_zero_u32", default) @@ -52,6 +54,8 @@ auto_derived_partial!( pub status: Option, /// Enum of user flags + /// + /// https://docs.rs/revolt-models/latest/revolt_models/v0/enum.UserFlags.html #[cfg_attr( feature = "serde", serde(skip_serializing_if = "crate::if_zero_u32", default) From d9cfc9d0e10c8f95fe34568a0f49953bf25c3b44 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 26 Jun 2024 18:47:40 +0100 Subject: [PATCH 052/184] chore: bump version to 0.7.12 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6bc715a..92da288e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.11" +version = "0.7.12" dependencies = [ "async-channel 2.3.1", "async-std", @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.11" +version = "0.7.12" dependencies = [ "async-std", "cached", @@ -3498,7 +3498,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.11" +version = "0.7.12" dependencies = [ "async-lock", "async-recursion", @@ -3544,7 +3544,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.11" +version = "0.7.12" dependencies = [ "async-channel 1.6.1", "async-std", @@ -3591,7 +3591,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.11" +version = "0.7.12" dependencies = [ "indexmap", "iso8601-timestamp 0.2.11", @@ -3608,7 +3608,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.11" +version = "0.7.12" dependencies = [ "async-std", "async-trait", @@ -3623,7 +3623,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.11" +version = "0.7.12" dependencies = [ "async-std", "log", @@ -3634,7 +3634,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.11" +version = "0.7.12" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index aadf82da..46a53ff8 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.11" +version = "0.7.12" license = "AGPL-3.0-or-later" edition = "2021" @@ -41,7 +41,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.11", path = "../core/permissions" } +revolt-permissions = { version = "0.7.12", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index 79ce9846..5cc29902 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.11" +version = "0.7.12" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 44ed346b..16615f16 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.11" +version = "0.7.12" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.11", path = "../config" } -revolt-result = { version = "0.7.11", path = "../result" } -revolt-models = { version = "0.7.11", path = "../models", features = [ +revolt-config = { version = "0.7.12", path = "../config" } +revolt-result = { version = "0.7.12", path = "../result" } +revolt-models = { version = "0.7.12", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.11", path = "../presence" } -revolt-permissions = { version = "0.7.11", path = "../permissions", features = [ +revolt-presence = { version = "0.7.12", path = "../presence" } +revolt-permissions = { version = "0.7.12", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 6ba82f04..b0e9d31c 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.11" +version = "0.7.12" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.11", path = "../config" } -revolt-permissions = { version = "0.7.11", path = "../permissions" } +revolt-config = { version = "0.7.12", path = "../config" } +revolt-permissions = { version = "0.7.12", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index eacd0184..1c7f8ccf 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.11" +version = "0.7.12" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.11", path = "../result" } +revolt-result = { version = "0.7.12", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index 361bcd32..d6b5cd51 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.11" +version = "0.7.12" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index 16582817..bef4004e 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.11" +version = "0.7.12" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 000accd7..3af0bc19 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.11" +version = "0.7.12" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From ce20e689cc58867a6828f0354a44af3f0ac9a4f2 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 26 Jun 2024 19:48:42 +0100 Subject: [PATCH 053/184] refactor: fix spelling of "suppress" --- crates/core/database/src/models/messages/model.rs | 4 ++-- crates/core/models/src/v0/messages.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/core/database/src/models/messages/model.rs b/crates/core/database/src/models/messages/model.rs index abb0597b..dd507a30 100644 --- a/crates/core/database/src/models/messages/model.rs +++ b/crates/core/database/src/models/messages/model.rs @@ -505,8 +505,8 @@ impl Message { /// Whether this message has suppressed notifications pub fn has_suppressed_notifications(&self) -> bool { if let Some(flags) = self.flags { - flags & MessageFlags::SupressNotifications as i32 - == MessageFlags::SupressNotifications as i32 + flags & MessageFlags::SuppressNotifications as i32 + == MessageFlags::SuppressNotifications as i32 } else { false } diff --git a/crates/core/models/src/v0/messages.rs b/crates/core/models/src/v0/messages.rs index 8d0d601c..2448c93f 100644 --- a/crates/core/models/src/v0/messages.rs +++ b/crates/core/models/src/v0/messages.rs @@ -353,7 +353,7 @@ auto_derived!( #[repr(u32)] pub enum MessageFlags { /// Message will not send push / desktop notifications - SupressNotifications = 1, + SuppressNotifications = 1, } ); From d6bcb844dbae30a1e06cda763091eb289da98819 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 26 Jun 2024 19:49:26 +0100 Subject: [PATCH 054/184] feat: include user in responses when creating and editing bots --- crates/core/database/src/models/bots/model.rs | 11 ++++++++--- crates/core/models/src/v0/bots.rs | 7 +++++++ crates/delta/src/routes/bots/create.rs | 9 ++++++--- crates/delta/src/routes/bots/delete.rs | 2 +- crates/delta/src/routes/bots/edit.rs | 16 +++++++++++----- crates/delta/src/routes/bots/fetch.rs | 2 +- crates/delta/src/routes/bots/fetch_owned.rs | 2 +- crates/delta/src/routes/bots/fetch_public.rs | 2 +- crates/delta/src/routes/bots/invite.rs | 4 ++-- 9 files changed, 38 insertions(+), 17 deletions(-) diff --git a/crates/core/database/src/models/bots/model.rs b/crates/core/database/src/models/bots/model.rs index cb36d008..922d574d 100644 --- a/crates/core/database/src/models/bots/model.rs +++ b/crates/core/database/src/models/bots/model.rs @@ -72,7 +72,12 @@ impl Default for Bot { #[allow(clippy::disallowed_methods)] impl Bot { /// Create a new bot - pub async fn create(db: &Database, username: String, owner: &User, data: D) -> Result + pub async fn create( + db: &Database, + username: String, + owner: &User, + data: D, + ) -> Result<(Bot, User)> where D: Into>, { @@ -86,7 +91,7 @@ impl Bot { let id = Ulid::new().to_string(); - User::create( + let user = User::create( db, username, Some(id.to_string()), @@ -111,7 +116,7 @@ impl Bot { } db.insert_bot(&bot).await?; - Ok(bot) + Ok((bot, user)) } /// Remove a field from this object diff --git a/crates/core/models/src/v0/bots.rs b/crates/core/models/src/v0/bots.rs index 57995ae9..0e58bd15 100644 --- a/crates/core/models/src/v0/bots.rs +++ b/crates/core/models/src/v0/bots.rs @@ -162,4 +162,11 @@ auto_derived!( /// User objects pub users: Vec, } + + /// Bot with user response + pub struct BotWithUserResponse { + #[serde(flatten)] + pub bot: Bot, + pub user: User, + } ); diff --git a/crates/delta/src/routes/bots/create.rs b/crates/delta/src/routes/bots/create.rs index f92b98e9..67a97f07 100644 --- a/crates/delta/src/routes/bots/create.rs +++ b/crates/delta/src/routes/bots/create.rs @@ -14,7 +14,7 @@ pub async fn create_bot( db: &State, user: User, info: Json, -) -> Result> { +) -> Result> { let info = info.into_inner(); info.validate().map_err(|error| { create_error!(FailedValidation { @@ -22,8 +22,11 @@ pub async fn create_bot( }) })?; - let bot = Bot::create(db, info.name, &user, None).await?; - Ok(Json(bot.into())) + let (bot, user) = Bot::create(db, info.name, &user, None).await?; + Ok(Json(v0::BotWithUserResponse { + bot: bot.into(), + user: user.into_self(false).await, + })) } #[cfg(test)] diff --git a/crates/delta/src/routes/bots/delete.rs b/crates/delta/src/routes/bots/delete.rs index 6b421b67..7785a544 100644 --- a/crates/delta/src/routes/bots/delete.rs +++ b/crates/delta/src/routes/bots/delete.rs @@ -32,7 +32,7 @@ mod test { let mut harness = TestHarness::new().await; let (_, session, user) = harness.new_user().await; - let bot = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) + let (bot, _) = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) .await .expect("`Bot`"); diff --git a/crates/delta/src/routes/bots/edit.rs b/crates/delta/src/routes/bots/edit.rs index 70170177..e90b4bef 100644 --- a/crates/delta/src/routes/bots/edit.rs +++ b/crates/delta/src/routes/bots/edit.rs @@ -16,7 +16,7 @@ pub async fn edit_bot( user: User, target: Reference, data: Json, -) -> Result> { +) -> Result> { let data = data.into_inner(); data.validate().map_err(|error| { create_error!(FailedValidation { @@ -29,8 +29,8 @@ pub async fn edit_bot( return Err(create_error!(NotFound)); } + let mut user = db.fetch_user(&bot.id).await?; if let Some(name) = data.name { - let mut user = db.fetch_user(&bot.id).await?; user.update_username(db, name).await?; } @@ -39,7 +39,10 @@ pub async fn edit_bot( && data.interactions_url.is_none() && data.remove.is_none() { - return Ok(Json(bot.into())); + return Ok(Json(v0::BotWithUserResponse { + bot: bot.into(), + user: user.into_self(false).await, + })); } let DataEditBot { @@ -68,7 +71,10 @@ pub async fn edit_bot( ) .await?; - Ok(Json(bot.into())) + Ok(Json(v0::BotWithUserResponse { + bot: bot.into(), + user: user.into_self(false).await, + })) } #[cfg(test)] @@ -83,7 +89,7 @@ mod test { let harness = TestHarness::new().await; let (_, session, user) = harness.new_user().await; - let bot = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) + let (bot, _) = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) .await .expect("`Bot`"); diff --git a/crates/delta/src/routes/bots/fetch.rs b/crates/delta/src/routes/bots/fetch.rs index 6e922f71..a5222b8f 100644 --- a/crates/delta/src/routes/bots/fetch.rs +++ b/crates/delta/src/routes/bots/fetch.rs @@ -40,7 +40,7 @@ mod test { let harness = TestHarness::new().await; let (_, session, user) = harness.new_user().await; - let bot = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) + let (bot, _) = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) .await .expect("`Bot`"); diff --git a/crates/delta/src/routes/bots/fetch_owned.rs b/crates/delta/src/routes/bots/fetch_owned.rs index db4017ce..9eb320ef 100644 --- a/crates/delta/src/routes/bots/fetch_owned.rs +++ b/crates/delta/src/routes/bots/fetch_owned.rs @@ -41,7 +41,7 @@ mod test { let harness = TestHarness::new().await; let (_, session, user) = harness.new_user().await; - let bot = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) + let (bot, _) = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) .await .expect("`Bot`"); diff --git a/crates/delta/src/routes/bots/fetch_public.rs b/crates/delta/src/routes/bots/fetch_public.rs index 93ea1e7a..5363772c 100644 --- a/crates/delta/src/routes/bots/fetch_public.rs +++ b/crates/delta/src/routes/bots/fetch_public.rs @@ -35,7 +35,7 @@ mod test { let harness = TestHarness::new().await; let (_, _, user) = harness.new_user().await; - let mut bot = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) + let (mut bot, _) = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) .await .expect("`Bot`"); diff --git a/crates/delta/src/routes/bots/invite.rs b/crates/delta/src/routes/bots/invite.rs index 24228627..70e68bb6 100644 --- a/crates/delta/src/routes/bots/invite.rs +++ b/crates/delta/src/routes/bots/invite.rs @@ -74,7 +74,7 @@ mod test { let mut harness = TestHarness::new().await; let (_, session, user) = harness.new_user().await; - let bot = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) + let (bot, _) = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) .await .expect("`Bot`"); @@ -121,7 +121,7 @@ mod test { let mut harness = TestHarness::new().await; let (_, session, user) = harness.new_user().await; - let bot = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) + let (bot, _) = Bot::create(&harness.db, TestHarness::rand_string(), &user, None) .await .expect("`Bot`"); From 9ea2bd9f2fd2002361c5a74d4dd0ef2559d90bb1 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 29 Jun 2024 17:35:47 +0100 Subject: [PATCH 055/184] feat: implement apple push notifications --- Cargo.lock | 385 +++++++++++++++--- crates/core/config/Revolt.toml | 5 + crates/core/config/src/lib.rs | 10 + crates/core/database/Cargo.toml | 3 + crates/core/database/src/models/users/ops.rs | 3 + .../database/src/models/users/ops/mongodb.rs | 19 + .../src/models/users/ops/reference.rs | 5 + .../database/src/tasks/apple_notifications.rs | 113 +++++ crates/core/database/src/tasks/mod.rs | 3 + crates/core/database/src/tasks/web_push.rs | 12 + 10 files changed, 499 insertions(+), 59 deletions(-) create mode 100644 crates/core/database/src/tasks/apple_notifications.rs diff --git a/Cargo.lock b/Cargo.lock index 92da288e..8623be67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,9 +40,9 @@ dependencies = [ [[package]] name = "aes-gcm" -version = "0.9.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df5f85a83a7d8b0442b6aa7b504b8212c1733da07b98aae43d4bc21b2cb3cdf6" +checksum = "bc3be92e19a7ef47457b8e6f90707e12b6ac5d20c6f3866584fa3be0787d839f" dependencies = [ "aead", "aes", @@ -457,6 +457,12 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.6.0" @@ -561,7 +567,7 @@ dependencies = [ "base64 0.13.0", "chrono", "hex", - "indexmap", + "indexmap 1.9.3", "lazy_static", "rand 0.8.5", "serde", @@ -898,9 +904,9 @@ dependencies = [ [[package]] name = "ctr" -version = "0.8.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +checksum = "a232f92a03f37dd7d7dd2adc67166c77e9cd88de5b019b9a9eecfaeaf7bfd481" dependencies = [ "cipher", ] @@ -1297,6 +1303,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "erased-serde" version = "0.3.20" @@ -1625,13 +1637,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.6" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "libc", - "wasi 0.10.0+wasi-snapshot-preview1", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -1715,8 +1727,27 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", - "indexmap", + "http 0.2.7", + "indexmap 1.9.3", + "slab", + "tokio 1.35.1", + "tokio-util 0.7.2", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +dependencies = [ + "atomic-waker", + "bytes 1.5.0", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", + "indexmap 2.0.1", "slab", "tokio 1.35.1", "tokio-util 0.7.2", @@ -1856,6 +1887,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes 1.5.0", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -1863,7 +1905,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes 1.5.0", - "http", + "http 0.2.7", + "pin-project-lite 0.2.13", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes 1.5.0", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes 1.5.0", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite 0.2.13", ] @@ -1898,9 +1963,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.13", + "http 0.2.7", + "http-body 0.4.5", "httparse", "httpdate", "itoa", @@ -1912,6 +1977,42 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes 1.5.0", + "futures-channel", + "futures-util", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.0", + "pin-project-lite 0.2.13", + "smallvec", + "tokio 1.35.1", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.3.1", + "hyper-util", + "rustls 0.22.4", + "rustls-pki-types", + "tokio 1.35.1", + "tokio-rustls 0.25.0", + "tower-service", + "webpki-roots 0.26.3", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -1919,12 +2020,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.5.0", - "hyper", + "hyper 0.14.19", "native-tls", "tokio 1.35.1", "tokio-native-tls", ] +[[package]] +name = "hyper-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +dependencies = [ + "bytes 1.5.0", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", + "pin-project-lite 0.2.13", + "socket2 0.5.5", + "tokio 1.35.1", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1975,6 +2096,16 @@ dependencies = [ "serde", ] +[[package]] +name = "indexmap" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "inlinable_string" version = "0.1.15" @@ -2022,7 +2153,7 @@ dependencies = [ "encoding_rs", "event-listener 2.5.2", "futures-lite", - "http", + "http 0.2.7", "log", "mime", "once_cell", @@ -2461,8 +2592,8 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "rustc_version_runtime", - "rustls", - "rustls-pemfile", + "rustls 0.20.6", + "rustls-pemfile 0.3.0", "serde", "serde_with", "sha-1 0.10.0", @@ -2473,13 +2604,13 @@ dependencies = [ "take_mut", "thiserror", "tokio 1.35.1", - "tokio-rustls", + "tokio-rustls 0.23.4", "tokio-util 0.7.2", "trust-dns-proto", "trust-dns-resolver", "typed-builder", "uuid 0.8.2", - "webpki-roots", + "webpki-roots 0.22.3", ] [[package]] @@ -2491,7 +2622,7 @@ dependencies = [ "bytes 1.5.0", "encoding_rs", "futures-util", - "http", + "http 0.2.7", "httparse", "log", "memchr", @@ -2856,6 +2987,16 @@ dependencies = [ "base64 0.13.0", ] +[[package]] +name = "pem" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" +dependencies = [ + "base64 0.22.1", + "serde", +] + [[package]] name = "pem-rfc7468" version = "0.6.0" @@ -3415,10 +3556,10 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.13", + "http 0.2.7", + "http-body 0.4.5", + "hyper 0.14.19", "hyper-tls", "ipnet", "js-sys", @@ -3511,7 +3652,7 @@ dependencies = [ "decancer", "fcm", "futures", - "indexmap", + "indexmap 1.9.3", "isahc", "iso8601-timestamp 0.2.11", "linkify 0.8.1", @@ -3528,6 +3669,7 @@ dependencies = [ "revolt-permissions", "revolt-presence", "revolt-result", + "revolt_a2", "revolt_okapi", "revolt_optional_struct", "revolt_rocket_okapi", @@ -3593,7 +3735,7 @@ dependencies = [ name = "revolt-models" version = "0.7.12" dependencies = [ - "indexmap", + "indexmap 1.9.3", "iso8601-timestamp 0.2.11", "once_cell", "regex", @@ -3644,6 +3786,30 @@ dependencies = [ "serde_json", ] +[[package]] +name = "revolt_a2" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "466eb5262fcbb26e6e10b8a0acf56eb7cc095008132ff4ebddfc44d8672f8066" +dependencies = [ + "base64 0.21.3", + "erased-serde", + "http 1.1.0", + "http-body-util", + "hyper 1.3.1", + "hyper-rustls", + "hyper-util", + "parking_lot", + "pem 3.0.4", + "ring 0.17.3", + "rustls 0.22.4", + "rustls-pemfile 2.1.2", + "serde", + "serde_json", + "thiserror", + "tokio 1.35.1", +] + [[package]] name = "revolt_okapi" version = "0.9.1" @@ -3716,11 +3882,25 @@ dependencies = [ "libc", "once_cell", "spin 0.5.2", - "untrusted", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babe80d5c16becf6594aa32ad2be8fe08498e7ae60b77de8df700e67f191d7e" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin 0.9.3", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "rmp" version = "0.8.11" @@ -3758,7 +3938,7 @@ dependencies = [ "either", "figment", "futures", - "indexmap", + "indexmap 1.9.3", "log", "memchr", "multer", @@ -3806,7 +3986,7 @@ checksum = "d6aeb6bb9c61e9cd2c00d70ea267bf36f76a4cc615e5908b349c2f9d93999b47" dependencies = [ "devise", "glob", - "indexmap", + "indexmap 1.9.3", "proc-macro2", "quote 1.0.26", "rocket_http", @@ -3819,7 +3999,7 @@ name = "rocket_cors" version = "0.6.0-alpha1" source = "git+https://github.com/lawliet89/rocket_cors?rev=c17e8145baa4790319fdb6a473e465b960f55e7c#c17e8145baa4790319fdb6a473e465b960f55e7c" dependencies = [ - "http", + "http 0.2.7", "log", "regex", "rocket", @@ -3850,9 +4030,9 @@ dependencies = [ "cookie", "either", "futures", - "http", - "hyper", - "indexmap", + "http 0.2.7", + "hyper 0.14.19", + "indexmap 1.9.3", "log", "memchr", "pear", @@ -3973,11 +4153,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" dependencies = [ "log", - "ring", + "ring 0.16.20", "sct", "webpki", ] +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring 0.17.3", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + [[package]] name = "rustls-pemfile" version = "0.3.0" @@ -3987,6 +4181,33 @@ dependencies = [ "base64 0.13.0", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + +[[package]] +name = "rustls-webpki" +version = "0.102.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +dependencies = [ + "ring 0.17.3", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.11" @@ -4016,7 +4237,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1847b767a3d62d95cbf3d8a9f0e421cf57a0d8aa4f411d4b16525afb0284d4ed" dependencies = [ "dyn-clone", - "indexmap", + "indexmap 1.9.3", "schemars_derive", "serde", "serde_json", @@ -4052,8 +4273,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -4279,7 +4500,7 @@ version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ - "indexmap", + "indexmap 1.9.3", "itoa", "ryu", "serde", @@ -4416,9 +4637,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" @@ -4498,9 +4719,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "subtle" -version = "2.4.1" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -4738,11 +4959,22 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.6", "tokio 1.35.1", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", + "tokio 1.35.1", +] + [[package]] name = "tokio-stream" version = "0.1.8" @@ -4804,6 +5036,27 @@ dependencies = [ "sha2", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite 0.2.13", + "tokio 1.35.1", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.1" @@ -4812,11 +5065,10 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.34" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite 0.2.13", "tracing-attributes", @@ -4825,22 +5077,22 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.21" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote 1.0.26", - "syn 1.0.107", + "syn 2.0.15", ] [[package]] name = "tracing-core" -version = "0.1.26" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ - "lazy_static", + "once_cell", "valuable", ] @@ -4943,7 +5195,7 @@ dependencies = [ "base64 0.13.0", "byteorder", "bytes 1.5.0", - "http", + "http 0.2.7", "httparse", "log", "rand 0.8.5", @@ -5094,9 +5346,9 @@ checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "universal-hash" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" +checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" dependencies = [ "generic-array 0.14.5", "subtle", @@ -5108,6 +5360,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "ureq" version = "2.7.1" @@ -5382,7 +5640,7 @@ dependencies = [ "chrono", "ece", "futures-lite", - "http", + "http 0.2.7", "isahc", "jwt-simple", "log", @@ -5409,8 +5667,8 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -5422,6 +5680,15 @@ dependencies = [ "webpki", ] +[[package]] +name = "webpki-roots" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "wepoll-ffi" version = "0.1.2" diff --git a/crates/core/config/Revolt.toml b/crates/core/config/Revolt.toml index 3fa1fe80..611edfa0 100644 --- a/crates/core/config/Revolt.toml +++ b/crates/core/config/Revolt.toml @@ -29,6 +29,11 @@ public_key = "BGcvgR-i2z4IQ5Mw841vJvkLjt8wY-FjmWrw83jOLCY52qcGZS0OF7nfLzuYbjsQIS [api.fcm] api_key = "" +[api.apn] +pkcs8 = "" +key_id = "" +team_id = "" + [api.security] authifier_shield_key = "" voso_legacy_token = "" diff --git a/crates/core/config/src/lib.rs b/crates/core/config/src/lib.rs index 701fc14d..46cfb82e 100644 --- a/crates/core/config/src/lib.rs +++ b/crates/core/config/src/lib.rs @@ -6,6 +6,8 @@ use futures_locks::RwLock; use once_cell::sync::Lazy; use serde::Deserialize; +pub use sentry::capture_error; + #[cfg(not(debug_assertions))] use std::env; @@ -75,6 +77,13 @@ pub struct ApiFcm { pub api_key: String, } +#[derive(Deserialize, Debug, Clone)] +pub struct ApiApn { + pub pkcs8: String, + pub key_id: String, + pub team_id: String, +} + #[derive(Deserialize, Debug, Clone)] pub struct ApiSecurityCaptcha { pub hcaptcha_key: String, @@ -100,6 +109,7 @@ pub struct Api { pub smtp: ApiSmtp, pub vapid: ApiVapid, pub fcm: ApiFcm, + pub apn: ApiApn, pub security: ApiSecurity, pub workers: ApiWorkers, } diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 16615f16..000786a6 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -87,6 +87,9 @@ revolt_rocket_okapi = { version = "0.9.1", optional = true } # Notifications fcm = "0.9.2" web-push = "0.10.0" +revolt_a2 = { version = "0.10.0", default-features = false, features = [ + "ring", +] } # Authifier authifier = { version = "1.0.8" } diff --git a/crates/core/database/src/models/users/ops.rs b/crates/core/database/src/models/users/ops.rs index 62c7fd7c..fe6bb2ff 100644 --- a/crates/core/database/src/models/users/ops.rs +++ b/crates/core/database/src/models/users/ops.rs @@ -58,4 +58,7 @@ pub trait AbstractUsers: Sync + Send { /// Delete a user by their id async fn delete_user(&self, id: &str) -> Result<()>; + + /// Remove push subscription for a session by session id (TODO: remove) + async fn remove_push_subscription_by_session_id(&self, session_id: &str) -> Result<()>; } diff --git a/crates/core/database/src/models/users/ops/mongodb.rs b/crates/core/database/src/models/users/ops/mongodb.rs index 4f812a15..8e132eb1 100644 --- a/crates/core/database/src/models/users/ops/mongodb.rs +++ b/crates/core/database/src/models/users/ops/mongodb.rs @@ -316,6 +316,25 @@ impl AbstractUsers for MongoDb { async fn delete_user(&self, id: &str) -> Result<()> { query!(self, delete_one_by_id, COL, id).map(|_| ()) } + + /// Remove push subscription for a session by session id (TODO: remove) + async fn remove_push_subscription_by_session_id(&self, session_id: &str) -> Result<()> { + self.col::("sessions") + .update_one( + doc! { + "_id": session_id + }, + doc! { + "$unset": { + "subscription": 1 + } + }, + None, + ) + .await + .map(|_| ()) + .map_err(|_| create_database_error!("update_one", COL)) + } } impl IntoDocumentPath for FieldsUser { diff --git a/crates/core/database/src/models/users/ops/reference.rs b/crates/core/database/src/models/users/ops/reference.rs index 47028276..50349080 100644 --- a/crates/core/database/src/models/users/ops/reference.rs +++ b/crates/core/database/src/models/users/ops/reference.rs @@ -163,4 +163,9 @@ impl AbstractUsers for ReferenceDb { Err(create_error!(NotFound)) } } + + /// Remove push subscription for a session by session id (TODO: remove) + async fn remove_push_subscription_by_session_id(&self, _session_id: &str) -> Result<()> { + todo!() + } } diff --git a/crates/core/database/src/tasks/apple_notifications.rs b/crates/core/database/src/tasks/apple_notifications.rs new file mode 100644 index 00000000..cbf9a634 --- /dev/null +++ b/crates/core/database/src/tasks/apple_notifications.rs @@ -0,0 +1,113 @@ +use std::io::Cursor; + +use base64::{ + engine::{self}, + Engine as _, +}; +use deadqueue::limited::Queue; +use once_cell::sync::Lazy; +use revolt_a2::{Client, ClientConfig, DefaultNotificationBuilder}; +use revolt_a2::{Error, ErrorBody, ErrorReason, NotificationBuilder, Response}; +use revolt_config::config; +use revolt_models::v0::PushNotification; + +use crate::Database; + +/// Task information +#[derive(Debug)] +pub struct ApnTask { + /// Session Id + session_id: String, + + /// Device token + device_token: String, + + /// Title + title: String, + + /// Body + body: String, + + /// Thread Id + thread_id: String, +} + +impl ApnTask { + pub fn from_notification( + session_id: String, + device_token: String, + notification: &PushNotification, + ) -> ApnTask { + ApnTask { + session_id, + device_token, + title: notification.author.to_string(), + body: notification.body.to_string(), + thread_id: notification.tag.to_string(), + } + } +} + +static Q: Lazy> = Lazy::new(|| Queue::new(10_000)); + +/// Queue a new task for a worker +pub async fn queue(task: ApnTask) { + Q.try_push(task).ok(); + info!("Queue is using {} slots from {}.", Q.len(), Q.capacity()); +} + +/// Start a new worker +pub async fn worker(db: Database) { + let config = config().await; + if config.api.apn.pkcs8.is_empty() + || config.api.apn.key_id.is_empty() + || config.api.apn.team_id.is_empty() + { + eprintln!("Missing APN keys."); + return; + } + + let pkcs8 = engine::general_purpose::STANDARD + .decode(config.api.apn.pkcs8) + .expect("valid `pcks8`"); + + let client = Client::token( + &mut Cursor::new(pkcs8), + config.api.apn.key_id, + config.api.apn.team_id, + ClientConfig::default(), + ) + .expect("could not create APN client"); + + loop { + let task = Q.pop().await; + let payload = DefaultNotificationBuilder::new() + .set_title(&task.title) + .set_body(&task.body) + .set_thread_id(&task.thread_id) + .build(&task.device_token, Default::default()); + + if let Err(err) = client.send(payload).await { + match err { + Error::ResponseError(Response { + error: + Some(ErrorBody { + reason: ErrorReason::BadDeviceToken | ErrorReason::Unregistered, + .. + }), + .. + }) => { + if let Err(err) = db + .remove_push_subscription_by_session_id(&task.session_id) + .await + { + revolt_config::capture_error(&err); + } + } + err => { + revolt_config::capture_error(&err); + } + } + } + } +} diff --git a/crates/core/database/src/tasks/mod.rs b/crates/core/database/src/tasks/mod.rs index 12a5d122..24a090ed 100644 --- a/crates/core/database/src/tasks/mod.rs +++ b/crates/core/database/src/tasks/mod.rs @@ -8,12 +8,15 @@ use std::time::Instant; const WORKER_COUNT: usize = 5; pub mod ack; +pub mod apple_notifications; pub mod last_message_id; pub mod process_embeds; pub mod web_push; /// Spawn background workers pub async fn start_workers(db: Database, authifier_db: authifier::Database) { + task::spawn(apple_notifications::worker(db.clone())); + for _ in 0..WORKER_COUNT { task::spawn(ack::worker(db.clone())); task::spawn(last_message_id::worker(db.clone())); diff --git a/crates/core/database/src/tasks/web_push.rs b/crates/core/database/src/tasks/web_push.rs index 8a055afe..efe5fd7e 100644 --- a/crates/core/database/src/tasks/web_push.rs +++ b/crates/core/database/src/tasks/web_push.rs @@ -6,6 +6,7 @@ use base64::{ Engine as _, }; use deadqueue::limited::Queue; +use fcm::FcmError; use once_cell::sync::Lazy; use revolt_config::config; use revolt_models::v0::PushNotification; @@ -16,6 +17,8 @@ use web_push::{ WebPushClient, WebPushMessageBuilder, }; +use super::apple_notifications; + /// Task information #[derive(Debug)] struct PushTask { @@ -101,6 +104,15 @@ pub async fn worker(db: Database) { } else { info!("No FCM token was specified!"); } + } else if sub.endpoint == "apn" { + apple_notifications::queue( + apple_notifications::ApnTask::from_notification( + session.id, + sub.auth, + &task.payload, + ), + ) + .await; } else { // Use Web Push Standard let subscription = SubscriptionInfo { From 0954d71f516cc10004bf0c8b3eb527c5e8518982 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 29 Jun 2024 17:36:54 +0100 Subject: [PATCH 056/184] chore: bump version to 0.7.13 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8623be67..be82b60a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3593,7 +3593,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.12" +version = "0.7.13" dependencies = [ "async-channel 2.3.1", "async-std", @@ -3623,7 +3623,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.12" +version = "0.7.13" dependencies = [ "async-std", "cached", @@ -3639,7 +3639,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.12" +version = "0.7.13" dependencies = [ "async-lock", "async-recursion", @@ -3686,7 +3686,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.12" +version = "0.7.13" dependencies = [ "async-channel 1.6.1", "async-std", @@ -3733,7 +3733,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.12" +version = "0.7.13" dependencies = [ "indexmap 1.9.3", "iso8601-timestamp 0.2.11", @@ -3750,7 +3750,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.12" +version = "0.7.13" dependencies = [ "async-std", "async-trait", @@ -3765,7 +3765,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.12" +version = "0.7.13" dependencies = [ "async-std", "log", @@ -3776,7 +3776,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.12" +version = "0.7.13" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 46a53ff8..574a243a 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.12" +version = "0.7.13" license = "AGPL-3.0-or-later" edition = "2021" @@ -41,7 +41,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.12", path = "../core/permissions" } +revolt-permissions = { version = "0.7.13", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index 5cc29902..b7639482 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.12" +version = "0.7.13" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 000786a6..43bad241 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.12" +version = "0.7.13" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.12", path = "../config" } -revolt-result = { version = "0.7.12", path = "../result" } -revolt-models = { version = "0.7.12", path = "../models", features = [ +revolt-config = { version = "0.7.13", path = "../config" } +revolt-result = { version = "0.7.13", path = "../result" } +revolt-models = { version = "0.7.13", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.12", path = "../presence" } -revolt-permissions = { version = "0.7.12", path = "../permissions", features = [ +revolt-presence = { version = "0.7.13", path = "../presence" } +revolt-permissions = { version = "0.7.13", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index b0e9d31c..17c02575 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.12" +version = "0.7.13" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.12", path = "../config" } -revolt-permissions = { version = "0.7.12", path = "../permissions" } +revolt-config = { version = "0.7.13", path = "../config" } +revolt-permissions = { version = "0.7.13", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index 1c7f8ccf..c378f789 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.12" +version = "0.7.13" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.12", path = "../result" } +revolt-result = { version = "0.7.13", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index d6b5cd51..c9cc676d 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.12" +version = "0.7.13" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index bef4004e..f083deed 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.12" +version = "0.7.13" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 3af0bc19..83420a58 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.12" +version = "0.7.13" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From f9d9059e7346801ca8a4cbe2e2d3bc6fa48d85e0 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 29 Jun 2024 18:20:01 +0100 Subject: [PATCH 057/184] fix: bot model test did not destruct tuple --- README.md | 2 +- crates/core/database/src/models/bots/model.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c47a625a..b391cc46 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ Now run tests for whichever database: ```sh TEST_DB=REFERENCE cargo nextest run -TEST_DB=MONGOBD cargo nextest run +TEST_DB=MONGODB cargo nextest run ``` ## License diff --git a/crates/core/database/src/models/bots/model.rs b/crates/core/database/src/models/bots/model.rs index 922d574d..0a796049 100644 --- a/crates/core/database/src/models/bots/model.rs +++ b/crates/core/database/src/models/bots/model.rs @@ -172,7 +172,7 @@ mod tests { .await .unwrap(); - let bot = Bot::create( + let (bot, _) = Bot::create( &db, "Bot Name".to_string(), &owner, From 78cd89ec32e0caac580e9bb3e313fa4de1010e10 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 10 Jul 2024 14:04:17 +0100 Subject: [PATCH 058/184] fix: execute query to remove users from groups --- crates/core/database/src/models/channels/model.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/core/database/src/models/channels/model.rs b/crates/core/database/src/models/channels/model.rs index d48d7aad..082acaa9 100644 --- a/crates/core/database/src/models/channels/model.rs +++ b/crates/core/database/src/models/channels/model.rs @@ -702,6 +702,8 @@ impl Channel { } } + db.remove_user_from_group(id, &user.id).await?; + EventV1::ChannelGroupLeave { id: id.to_string(), user: user.id.to_string(), From e290d168acde6ac28a951bc951b356586a615148 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 10 Jul 2024 14:12:53 +0100 Subject: [PATCH 059/184] feat: add `reason` to ServerMemberLeave event closes #314 --- crates/bonfire/src/events/impl.rs | 2 +- crates/core/database/src/events/client.rs | 10 +++++++--- .../core/database/src/models/server_members/model.rs | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/bonfire/src/events/impl.rs b/crates/bonfire/src/events/impl.rs index 325267f4..f574e578 100644 --- a/crates/bonfire/src/events/impl.rs +++ b/crates/bonfire/src/events/impl.rs @@ -443,7 +443,7 @@ impl State { EventV1::ServerMemberJoin { .. } => { // We will always receive ServerCreate when joining a new server. } - EventV1::ServerMemberLeave { id, user } => { + EventV1::ServerMemberLeave { id, user, .. } => { if user == &self.cache.user_id { self.remove_subscription(id).await; diff --git a/crates/core/database/src/events/client.rs b/crates/core/database/src/events/client.rs index dd20dd4f..74338bfe 100644 --- a/crates/core/database/src/events/client.rs +++ b/crates/core/database/src/events/client.rs @@ -4,8 +4,8 @@ use serde::{Deserialize, Serialize}; use revolt_models::v0::{ AppendMessage, Channel, Emoji, FieldsChannel, FieldsMember, FieldsRole, FieldsServer, FieldsUser, FieldsWebhook, Member, MemberCompositeKey, Message, PartialChannel, PartialMember, - PartialMessage, PartialRole, PartialServer, PartialUser, PartialWebhook, Report, Server, User, - UserSettings, Webhook, + PartialMessage, PartialRole, PartialServer, PartialUser, PartialWebhook, RemovalIntention, + Report, Server, User, UserSettings, Webhook, }; use revolt_result::Error; @@ -136,7 +136,11 @@ pub enum EventV1 { ServerMemberJoin { id: String, user: String }, /// User left server - ServerMemberLeave { id: String, user: String }, + ServerMemberLeave { + id: String, + user: String, + reason: RemovalIntention, + }, /// Server role created or updated ServerRoleUpdate { diff --git a/crates/core/database/src/models/server_members/model.rs b/crates/core/database/src/models/server_members/model.rs index 2a38b8bb..0ca09ed4 100644 --- a/crates/core/database/src/models/server_members/model.rs +++ b/crates/core/database/src/models/server_members/model.rs @@ -229,6 +229,7 @@ impl Member { EventV1::ServerMemberLeave { id: self.id.server.to_string(), user: self.id.user.to_string(), + reason: intention.clone().into(), } .p(self.id.server.to_string()) .await; From 78b9fa168bf5970c1b90e8091c94c35b06fd84ad Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 10 Jul 2024 14:17:31 +0100 Subject: [PATCH 060/184] chore: bump version to 0.7.14 --- Cargo.lock | 16 ++++++++-------- crates/bonfire/Cargo.toml | 4 ++-- crates/core/config/Cargo.toml | 2 +- crates/core/database/Cargo.toml | 12 ++++++------ crates/core/models/Cargo.toml | 6 +++--- crates/core/permissions/Cargo.toml | 4 ++-- crates/core/presence/Cargo.toml | 2 +- crates/core/result/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be82b60a..8d36ddc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3593,7 +3593,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.7.13" +version = "0.7.14" dependencies = [ "async-channel 2.3.1", "async-std", @@ -3623,7 +3623,7 @@ dependencies = [ [[package]] name = "revolt-config" -version = "0.7.13" +version = "0.7.14" dependencies = [ "async-std", "cached", @@ -3639,7 +3639,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.7.13" +version = "0.7.14" dependencies = [ "async-lock", "async-recursion", @@ -3686,7 +3686,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.7.13" +version = "0.7.14" dependencies = [ "async-channel 1.6.1", "async-std", @@ -3733,7 +3733,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.7.13" +version = "0.7.14" dependencies = [ "indexmap 1.9.3", "iso8601-timestamp 0.2.11", @@ -3750,7 +3750,7 @@ dependencies = [ [[package]] name = "revolt-permissions" -version = "0.7.13" +version = "0.7.14" dependencies = [ "async-std", "async-trait", @@ -3765,7 +3765,7 @@ dependencies = [ [[package]] name = "revolt-presence" -version = "0.7.13" +version = "0.7.14" dependencies = [ "async-std", "log", @@ -3776,7 +3776,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.7.13" +version = "0.7.14" dependencies = [ "revolt_okapi", "revolt_rocket_okapi", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 574a243a..76ff8823 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.7.13" +version = "0.7.14" license = "AGPL-3.0-or-later" edition = "2021" @@ -41,7 +41,7 @@ revolt-result = { path = "../core/result" } revolt-models = { path = "../core/models" } revolt-config = { path = "../core/config" } revolt-database = { path = "../core/database" } -revolt-permissions = { version = "0.7.13", path = "../core/permissions" } +revolt-permissions = { version = "0.7.14", path = "../core/permissions" } revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] } # redis diff --git a/crates/core/config/Cargo.toml b/crates/core/config/Cargo.toml index b7639482..94cfbac4 100644 --- a/crates/core/config/Cargo.toml +++ b/crates/core/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-config" -version = "0.7.13" +version = "0.7.14" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 43bad241..d69cbfa0 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.7.13" +version = "0.7.14" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] @@ -23,13 +23,13 @@ default = ["mongodb", "async-std-runtime", "tasks"] [dependencies] # Core -revolt-config = { version = "0.7.13", path = "../config" } -revolt-result = { version = "0.7.13", path = "../result" } -revolt-models = { version = "0.7.13", path = "../models", features = [ +revolt-config = { version = "0.7.14", path = "../config" } +revolt-result = { version = "0.7.14", path = "../result" } +revolt-models = { version = "0.7.14", path = "../models", features = [ "validator", ] } -revolt-presence = { version = "0.7.13", path = "../presence" } -revolt-permissions = { version = "0.7.13", path = "../permissions", features = [ +revolt-presence = { version = "0.7.14", path = "../presence" } +revolt-permissions = { version = "0.7.14", path = "../permissions", features = [ "serde", "bson", ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 17c02575..541ffb46 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.7.13" +version = "0.7.14" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -19,8 +19,8 @@ default = ["serde", "partials", "rocket"] [dependencies] # Core -revolt-config = { version = "0.7.13", path = "../config" } -revolt-permissions = { version = "0.7.13", path = "../permissions" } +revolt-config = { version = "0.7.14", path = "../config" } +revolt-permissions = { version = "0.7.14", path = "../permissions" } # Utility regex = "1" diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index c378f789..5097796d 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.7.13" +version = "0.7.14" edition = "2021" license = "MIT" authors = ["Paul Makles "] @@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] } [dependencies] # Core -revolt-result = { version = "0.7.13", path = "../result" } +revolt-result = { version = "0.7.14", path = "../result" } # Utility auto_ops = "0.3.0" diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index c9cc676d..7103bfe2 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-presence" -version = "0.7.13" +version = "0.7.14" edition = "2021" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index f083deed..5a629796 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.7.13" +version = "0.7.14" edition = "2021" license = "MIT" authors = ["Paul Makles "] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 83420a58..eb97eff7 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.7.13" +version = "0.7.14" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" From 27f15f7b02ef6524789985983bc95017ced318d8 Mon Sep 17 00:00:00 2001 From: IAmTomahawkx Date: Fri, 5 Jul 2024 20:43:08 -0700 Subject: [PATCH 061/184] change apns to use sandbox, and (somewhat) provide a custom payload Signed-off-by: IAmTomahawkx --- crates/core/config/Revolt.toml | 1 + crates/core/config/src/lib.rs | 1 + .../database/src/tasks/apple_notifications.rs | 56 +++++++++++++++++-- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/crates/core/config/Revolt.toml b/crates/core/config/Revolt.toml index 611edfa0..64a5fe92 100644 --- a/crates/core/config/Revolt.toml +++ b/crates/core/config/Revolt.toml @@ -30,6 +30,7 @@ public_key = "BGcvgR-i2z4IQ5Mw841vJvkLjt8wY-FjmWrw83jOLCY52qcGZS0OF7nfLzuYbjsQIS api_key = "" [api.apn] +sandbox = false pkcs8 = "" key_id = "" team_id = "" diff --git a/crates/core/config/src/lib.rs b/crates/core/config/src/lib.rs index 46cfb82e..5c49f238 100644 --- a/crates/core/config/src/lib.rs +++ b/crates/core/config/src/lib.rs @@ -79,6 +79,7 @@ pub struct ApiFcm { #[derive(Deserialize, Debug, Clone)] pub struct ApiApn { + pub sandbox: bool, pub pkcs8: String, pub key_id: String, pub team_id: String, diff --git a/crates/core/database/src/tasks/apple_notifications.rs b/crates/core/database/src/tasks/apple_notifications.rs index cbf9a634..4271afe5 100644 --- a/crates/core/database/src/tasks/apple_notifications.rs +++ b/crates/core/database/src/tasks/apple_notifications.rs @@ -6,13 +6,23 @@ use base64::{ }; use deadqueue::limited::Queue; use once_cell::sync::Lazy; -use revolt_a2::{Client, ClientConfig, DefaultNotificationBuilder}; +use revolt_a2::{Client, ClientConfig, DefaultNotificationBuilder, Endpoint}; use revolt_a2::{Error, ErrorBody, ErrorReason, NotificationBuilder, Response}; use revolt_config::config; -use revolt_models::v0::PushNotification; +use revolt_models::v0::{Message, PushNotification}; use crate::Database; +/// Payload information +#[derive(Debug)] +pub struct ApnCustomPayload { + message: Message, + serverId: String, + authorAvatar: String, + authorDisplayName: String, + channelName: String, +} + /// Task information #[derive(Debug)] pub struct ApnTask { @@ -30,9 +40,24 @@ pub struct ApnTask { /// Thread Id thread_id: String, + + /// Category (informs the client what kind of notification is being sent.) + category: String, + + /// Payload used by the iOS client to modify the notification + payload: ApnCustomPayload, } impl ApnTask { + fn format_title(notification: &PushNotification) -> String { + // ideally this changes depending on context + // in a server, it would look like "Sendername, #channelname in servername" + // in a group, it would look like "Sendername in groupname" + // in a dm it should just be "Sendername". + // not sure how feasible all those are given the PushNotification object as it currently stands. + todo!(); + } + pub fn from_notification( session_id: String, device_token: String, @@ -41,9 +66,17 @@ impl ApnTask { ApnTask { session_id, device_token, - title: notification.author.to_string(), + title: ApnTask::format_title(notification), body: notification.body.to_string(), thread_id: notification.tag.to_string(), + category: "ALERT_MESSAGE".to_string(), + payload: ApnCustomPayload { + message: (), + serverId: (), + authorAvatar: notification.icon.to_string(), + authorDisplayName: notification.author.to_string(), + channelName: (), + }, } } } @@ -67,15 +100,24 @@ pub async fn worker(db: Database) { return; } + let endpoint = if config.api.apn.sandbox { + Endpoint::Sandbox + } else { + Endpoint::Production + }; + let pkcs8 = engine::general_purpose::STANDARD .decode(config.api.apn.pkcs8) .expect("valid `pcks8`"); + let client_config = ClientConfig::default(); + client_config.endpoint = endpoint; + let client = Client::token( &mut Cursor::new(pkcs8), config.api.apn.key_id, config.api.apn.team_id, - ClientConfig::default(), + client_config, ) .expect("could not create APN client"); @@ -85,8 +127,14 @@ pub async fn worker(db: Database) { .set_title(&task.title) .set_body(&task.body) .set_thread_id(&task.thread_id) + .set_category(&task.category) + .set_mutable_content() // allows the service extension to execute .build(&task.device_token, Default::default()); + payload.data = &task.payload; // this looks like a job for someone more rust-inclined than me. -tom + + println!("sending APNS payload: {:?}", payload.data); + if let Err(err) = client.send(payload).await { match err { Error::ResponseError(Response { From 8468ce19cbc3ad8dd35c8792cc117be800e0cc42 Mon Sep 17 00:00:00 2001 From: IAmTomahawkx Date: Thu, 11 Jul 2024 19:44:52 -0700 Subject: [PATCH 062/184] fix apple push notifications Signed-off-by: IAmTomahawkx --- Cargo.lock | 4 +- .../database/src/models/messages/model.rs | 6 +- .../database/src/tasks/apple_notifications.rs | 113 ++++++++++++++---- crates/core/database/src/tasks/web_push.rs | 2 +- crates/core/models/src/v0/embeds.rs | 20 ++-- crates/core/models/src/v0/messages.rs | 26 +++- 6 files changed, 126 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be82b60a..6402cac7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3788,9 +3788,9 @@ dependencies = [ [[package]] name = "revolt_a2" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "466eb5262fcbb26e6e10b8a0acf56eb7cc095008132ff4ebddfc44d8672f8066" +checksum = "edbe1f79cb41271d3cd8f932d75dddeba963c19dc93d1ee6cbe0391b495ab2f5" dependencies = [ "base64 0.21.3", "erased-serde", diff --git a/crates/core/database/src/models/messages/model.rs b/crates/core/database/src/models/messages/model.rs index dd507a30..56b289a3 100644 --- a/crates/core/database/src/models/messages/model.rs +++ b/crates/core/database/src/models/messages/model.rs @@ -444,8 +444,8 @@ impl Message { ) -> Result<()> { self.send_without_notifications( db, - user, - member, + user.clone(), + member.clone(), matches!(channel, Channel::DirectMessage { .. }), generate_embeds, ) @@ -463,7 +463,7 @@ impl Message { } }, PushNotification::from( - self.clone().into_model(None, None), + self.clone().into_model(user, member), Some(author), &channel.id(), ) diff --git a/crates/core/database/src/tasks/apple_notifications.rs b/crates/core/database/src/tasks/apple_notifications.rs index 4271afe5..bde459bb 100644 --- a/crates/core/database/src/tasks/apple_notifications.rs +++ b/crates/core/database/src/tasks/apple_notifications.rs @@ -6,23 +6,53 @@ use base64::{ }; use deadqueue::limited::Queue; use once_cell::sync::Lazy; -use revolt_a2::{Client, ClientConfig, DefaultNotificationBuilder, Endpoint}; -use revolt_a2::{Error, ErrorBody, ErrorReason, NotificationBuilder, Response}; +use revolt_a2::{ + request::{ + notification::{DefaultAlert, NotificationOptions}, + payload::{APSAlert, PayloadLike, APS}, + }, + Client, ClientConfig, DefaultNotificationBuilder, Endpoint, Error, ErrorBody, ErrorReason, + NotificationBuilder, Priority, PushType, Response, +}; use revolt_config::config; use revolt_models::v0::{Message, PushNotification}; use crate::Database; -/// Payload information +/// Payload information, before assembly #[derive(Debug)] -pub struct ApnCustomPayload { +pub struct ApnPayload { message: Message, - serverId: String, + url: String, authorAvatar: String, authorDisplayName: String, channelName: String, } +#[derive(Serialize, Debug)] +struct Payload<'a> { + aps: APS<'a>, + #[serde(skip_serializing)] + options: NotificationOptions<'a>, + #[serde(skip_serializing)] + device_token: &'a str, + + message: Message, + url: String, + authorAvatar: String, + authorDisplayName: String, + channelName: String, +} + +impl<'a> PayloadLike for Payload<'a> { + fn get_device_token(&self) -> &'a str { + self.device_token + } + fn get_options(&self) -> &NotificationOptions { + &self.options + } +} + /// Task information #[derive(Debug)] pub struct ApnTask { @@ -45,7 +75,7 @@ pub struct ApnTask { category: String, /// Payload used by the iOS client to modify the notification - payload: ApnCustomPayload, + custom_payload: ApnPayload, } impl ApnTask { @@ -55,7 +85,10 @@ impl ApnTask { // in a group, it would look like "Sendername in groupname" // in a dm it should just be "Sendername". // not sure how feasible all those are given the PushNotification object as it currently stands. - todo!(); + format!( + "{} in {}", + notification.author, notification.message.channel + ) // TODO: this absolutely needs a channel name } pub fn from_notification( @@ -70,12 +103,12 @@ impl ApnTask { body: notification.body.to_string(), thread_id: notification.tag.to_string(), category: "ALERT_MESSAGE".to_string(), - payload: ApnCustomPayload { - message: (), - serverId: (), - authorAvatar: notification.icon.to_string(), - authorDisplayName: notification.author.to_string(), - channelName: (), + custom_payload: ApnPayload { + message: notification.message.clone(), + url: notification.url.clone(), + authorAvatar: notification.icon.clone(), + authorDisplayName: notification.author.clone(), + channelName: "#fetchchannelnamehere".to_string(), // TODO: get actual channel name }, } } @@ -110,8 +143,7 @@ pub async fn worker(db: Database) { .decode(config.api.apn.pkcs8) .expect("valid `pcks8`"); - let client_config = ClientConfig::default(); - client_config.endpoint = endpoint; + let client_config = ClientConfig::new(endpoint); let client = Client::token( &mut Cursor::new(pkcs8), @@ -121,21 +153,52 @@ pub async fn worker(db: Database) { ) .expect("could not create APN client"); + let payload_options = NotificationOptions { + apns_id: None, + apns_push_type: Some(PushType::Alert), + apns_expiration: None, + apns_priority: Some(Priority::High), + apns_topic: Some("chat.revolt.app"), + apns_collapse_id: None, + }; + loop { let task = Q.pop().await; - let payload = DefaultNotificationBuilder::new() - .set_title(&task.title) - .set_body(&task.body) - .set_thread_id(&task.thread_id) - .set_category(&task.category) - .set_mutable_content() // allows the service extension to execute - .build(&task.device_token, Default::default()); - payload.data = &task.payload; // this looks like a job for someone more rust-inclined than me. -tom + let payload: Payload = Payload { + aps: APS { + alert: Some(APSAlert::Default(DefaultAlert { + title: Some(&task.title), + subtitle: None, + body: Some(&task.body), + title_loc_key: None, + title_loc_args: None, + action_loc_key: None, + loc_key: None, + loc_args: None, + launch_image: None, + })), + badge: Some(1), + sound: None, + thread_id: Some(&task.thread_id), + content_available: Some(1), + category: Some(&task.category), + mutable_content: Some(1), + url_args: None, + }, + device_token: &task.device_token, + options: payload_options.clone(), + message: task.custom_payload.message, + url: task.custom_payload.url, + authorAvatar: task.custom_payload.authorAvatar, + authorDisplayName: task.custom_payload.authorDisplayName, + channelName: task.custom_payload.channelName, + }; - println!("sending APNS payload: {:?}", payload.data); + let resp = client.send(payload).await; + //println!("response from APNS: {:?}", resp); - if let Err(err) = client.send(payload).await { + if let Err(err) = resp { match err { Error::ResponseError(Response { error: diff --git a/crates/core/database/src/tasks/web_push.rs b/crates/core/database/src/tasks/web_push.rs index efe5fd7e..3508b64d 100644 --- a/crates/core/database/src/tasks/web_push.rs +++ b/crates/core/database/src/tasks/web_push.rs @@ -6,7 +6,6 @@ use base64::{ Engine as _, }; use deadqueue::limited::Queue; -use fcm::FcmError; use once_cell::sync::Lazy; use revolt_config::config; use revolt_models::v0::PushNotification; @@ -82,6 +81,7 @@ pub async fn worker(db: Database) { tag, timestamp: _, url: _, + message: _, } = &task.payload; let mut notification = fcm::NotificationBuilder::new(); diff --git a/crates/core/models/src/v0/embeds.rs b/crates/core/models/src/v0/embeds.rs index faeeeb3d..1db81bed 100644 --- a/crates/core/models/src/v0/embeds.rs +++ b/crates/core/models/src/v0/embeds.rs @@ -92,38 +92,38 @@ auto_derived!( pub struct WebsiteMetadata { /// Direct URL to web page #[serde(skip_serializing_if = "Option::is_none")] - url: Option, + pub url: Option, /// Original direct URL #[serde(skip_serializing_if = "Option::is_none")] - original_url: Option, + pub original_url: Option, /// Remote content #[serde(skip_serializing_if = "Option::is_none")] - special: Option, + pub special: Option, /// Title of website #[serde(skip_serializing_if = "Option::is_none")] - title: Option, + pub title: Option, /// Description of website #[serde(skip_serializing_if = "Option::is_none")] - description: Option, + pub description: Option, /// Embedded image #[serde(skip_serializing_if = "Option::is_none")] - image: Option, + pub image: Option, /// Embedded video #[serde(skip_serializing_if = "Option::is_none")] - video: Option