From d74104fae4b13bcdf6256b260cf059291fb67fb6 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 18 Jan 2023 12:35:49 +0000 Subject: [PATCH 1/4] feat: initial ideas for content reporting --- crates/delta/src/routes/mod.rs | 8 +++ crates/delta/src/routes/safety/mod.rs | 11 ++++ .../delta/src/routes/safety/report_content.rs | 52 +++++++++++++++++++ crates/quark/src/web/ratelimiter.rs | 4 ++ 4 files changed, 75 insertions(+) create mode 100644 crates/delta/src/routes/safety/mod.rs create mode 100644 crates/delta/src/routes/safety/report_content.rs diff --git a/crates/delta/src/routes/mod.rs b/crates/delta/src/routes/mod.rs index f50ef91f..f700b23a 100644 --- a/crates/delta/src/routes/mod.rs +++ b/crates/delta/src/routes/mod.rs @@ -10,6 +10,7 @@ mod invites; mod onboard; mod push; mod root; +mod safety; mod servers; mod sync; mod users; @@ -27,6 +28,7 @@ pub fn mount(mut rocket: Rocket) -> Rocket { "/servers" => servers::routes(), "/invites" => invites::routes(), "/custom" => customisation::routes(), + "/safety" => safety::routes(), "/auth/account" => rocket_rauth::routes::account::routes(), "/auth/session" => rocket_rauth::routes::session::routes(), "/auth/mfa" => rocket_rauth::routes::mfa::routes(), @@ -105,6 +107,12 @@ fn custom_openapi_spec() -> OpenApi { "Emojis" ] }, + { + "name": "Platform Moderation", + "tags": [ + "User Safety" + ] + }, { "name": "Authentication", "tags": [ diff --git a/crates/delta/src/routes/safety/mod.rs b/crates/delta/src/routes/safety/mod.rs new file mode 100644 index 00000000..fea6c823 --- /dev/null +++ b/crates/delta/src/routes/safety/mod.rs @@ -0,0 +1,11 @@ +use rocket::Route; +use rocket_okapi::okapi::openapi3::OpenApi; + +mod report_content; + +pub fn routes() -> (Vec, OpenApi) { + openapi_get_routes_spec![ + // Reports + report_content::report_content + ] +} diff --git a/crates/delta/src/routes/safety/report_content.rs b/crates/delta/src/routes/safety/report_content.rs new file mode 100644 index 00000000..f6b848c7 --- /dev/null +++ b/crates/delta/src/routes/safety/report_content.rs @@ -0,0 +1,52 @@ +use revolt_quark::models::User; +use revolt_quark::{Db, Error, Result}; +use serde::Deserialize; +use validator::Validate; + +use rocket::serde::json::Json; + +#[derive(Deserialize, JsonSchema)] +enum UserReportReason { + NoneSpecified, + InappropriateProfile, +} + +// TODO: move me into models +#[derive(Deserialize, JsonSchema)] +pub enum ReportedContent { + User { + id: String, + report_reason: UserReportReason, + }, +} + +/// # Report Data +#[derive(Validate, Deserialize, JsonSchema)] +pub struct DataReportContent { + /// Content being reported + content: ReportedContent, + /// Additional report description + #[validate(length(min = 0, max = 1000))] + #[serde(default)] + additional_context: String, +} + +/// # Report Content +/// +/// Report a piece of content to the moderation team. +#[openapi(tag = "User Safety")] +#[post("/report", data = "")] +pub async fn report_content(db: &Db, user: User, data: Json) -> Result<()> { + let data = data.into_inner(); + data.validate() + .map_err(|error| Error::FailedValidation { error })?; + + // Bots cannot create reports + if user.bot.is_some() { + return Err(Error::IsBot); + } + + // find the content and create the report here + + Ok(()) +} diff --git a/crates/quark/src/web/ratelimiter.rs b/crates/quark/src/web/ratelimiter.rs index f7e0dd62..003881cc 100644 --- a/crates/quark/src/web/ratelimiter.rs +++ b/crates/quark/src/web/ratelimiter.rs @@ -126,6 +126,8 @@ fn resolve_bucket<'r>(request: &'r rocket::Request<'_>) -> (&'r str, Option<&'r } } ("swagger", _) => ("swagger", None), + ("safety", Some("report")) => ("safety_report", Some("report")), + ("safety", _) => ("safety", None), _ => ("any", None), } } else { @@ -145,6 +147,8 @@ fn resolve_bucket_limit(bucket: &str) -> u8 { "auth_delete" => 255, "default_avatar" => 255, "swagger" => 100, + "safety" => 15, + "safety_report" => 3, _ => 20, } } From 49598daf70c896efcf8ddecdda21186e03cd48f7 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Fri, 17 Feb 2023 09:45:34 +0000 Subject: [PATCH 2/4] fix: port user safety API to new codebase --- crates/delta/src/routes/safety/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/delta/src/routes/safety/mod.rs b/crates/delta/src/routes/safety/mod.rs index fea6c823..29102aa6 100644 --- a/crates/delta/src/routes/safety/mod.rs +++ b/crates/delta/src/routes/safety/mod.rs @@ -1,5 +1,5 @@ +use revolt_rocket_okapi::revolt_okapi::openapi3::OpenApi; use rocket::Route; -use rocket_okapi::okapi::openapi3::OpenApi; mod report_content; From 056c0380b24a79c298ee3c0f78d9e9bda96ad483 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 21 Feb 2023 12:38:35 +0100 Subject: [PATCH 3/4] feat(safety): implement report content API --- Cargo.lock | 6 +- crates/bonfire/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- .../delta/src/routes/safety/report_content.rs | 91 +++++++++++++++---- crates/quark/Cargo.toml | 2 +- crates/quark/src/impl/dummy/mod.rs | 5 + crates/quark/src/impl/dummy/safety/report.rs | 12 +++ .../quark/src/impl/dummy/safety/snapshot.rs | 12 +++ crates/quark/src/impl/mongo/mod.rs | 5 + crates/quark/src/impl/mongo/safety/report.rs | 13 +++ .../quark/src/impl/mongo/safety/snapshot.rs | 13 +++ crates/quark/src/models/mod.rs | 8 ++ crates/quark/src/models/safety/report.rs | 85 +++++++++++++++++ crates/quark/src/models/safety/snapshot.rs | 25 +++++ crates/quark/src/traits/mod.rs | 10 ++ crates/quark/src/traits/safety/report.rs | 8 ++ crates/quark/src/traits/safety/snapshot.rs | 8 ++ 17 files changed, 284 insertions(+), 23 deletions(-) create mode 100644 crates/quark/src/impl/dummy/safety/report.rs create mode 100644 crates/quark/src/impl/dummy/safety/snapshot.rs create mode 100644 crates/quark/src/impl/mongo/safety/report.rs create mode 100644 crates/quark/src/impl/mongo/safety/snapshot.rs create mode 100644 crates/quark/src/models/safety/report.rs create mode 100644 crates/quark/src/models/safety/snapshot.rs create mode 100644 crates/quark/src/traits/safety/report.rs create mode 100644 crates/quark/src/traits/safety/snapshot.rs diff --git a/Cargo.lock b/Cargo.lock index 05d72cb0..a1d5ff69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2792,7 +2792,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.5.7" +version = "0.5.8" dependencies = [ "async-std", "async-tungstenite", @@ -2808,7 +2808,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.5.7" +version = "0.5.8" dependencies = [ "async-channel", "async-std", @@ -2848,7 +2848,7 @@ dependencies = [ [[package]] name = "revolt-quark" -version = "0.5.7" +version = "0.5.8" dependencies = [ "async-lock", "async-recursion", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 29cdab6a..9978eaa7 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.5.7" +version = "0.5.8" license = "AGPL-3.0-or-later" edition = "2021" diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 7ae7064b..93ba97d8 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.5.7" +version = "0.5.8" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" diff --git a/crates/delta/src/routes/safety/report_content.rs b/crates/delta/src/routes/safety/report_content.rs index f6b848c7..b8967706 100644 --- a/crates/delta/src/routes/safety/report_content.rs +++ b/crates/delta/src/routes/safety/report_content.rs @@ -1,25 +1,13 @@ -use revolt_quark::models::User; +use revolt_quark::models::report::ReportedContent; +use revolt_quark::models::snapshot::{Snapshot, SnapshotContent}; +use revolt_quark::models::{Report, User}; use revolt_quark::{Db, Error, Result}; use serde::Deserialize; +use ulid::Ulid; use validator::Validate; use rocket::serde::json::Json; -#[derive(Deserialize, JsonSchema)] -enum UserReportReason { - NoneSpecified, - InappropriateProfile, -} - -// TODO: move me into models -#[derive(Deserialize, JsonSchema)] -pub enum ReportedContent { - User { - id: String, - report_reason: UserReportReason, - }, -} - /// # Report Data #[derive(Validate, Deserialize, JsonSchema)] pub struct DataReportContent { @@ -46,7 +34,76 @@ pub async fn report_content(db: &Db, user: User, data: Json) return Err(Error::IsBot); } - // find the content and create the report here + // Find the content and create a snapshot of it + // Also retrieve any references to Files + let (content, files): (SnapshotContent, Vec) = match &data.content { + ReportedContent::Message { id, .. } => { + let message = db.fetch_message(id).await?; + + // Collect message attachments + let files = message + .attachments + .as_ref() + .map(|attachments| attachments.iter().map(|x| x.id.to_string()).collect()) + .unwrap_or_default(); + + (SnapshotContent::Message(message), files) + } + ReportedContent::Server { id, .. } => { + let server = db.fetch_server(id).await?; + + // Collect server's icon and banner + let files = [&server.icon, &server.banner] + .iter() + .filter_map(|x| x.as_ref().map(|x| x.id.to_string())) + .collect(); + + (SnapshotContent::Server(server), files) + } + ReportedContent::User { id, .. } => { + let user = db.fetch_user(id).await?; + + // Collect user's avatar and profile background + let files = [ + user.avatar.as_ref(), + user.profile + .as_ref() + .and_then(|profile| profile.background.as_ref()), + ] + .iter() + .filter_map(|x| x.as_ref().map(|x| x.id.to_string())) + .collect(); + + (SnapshotContent::User(user), files) + } + }; + + // Mark all the attachments as reported + for file in files { + db.mark_attachment_as_reported(&file).await?; + } + + // Generate an id for the report + let id = Ulid::new().to_string(); + + // Save a snapshot of the content + let snapshot = Snapshot { + id: Ulid::new().to_string(), + report_id: id.to_string(), + content, + }; + + db.insert_snapshot(&snapshot).await?; + + // Save the report + let report = Report { + id, + author_id: user.id, + content: data.content, + additional_context: data.additional_context, + }; + + db.insert_report(&report).await?; Ok(()) } diff --git a/crates/quark/Cargo.toml b/crates/quark/Cargo.toml index 0dff3b61..02f98e36 100644 --- a/crates/quark/Cargo.toml +++ b/crates/quark/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-quark" -version = "0.5.7" +version = "0.5.8" license = "AGPL-3.0-or-later" edition = "2021" diff --git a/crates/quark/src/impl/dummy/mod.rs b/crates/quark/src/impl/dummy/mod.rs index cfb9b8ea..46ad4593 100644 --- a/crates/quark/src/impl/dummy/mod.rs +++ b/crates/quark/src/impl/dummy/mod.rs @@ -28,6 +28,11 @@ pub mod users { pub mod user_settings; } +pub mod safety { + pub mod report; + pub mod snapshot; +} + #[derive(Debug, Clone)] pub struct DummyDb; diff --git a/crates/quark/src/impl/dummy/safety/report.rs b/crates/quark/src/impl/dummy/safety/report.rs new file mode 100644 index 00000000..5286976c --- /dev/null +++ b/crates/quark/src/impl/dummy/safety/report.rs @@ -0,0 +1,12 @@ +use crate::models::Report; +use crate::{AbstractReport, Result}; + +use super::super::DummyDb; + +#[async_trait] +impl AbstractReport for DummyDb { + async fn insert_report(&self, report: &Report) -> Result<()> { + info!("Insert {:?}", report); + Ok(()) + } +} diff --git a/crates/quark/src/impl/dummy/safety/snapshot.rs b/crates/quark/src/impl/dummy/safety/snapshot.rs new file mode 100644 index 00000000..8d384059 --- /dev/null +++ b/crates/quark/src/impl/dummy/safety/snapshot.rs @@ -0,0 +1,12 @@ +use crate::models::Snapshot; +use crate::{AbstractSnapshot, Result}; + +use super::super::DummyDb; + +#[async_trait] +impl AbstractSnapshot for DummyDb { + async fn insert_snapshot(&self, snapshot: &Snapshot) -> Result<()> { + info!("Insert {:?}", snapshot); + Ok(()) + } +} diff --git a/crates/quark/src/impl/mongo/mod.rs b/crates/quark/src/impl/mongo/mod.rs index fc4d7a2e..d0639dbb 100644 --- a/crates/quark/src/impl/mongo/mod.rs +++ b/crates/quark/src/impl/mongo/mod.rs @@ -39,6 +39,11 @@ pub mod users { pub mod user_settings; } +pub mod safety { + pub mod report; + pub mod snapshot; +} + #[derive(Debug, Clone)] pub struct MongoDb(pub mongodb::Client); diff --git a/crates/quark/src/impl/mongo/safety/report.rs b/crates/quark/src/impl/mongo/safety/report.rs new file mode 100644 index 00000000..6b364d11 --- /dev/null +++ b/crates/quark/src/impl/mongo/safety/report.rs @@ -0,0 +1,13 @@ +use crate::models::Report; +use crate::{AbstractReport, Result}; + +use super::super::MongoDb; + +static COL: &str = "safety_reports"; + +#[async_trait] +impl AbstractReport for MongoDb { + async fn insert_report(&self, report: &Report) -> Result<()> { + self.insert_one(COL, report).await.map(|_| ()) + } +} diff --git a/crates/quark/src/impl/mongo/safety/snapshot.rs b/crates/quark/src/impl/mongo/safety/snapshot.rs new file mode 100644 index 00000000..d4dd29f6 --- /dev/null +++ b/crates/quark/src/impl/mongo/safety/snapshot.rs @@ -0,0 +1,13 @@ +use crate::models::Snapshot; +use crate::{AbstractSnapshot, Result}; + +use super::super::MongoDb; + +static COL: &str = "safety_snapshots"; + +#[async_trait] +impl AbstractSnapshot for MongoDb { + async fn insert_snapshot(&self, snapshot: &Snapshot) -> Result<()> { + self.insert_one(COL, snapshot).await.map(|_| ()) + } +} diff --git a/crates/quark/src/models/mod.rs b/crates/quark/src/models/mod.rs index 20bf360e..4be15903 100644 --- a/crates/quark/src/models/mod.rs +++ b/crates/quark/src/models/mod.rs @@ -27,9 +27,15 @@ mod users { pub mod user_settings; } +mod safety { + pub mod report; + pub mod snapshot; +} + pub use admin::*; pub use channels::*; pub use media::*; +pub use safety::*; pub use servers::*; pub use users::*; @@ -41,9 +47,11 @@ pub use channel_unread::ChannelUnread; pub use emoji::Emoji; pub use message::Message; pub use migrations::MigrationInfo; +pub use report::Report; pub use server::Server; pub use server_ban::ServerBan; pub use server_member::Member; pub use simple::SimpleModel; +pub use snapshot::Snapshot; pub use user::User; pub use user_settings::UserSettings; diff --git a/crates/quark/src/models/safety/report.rs b/crates/quark/src/models/safety/report.rs new file mode 100644 index 00000000..069f3036 --- /dev/null +++ b/crates/quark/src/models/safety/report.rs @@ -0,0 +1,85 @@ +use serde::{Deserialize, Serialize}; + +/// Reason for reporting content (message or server) +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +pub enum ContentReportReason { + /// No reason has been specified + NoneSpecified, + + /// Blatantly illegal content + Illegal, + + /// Content that promotes harm to others / self + PromotesHarm, + + /// Spam or platform abuse + SpamAbuse, + + /// Distribution of malware + Malware, + + /// Harassment or abuse targeted at another user + Harassment, +} + +/// Reason for reporting a user +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +pub enum UserReportReason { + /// No reason has been specified + NoneSpecified, + + /// User is sending spam or otherwise abusing the platform + SpamAbuse, + + /// User's profile contains inappropriate content for a general audience + InappropriateProfile, + + /// User is impersonating another user + Impersonation, + + /// User is evading a ban + BanEvasion, + + /// User is not of minimum age to use the platform + Underage, +} + +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +#[serde(tag = "type")] +pub enum ReportedContent { + /// Report a message + Message { + /// ID of the message + id: String, + /// Reason for reporting message + report_reason: ContentReportReason, + }, + /// Report a server + Server { + /// ID of the server + id: String, + /// Reason for reporting server + report_reason: ContentReportReason, + }, + /// Report a user + User { + /// ID of the user + id: String, + /// Reason for reporting a user + report_reason: UserReportReason, + }, +} + +/// User-generated platform moderation report. +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +pub struct Report { + /// Unique Id + #[serde(rename = "_id")] + pub id: String, + /// Id of the user creating this report + pub author_id: String, + /// Reported content + pub content: ReportedContent, + /// Additional report context + pub additional_context: String, +} diff --git a/crates/quark/src/models/safety/snapshot.rs b/crates/quark/src/models/safety/snapshot.rs new file mode 100644 index 00000000..376434f4 --- /dev/null +++ b/crates/quark/src/models/safety/snapshot.rs @@ -0,0 +1,25 @@ +use serde::{Deserialize, Serialize}; + +use crate::models::{Message, Server, User}; + +/// Enum to map into different models +/// that can be saved in a snapshot +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +#[serde(tag = "_type")] +pub enum SnapshotContent { + Message(Message), + Server(Server), + User(User), +} + +/// Snapshot of some content +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +pub struct Snapshot { + /// Unique Id + #[serde(rename = "_id")] + pub id: String, + /// Report parent Id + pub report_id: String, + /// Snapshot of content + pub content: SnapshotContent, +} diff --git a/crates/quark/src/traits/mod.rs b/crates/quark/src/traits/mod.rs index 93764ed8..24855f8c 100644 --- a/crates/quark/src/traits/mod.rs +++ b/crates/quark/src/traits/mod.rs @@ -26,6 +26,11 @@ mod users { pub mod user_settings; } +mod safety { + pub mod report; + pub mod snapshot; +} + pub use admin::migrations::AbstractMigrations; pub use media::attachment::AbstractAttachment; @@ -44,6 +49,9 @@ pub use users::bot::AbstractBot; pub use users::user::AbstractUser; pub use users::user_settings::AbstractUserSettings; +pub use safety::report::AbstractReport; +pub use safety::snapshot::AbstractSnapshot; + pub trait AbstractDatabase: Sync + Send @@ -60,5 +68,7 @@ pub trait AbstractDatabase: + AbstractBot + AbstractUser + AbstractUserSettings + + AbstractReport + + AbstractSnapshot { } diff --git a/crates/quark/src/traits/safety/report.rs b/crates/quark/src/traits/safety/report.rs new file mode 100644 index 00000000..f5a15e39 --- /dev/null +++ b/crates/quark/src/traits/safety/report.rs @@ -0,0 +1,8 @@ +use crate::models::Report; +use crate::Result; + +#[async_trait] +pub trait AbstractReport: Sync + Send { + /// Insert a new report into the database + async fn insert_report(&self, report: &Report) -> Result<()>; +} diff --git a/crates/quark/src/traits/safety/snapshot.rs b/crates/quark/src/traits/safety/snapshot.rs new file mode 100644 index 00000000..9f6ac84e --- /dev/null +++ b/crates/quark/src/traits/safety/snapshot.rs @@ -0,0 +1,8 @@ +use crate::models::Snapshot; +use crate::Result; + +#[async_trait] +pub trait AbstractSnapshot: Sync + Send { + /// Insert a new snapshot into the database + async fn insert_snapshot(&self, snapshot: &Snapshot) -> Result<()>; +} From 0c072b01d7ef78937ac98701fae22244400fe5b8 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Tue, 21 Feb 2023 12:42:08 +0100 Subject: [PATCH 4/4] fix(safety): prevent reporting own content or self --- .../delta/src/routes/safety/report_content.rs | 24 +++++++++++++++---- crates/quark/src/util/result.rs | 5 ++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/crates/delta/src/routes/safety/report_content.rs b/crates/delta/src/routes/safety/report_content.rs index b8967706..25e97833 100644 --- a/crates/delta/src/routes/safety/report_content.rs +++ b/crates/delta/src/routes/safety/report_content.rs @@ -40,6 +40,11 @@ pub async fn report_content(db: &Db, user: User, data: Json) ReportedContent::Message { id, .. } => { let message = db.fetch_message(id).await?; + // Users cannot report themselves + if message.author == user.id { + return Err(Error::CannotReportYourself); + } + // Collect message attachments let files = message .attachments @@ -52,6 +57,11 @@ pub async fn report_content(db: &Db, user: User, data: Json) ReportedContent::Server { id, .. } => { let server = db.fetch_server(id).await?; + // Users cannot report their own server + if server.owner == user.id { + return Err(Error::CannotReportYourself); + } + // Collect server's icon and banner let files = [&server.icon, &server.banner] .iter() @@ -61,12 +71,18 @@ pub async fn report_content(db: &Db, user: User, data: Json) (SnapshotContent::Server(server), files) } ReportedContent::User { id, .. } => { - let user = db.fetch_user(id).await?; + let reported_user = db.fetch_user(id).await?; + + // Users cannot report themselves + if reported_user.id == user.id { + return Err(Error::CannotReportYourself); + } // Collect user's avatar and profile background let files = [ - user.avatar.as_ref(), - user.profile + reported_user.avatar.as_ref(), + reported_user + .profile .as_ref() .and_then(|profile| profile.background.as_ref()), ] @@ -74,7 +90,7 @@ pub async fn report_content(db: &Db, user: User, data: Json) .filter_map(|x| x.as_ref().map(|x| x.id.to_string())) .collect(); - (SnapshotContent::User(user), files) + (SnapshotContent::User(reported_user), files) } }; diff --git a/crates/quark/src/util/result.rs b/crates/quark/src/util/result.rs index 628c8303..6efbd01f 100644 --- a/crates/quark/src/util/result.rs +++ b/crates/quark/src/util/result.rs @@ -64,6 +64,9 @@ pub enum Error { IsBot, BotIsPrivate, + // ? User safety related errors + CannotReportYourself, + // ? Permission errors MissingPermission { permission: Permission, @@ -166,6 +169,8 @@ impl<'r> Responder<'r, 'static> for Error { Error::IsBot => Status::BadRequest, Error::BotIsPrivate => Status::Forbidden, + Error::CannotReportYourself => Status::BadRequest, + Error::MissingPermission { .. } => Status::Forbidden, Error::MissingUserPermission { .. } => Status::Forbidden, Error::NotElevated => Status::Forbidden,