merge: pull request #223 from revoltchat/insert/feat/user-safety-api

Cargo.lock

@@ -2792,7 +2792,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-bonfire"
-version = "0.5.7"
+version = "0.5.8"
 dependencies = [
  "async-std",
  "async-tungstenite",
@@ -2808,7 +2808,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-delta"
-version = "0.5.7"
+version = "0.5.8"
 dependencies = [
  "async-channel",
  "async-std",
@@ -2848,7 +2848,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-quark"
-version = "0.5.7"
+version = "0.5.8"
 dependencies = [
  "async-lock",
  "async-recursion",

crates/bonfire/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-bonfire"
-version = "0.5.7"
+version = "0.5.8"
 license = "AGPL-3.0-or-later"
 edition = "2021"
 

crates/delta/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-delta"
-version = "0.5.7"
+version = "0.5.8"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <paulmakles@gmail.com>"]
 edition = "2018"

crates/delta/src/routes/mod.rs

@@ -10,6 +10,7 @@ mod invites;
 mod onboard;
 mod push;
 mod root;
+mod safety;
 mod servers;
 mod sync;
 mod users;
@@ -27,6 +28,7 @@ pub fn mount(mut rocket: Rocket<Build>) -> Rocket<Build> {
         "/servers" => servers::routes(),
         "/invites" => invites::routes(),
         "/custom" => customisation::routes(),
+        "/safety" => safety::routes(),
         "/auth/account" => rocket_authifier::routes::account::routes(),
         "/auth/session" => rocket_authifier::routes::session::routes(),
         "/auth/mfa" => rocket_authifier::routes::mfa::routes(),
@@ -105,6 +107,12 @@ fn custom_openapi_spec() -> OpenApi {
               "Emojis"
             ]
           },
+          {
+            "name": "Platform Moderation",
+            "tags": [
+              "User Safety"
+            ]
+          },
           {
             "name": "Authentication",
             "tags": [

crates/delta/src/routes/safety/mod.rs

@@ -0,0 +1,11 @@
+use revolt_rocket_okapi::revolt_okapi::openapi3::OpenApi;
+use rocket::Route;
+
+mod report_content;
+
+pub fn routes() -> (Vec<Route>, OpenApi) {
+    openapi_get_routes_spec![
+        // Reports
+        report_content::report_content
+    ]
+}

crates/delta/src/routes/safety/report_content.rs

@@ -0,0 +1,125 @@
+use revolt_quark::models::report::ReportedContent;
+use revolt_quark::models::snapshot::{Snapshot, SnapshotContent};
+use revolt_quark::models::{Report, User};
+use revolt_quark::{Db, Error, Result};
+use serde::Deserialize;
+use ulid::Ulid;
+use validator::Validate;
+
+use rocket::serde::json::Json;
+
+/// # Report Data
+#[derive(Validate, Deserialize, JsonSchema)]
+pub struct DataReportContent {
+    /// Content being reported
+    content: ReportedContent,
+    /// Additional report description
+    #[validate(length(min = 0, max = 1000))]
+    #[serde(default)]
+    additional_context: String,
+}
+
+/// # Report Content
+///
+/// Report a piece of content to the moderation team.
+#[openapi(tag = "User Safety")]
+#[post("/report", data = "<data>")]
+pub async fn report_content(db: &Db, user: User, data: Json<DataReportContent>) -> Result<()> {
+    let data = data.into_inner();
+    data.validate()
+        .map_err(|error| Error::FailedValidation { error })?;
+
+    // Bots cannot create reports
+    if user.bot.is_some() {
+        return Err(Error::IsBot);
+    }
+
+    // Find the content and create a snapshot of it
+    // Also retrieve any references to Files
+    let (content, files): (SnapshotContent, Vec<String>) = match &data.content {
+        ReportedContent::Message { id, .. } => {
+            let message = db.fetch_message(id).await?;
+
+            // Users cannot report themselves
+            if message.author == user.id {
+                return Err(Error::CannotReportYourself);
+            }
+
+            // Collect message attachments
+            let files = message
+                .attachments
+                .as_ref()
+                .map(|attachments| attachments.iter().map(|x| x.id.to_string()).collect())
+                .unwrap_or_default();
+
+            (SnapshotContent::Message(message), files)
+        }
+        ReportedContent::Server { id, .. } => {
+            let server = db.fetch_server(id).await?;
+
+            // Users cannot report their own server
+            if server.owner == user.id {
+                return Err(Error::CannotReportYourself);
+            }
+
+            // Collect server's icon and banner
+            let files = [&server.icon, &server.banner]
+                .iter()
+                .filter_map(|x| x.as_ref().map(|x| x.id.to_string()))
+                .collect();
+
+            (SnapshotContent::Server(server), files)
+        }
+        ReportedContent::User { id, .. } => {
+            let reported_user = db.fetch_user(id).await?;
+
+            // Users cannot report themselves
+            if reported_user.id == user.id {
+                return Err(Error::CannotReportYourself);
+            }
+
+            // Collect user's avatar and profile background
+            let files = [
+                reported_user.avatar.as_ref(),
+                reported_user
+                    .profile
+                    .as_ref()
+                    .and_then(|profile| profile.background.as_ref()),
+            ]
+            .iter()
+            .filter_map(|x| x.as_ref().map(|x| x.id.to_string()))
+            .collect();
+
+            (SnapshotContent::User(reported_user), files)
+        }
+    };
+
+    // Mark all the attachments as reported
+    for file in files {
+        db.mark_attachment_as_reported(&file).await?;
+    }
+
+    // Generate an id for the report
+    let id = Ulid::new().to_string();
+
+    // Save a snapshot of the content
+    let snapshot = Snapshot {
+        id: Ulid::new().to_string(),
+        report_id: id.to_string(),
+        content,
+    };
+
+    db.insert_snapshot(&snapshot).await?;
+
+    // Save the report
+    let report = Report {
+        id,
+        author_id: user.id,
+        content: data.content,
+        additional_context: data.additional_context,
+    };
+
+    db.insert_report(&report).await?;
+
+    Ok(())
+}

crates/quark/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-quark"
-version = "0.5.7"
+version = "0.5.8"
 license = "AGPL-3.0-or-later"
 edition = "2021"
 

crates/quark/src/impl/dummy/mod.rs

@@ -28,6 +28,11 @@ pub mod users {
     pub mod user_settings;
 }
 
+pub mod safety {
+    pub mod report;
+    pub mod snapshot;
+}
+
 #[derive(Debug, Clone)]
 pub struct DummyDb;
 

crates/quark/src/impl/dummy/safety/report.rs

@@ -0,0 +1,12 @@
+use crate::models::Report;
+use crate::{AbstractReport, Result};
+
+use super::super::DummyDb;
+
+#[async_trait]
+impl AbstractReport for DummyDb {
+    async fn insert_report(&self, report: &Report) -> Result<()> {
+        info!("Insert {:?}", report);
+        Ok(())
+    }
+}

crates/quark/src/impl/dummy/safety/snapshot.rs

@@ -0,0 +1,12 @@
+use crate::models::Snapshot;
+use crate::{AbstractSnapshot, Result};
+
+use super::super::DummyDb;
+
+#[async_trait]
+impl AbstractSnapshot for DummyDb {
+    async fn insert_snapshot(&self, snapshot: &Snapshot) -> Result<()> {
+        info!("Insert {:?}", snapshot);
+        Ok(())
+    }
+}

crates/quark/src/impl/mongo/mod.rs

@@ -39,6 +39,11 @@ pub mod users {
     pub mod user_settings;
 }
 
+pub mod safety {
+    pub mod report;
+    pub mod snapshot;
+}
+
 #[derive(Debug, Clone)]
 pub struct MongoDb(pub mongodb::Client);
 

crates/quark/src/impl/mongo/safety/report.rs

@@ -0,0 +1,13 @@
+use crate::models::Report;
+use crate::{AbstractReport, Result};
+
+use super::super::MongoDb;
+
+static COL: &str = "safety_reports";
+
+#[async_trait]
+impl AbstractReport for MongoDb {
+    async fn insert_report(&self, report: &Report) -> Result<()> {
+        self.insert_one(COL, report).await.map(|_| ())
+    }
+}

crates/quark/src/impl/mongo/safety/snapshot.rs

@@ -0,0 +1,13 @@
+use crate::models::Snapshot;
+use crate::{AbstractSnapshot, Result};
+
+use super::super::MongoDb;
+
+static COL: &str = "safety_snapshots";
+
+#[async_trait]
+impl AbstractSnapshot for MongoDb {
+    async fn insert_snapshot(&self, snapshot: &Snapshot) -> Result<()> {
+        self.insert_one(COL, snapshot).await.map(|_| ())
+    }
+}

crates/quark/src/models/mod.rs

@@ -27,9 +27,15 @@ mod users {
     pub mod user_settings;
 }
 
+mod safety {
+    pub mod report;
+    pub mod snapshot;
+}
+
 pub use admin::*;
 pub use channels::*;
 pub use media::*;
+pub use safety::*;
 pub use servers::*;
 pub use users::*;
 
@@ -41,9 +47,11 @@ pub use channel_unread::ChannelUnread;
 pub use emoji::Emoji;
 pub use message::Message;
 pub use migrations::MigrationInfo;
+pub use report::Report;
 pub use server::Server;
 pub use server_ban::ServerBan;
 pub use server_member::Member;
 pub use simple::SimpleModel;
+pub use snapshot::Snapshot;
 pub use user::User;
 pub use user_settings::UserSettings;

crates/quark/src/models/safety/report.rs

@@ -0,0 +1,85 @@
+use serde::{Deserialize, Serialize};
+
+/// Reason for reporting content (message or server)
+#[derive(Serialize, Deserialize, JsonSchema, Debug)]
+pub enum ContentReportReason {
+    /// No reason has been specified
+    NoneSpecified,
+
+    /// Blatantly illegal content
+    Illegal,
+
+    /// Content that promotes harm to others / self
+    PromotesHarm,
+
+    /// Spam or platform abuse
+    SpamAbuse,
+
+    /// Distribution of malware
+    Malware,
+
+    /// Harassment or abuse targeted at another user
+    Harassment,
+}
+
+/// Reason for reporting a user
+#[derive(Serialize, Deserialize, JsonSchema, Debug)]
+pub enum UserReportReason {
+    /// No reason has been specified
+    NoneSpecified,
+
+    /// User is sending spam or otherwise abusing the platform
+    SpamAbuse,
+
+    /// User's profile contains inappropriate content for a general audience
+    InappropriateProfile,
+
+    /// User is impersonating another user
+    Impersonation,
+
+    /// User is evading a ban
+    BanEvasion,
+
+    /// User is not of minimum age to use the platform
+    Underage,
+}
+
+#[derive(Serialize, Deserialize, JsonSchema, Debug)]
+#[serde(tag = "type")]
+pub enum ReportedContent {
+    /// Report a message
+    Message {
+        /// ID of the message
+        id: String,
+        /// Reason for reporting message
+        report_reason: ContentReportReason,
+    },
+    /// Report a server
+    Server {
+        /// ID of the server
+        id: String,
+        /// Reason for reporting server
+        report_reason: ContentReportReason,
+    },
+    /// Report a user
+    User {
+        /// ID of the user
+        id: String,
+        /// Reason for reporting a user
+        report_reason: UserReportReason,
+    },
+}
+
+/// User-generated platform moderation report.
+#[derive(Serialize, Deserialize, JsonSchema, Debug)]
+pub struct Report {
+    /// Unique Id
+    #[serde(rename = "_id")]
+    pub id: String,
+    /// Id of the user creating this report
+    pub author_id: String,
+    /// Reported content
+    pub content: ReportedContent,
+    /// Additional report context
+    pub additional_context: String,
+}

crates/quark/src/models/safety/snapshot.rs

@@ -0,0 +1,25 @@
+use serde::{Deserialize, Serialize};
+
+use crate::models::{Message, Server, User};
+
+/// Enum to map into different models
+/// that can be saved in a snapshot
+#[derive(Serialize, Deserialize, JsonSchema, Debug)]
+#[serde(tag = "_type")]
+pub enum SnapshotContent {
+    Message(Message),
+    Server(Server),
+    User(User),
+}
+
+/// Snapshot of some content
+#[derive(Serialize, Deserialize, JsonSchema, Debug)]
+pub struct Snapshot {
+    /// Unique Id
+    #[serde(rename = "_id")]
+    pub id: String,
+    /// Report parent Id
+    pub report_id: String,
+    /// Snapshot of content
+    pub content: SnapshotContent,
+}

crates/quark/src/traits/mod.rs

@@ -26,6 +26,11 @@ mod users {
     pub mod user_settings;
 }
 
+mod safety {
+    pub mod report;
+    pub mod snapshot;
+}
+
 pub use admin::migrations::AbstractMigrations;
 
 pub use media::attachment::AbstractAttachment;
@@ -44,6 +49,9 @@ pub use users::bot::AbstractBot;
 pub use users::user::AbstractUser;
 pub use users::user_settings::AbstractUserSettings;
 
+pub use safety::report::AbstractReport;
+pub use safety::snapshot::AbstractSnapshot;
+
 pub trait AbstractDatabase:
     Sync
     + Send
@@ -60,5 +68,7 @@ pub trait AbstractDatabase:
     + AbstractBot
     + AbstractUser
     + AbstractUserSettings
+    + AbstractReport
+    + AbstractSnapshot
 {
 }

crates/quark/src/traits/safety/report.rs

@@ -0,0 +1,8 @@
+use crate::models::Report;
+use crate::Result;
+
+#[async_trait]
+pub trait AbstractReport: Sync + Send {
+    /// Insert a new report into the database
+    async fn insert_report(&self, report: &Report) -> Result<()>;
+}

crates/quark/src/traits/safety/snapshot.rs

@@ -0,0 +1,8 @@
+use crate::models::Snapshot;
+use crate::Result;
+
+#[async_trait]
+pub trait AbstractSnapshot: Sync + Send {
+    /// Insert a new snapshot into the database
+    async fn insert_snapshot(&self, snapshot: &Snapshot) -> Result<()>;
+}

crates/quark/src/util/result.rs

@@ -64,6 +64,9 @@ pub enum Error {
     IsBot,
     BotIsPrivate,
 
+    // ? User safety related errors
+    CannotReportYourself,
+
     // ? Permission errors
     MissingPermission {
         permission: Permission,
@@ -166,6 +169,8 @@ impl<'r> Responder<'r, 'static> for Error {
             Error::IsBot => Status::BadRequest,
             Error::BotIsPrivate => Status::Forbidden,
 
+            Error::CannotReportYourself => Status::BadRequest,
+
             Error::MissingPermission { .. } => Status::Forbidden,
             Error::MissingUserPermission { .. } => Status::Forbidden,
             Error::NotElevated => Status::Forbidden,

crates/quark/src/web/ratelimiter.rs

@@ -129,6 +129,8 @@ fn resolve_bucket<'r>(request: &'r rocket::Request<'_>) -> (&'r str, Option<&'r
                 }
             }
             ("swagger", _) => ("swagger", None),
+            ("safety", Some("report")) => ("safety_report", Some("report")),
+            ("safety", _) => ("safety", None),
             _ => ("any", None),
         }
     } else {
@@ -148,6 +150,8 @@ fn resolve_bucket_limit(bucket: &str) -> u8 {
         "auth_delete" => 255,
         "default_avatar" => 255,
         "swagger" => 100,
+        "safety" => 15,
+        "safety_report" => 3,
         _ => 20,
     }
 }