fix: add separate bucket for default_avatar

closes #166

Cargo.lock

@@ -2505,7 +2505,7 @@ dependencies = [
 [[package]]
 name = "rauth"
 version = "1.0.0"
-source = "git+https://github.com/insertish/rauth?rev=58aebdb6f9f25ff676f44726cce240570249ce47#58aebdb6f9f25ff676f44726cce240570249ce47"
+source = "git+https://github.com/insertish/rauth?rev=90c0c1375914d9f731a4ef1b590fcd0e5997fab5#90c0c1375914d9f731a4ef1b590fcd0e5997fab5"
 dependencies = [
  "async-std",
  "async-trait",
@@ -2986,23 +2986,15 @@ dependencies = [
 [[package]]
 name = "rocket_rauth"
 version = "1.0.0"
-source = "git+https://github.com/insertish/rauth?rev=58aebdb6f9f25ff676f44726cce240570249ce47#58aebdb6f9f25ff676f44726cce240570249ce47"
+source = "git+https://github.com/insertish/rauth?rev=90c0c1375914d9f731a4ef1b590fcd0e5997fab5#90c0c1375914d9f731a4ef1b590fcd0e5997fab5"
 dependencies = [
- "async-std",
- "base32",
- "chrono",
- "iso8601-timestamp",
- "mongodb",
  "okapi",
  "rauth",
- "regex",
- "reqwest",
  "rocket",
  "rocket_empty",
  "rocket_okapi",
  "schemars",
  "serde",
- "serde_json",
 ]
 
 [[package]]

crates/delta/Cargo.toml

@@ -52,7 +52,7 @@ mobc-redis = { version = "0.7.0", default-features = false, features = ["async-s
 # web
 rocket = { version = "0.5.0-rc.2", default-features = false, features = ["json"] }
 rocket_empty = { git = "https://github.com/insertish/rocket_empty", branch = "master" }
-rocket_rauth = { git = "https://github.com/insertish/rauth", rev = "58aebdb6f9f25ff676f44726cce240570249ce47" }
+rocket_rauth = { git = "https://github.com/insertish/rauth", rev = "90c0c1375914d9f731a4ef1b590fcd0e5997fab5" }
 
 # spec generation
 schemars = "0.8.8"

crates/delta/src/routes/channels/message_edit.rs

@@ -36,6 +36,8 @@ pub async fn req(
     edit.validate()
         .map_err(|error| Error::FailedValidation { error })?;
 
+    Message::validate_sum(&edit.content, &edit.embeds)?;
+
     let mut message = msg.as_message(db).await?;
     if message.channel != target {
         return Err(Error::NotFound);

crates/delta/src/routes/channels/message_send.rs

@@ -21,6 +21,7 @@ pub struct DataMessageSend {
     /// Unique token to prevent duplicate message sending
     ///
     /// **This is deprecated and replaced by `Idempotency-Key`!**
+    #[validate(length(min = 1, max = 64))]
     nonce: Option<String>,
 
     /// Message content to send
@@ -32,6 +33,8 @@ pub struct DataMessageSend {
     /// Messages to reply to
     replies: Option<Vec<Reply>>,
     /// Embeds to include in message
+    ///
+    /// Text embed content contributes to the content length cap
     #[validate(length(min = 1, max = 10))]
     embeds: Option<Vec<SendableEmbed>>,
     /// Masquerade to apply to this message
@@ -60,6 +63,8 @@ pub async fn message_send(
     data.validate()
         .map_err(|error| Error::FailedValidation { error })?;
 
+    Message::validate_sum(&data.content, &data.embeds)?;
+
     idempotency.consume_nonce(data.nonce).await?;
 
     let channel = target.as_channel(db).await?;

crates/delta/src/routes/onboard/complete.rs

@@ -32,13 +32,10 @@ pub async fn req(
     data.validate()
         .map_err(|error| Error::FailedValidation { error })?;
 
-    if db.is_username_taken(&data.username).await? {
-        return Err(Error::UsernameTaken);
-    }
-
+    let username = User::validate_username(db, data.username).await?;
     let user = User {
         id: session.user_id,
-        username: data.username,
+        username,
         ..Default::default()
     };
 

crates/delta/src/util/regex.rs

@@ -6,4 +6,4 @@ use regex::Regex;
 /// Block zero width space
 /// Block lookalike characters
 pub static RE_USERNAME: Lazy<Regex> =
-    Lazy::new(|| Regex::new(r"^[^\u200BА-Яа-яΑ-Ωα-ω]+$").unwrap());
+    Lazy::new(|| Regex::new(r"^[^\u200BА-Яа-яΑ-Ωα-ω@#:\n]+$").unwrap());

crates/quark/Cargo.toml

@@ -84,7 +84,7 @@ rocket_empty = { optional = true, git = "https://github.com/insertish/rocket_emp
 rocket_cors = { optional = true, git = "https://github.com/lawliet89/rocket_cors", rev = "5843861a88958c16bfaa0b40f0d8910772bcd2f6" }
 
 # rAuth
-rauth = { git = "https://github.com/insertish/rauth", rev = "58aebdb6f9f25ff676f44726cce240570249ce47", features = [ "async-std-runtime" ] }
+rauth = { git = "https://github.com/insertish/rauth", rev = "90c0c1375914d9f731a4ef1b590fcd0e5997fab5", features = [ "async-std-runtime" ] }
 
 # Sentry
 sentry = "0.25.0"

crates/quark/src/impl/dummy/autumn/attachment.rs

@@ -39,4 +39,9 @@ impl AbstractAttachment for DummyDb {
         info!("Marked {id} as deleted");
         Ok(())
     }
+
+    async fn mark_attachments_as_deleted(&self, ids: &[String]) -> Result<()> {
+        info!("Marked {ids:?} as deleted");
+        Ok(())
+    }
 }

crates/quark/src/impl/generic/channels/channel.rs

@@ -242,7 +242,12 @@ impl Channel {
     /// Add user to a group
     pub async fn add_user_to_group(&mut self, db: &Database, user: &str, by: &str) -> Result<()> {
         if let Channel::Group { recipients, .. } = self {
-            recipients.push(user.to_string());
+            let user = user.to_string();
+            if recipients.contains(&user) {
+                return Err(Error::AlreadyInGroup);
+            }
+
+            recipients.push(user);
         }
 
         match &self {

crates/quark/src/impl/generic/channels/message.rs

@@ -135,7 +135,17 @@ impl Message {
 
     /// Delete a message
     pub async fn delete(self, db: &Database) -> Result<()> {
+        let file_ids: Vec<String> = self
+            .attachments
+            .map(|files| files.iter().map(|file| file.id.to_string()).collect())
+            .unwrap_or_default();
+
+        if !file_ids.is_empty() {
+            db.mark_attachments_as_deleted(&file_ids).await?;
+        }
+
         db.delete_message(&self.id).await?;
+
         EventV1::MessageDelete {
             id: self.id,
             channel: self.channel.clone(),
@@ -156,6 +166,31 @@ impl Message {
         .await;
         Ok(())
     }
+
+    /// Validate the sum of content of a message is under threshold
+    pub fn validate_sum(
+        content: &Option<String>,
+        embeds: &Option<Vec<SendableEmbed>>,
+    ) -> Result<()> {
+        let mut running_total = 0;
+        if let Some(content) = content {
+            running_total += content.len();
+        }
+
+        if let Some(embeds) = embeds {
+            for embed in embeds {
+                if let Some(desc) = &embed.description {
+                    running_total += desc.len();
+                }
+            }
+        }
+
+        if running_total <= 2000 {
+            Ok(())
+        } else {
+            Err(Error::PayloadTooLarge)
+        }
+    }
 }
 
 pub trait IntoUsers {

crates/quark/src/impl/generic/users/user.rs

@@ -150,8 +150,8 @@ impl User {
         Ok(db.fetch_server_count(&self.id).await? <= 100)
     }
 
-    /// Update a user's username
-    pub async fn update_username(&mut self, db: &Database, username: String) -> Result<()> {
+    /// Sanitise and validate a username can be used
+    pub async fn validate_username(db: &Database, username: String) -> Result<String> {
         // Trim surrounding spaces
         let username = username.trim().to_string();
 
@@ -173,7 +173,7 @@ impl User {
         }
 
         // Ensure none of the following substrings show up in the username
-        const BLOCKED_SUBSTRINGS: &[&str] = &["@", "#", ":", "```", "\n"];
+        const BLOCKED_SUBSTRINGS: &[&str] = &["```"];
 
         for substr in BLOCKED_SUBSTRINGS {
             if username_lowercase.contains(substr) {
@@ -186,10 +186,15 @@ impl User {
             return Err(Error::UsernameTaken);
         }
 
+        Ok(username)
+    }
+
+    /// Update a user's username
+    pub async fn update_username(&mut self, db: &Database, username: String) -> Result<()> {
         self.update(
             db,
             PartialUser {
-                username: Some(username),
+                username: Some(User::validate_username(db, username).await?),
                 ..Default::default()
             },
             vec![],

crates/quark/src/impl/mongo/autumn/attachment.rs

@@ -122,4 +122,27 @@ impl AbstractAttachment for MongoDb {
                 with: "attachment",
             })
     }
+
+    async fn mark_attachments_as_deleted(&self, ids: &[String]) -> Result<()> {
+        self.col::<Document>(COL)
+            .update_many(
+                doc! {
+                    "_id": {
+                        "$in": ids
+                    }
+                },
+                doc! {
+                    "$set": {
+                        "deleted": true
+                    }
+                },
+                None,
+            )
+            .await
+            .map(|_| ())
+            .map_err(|_| Error::DatabaseError {
+                operation: "update",
+                with: "attachments",
+            })
+    }
 }

crates/quark/src/impl/mongo/channels/channel_unread.rs

@@ -37,30 +37,46 @@ impl AbstractChannelUnread for MongoDb {
     }
 
     async fn acknowledge_channels(&self, user: &str, channels: &[String]) -> Result<()> {
+        let current_time = Ulid::new().to_string();
+
         self.col::<Document>(COL)
-            .update_one(
+            .delete_many(
                 doc! {
                     "_id.channel": {
                         "$in": channels
                     },
-                    "_id.user": user,
+                    "_id.user": user
                 },
-                doc! {
-                    "$unset": {
-                        "mentions": 1_i32
-                    },
-                    "$set": {
-                        "last_id": Ulid::new().to_string()
-                    }
-                },
-                UpdateOptions::builder().upsert(true).build(),
+                None,
             )
             .await
-            .map(|_| ())
             .map_err(|_| Error::DatabaseError {
-                operation: "update",
-                with: "channel_unread",
+                operation: "delete_many",
+                with: "channel_unreads",
+            })?;
+
+        self.col::<Document>(COL)
+            .insert_many(
+                channels
+                    .iter()
+                    .map(|channel| {
+                        doc! {
+                            "_id": {
+                                "channel": channel,
+                                "user": user
+                            },
+                            "last_id": &current_time
+                        }
+                    })
+                    .collect::<Vec<Document>>(),
+                None,
+            )
+            .await
+            .map_err(|_| Error::DatabaseError {
+                operation: "update_many",
+                with: "channel_unreads",
             })
+            .map(|_| ())
     }
 
     async fn add_mention_to_unread<'a>(

crates/quark/src/impl/mongo/channels/message.rs

@@ -14,7 +14,7 @@ impl MongoDb {
     pub async fn delete_bulk_messages(&self, projection: Document) -> Result<()> {
         let mut for_attachments = projection.clone();
         for_attachments.insert(
-            "attachment",
+            "attachments",
             doc! {
                 "$exists": 1_i32
             },
@@ -126,10 +126,7 @@ impl AbstractMessage for MongoDb {
     }
 
     async fn delete_message(&self, id: &str) -> Result<()> {
-        self.delete_bulk_messages(doc! {
-            "_id": id
-        })
-        .await
+        self.delete_one_by_id(COL, id).await.map(|_| ())
     }
 
     async fn delete_messages(&self, channel: &str, ids: Vec<String>) -> Result<()> {

crates/quark/src/traits/autumn/attachment.rs

@@ -13,4 +13,5 @@ pub trait AbstractAttachment: Sync + Send {
     async fn insert_attachment(&self, attachment: &File) -> Result<()>;
     async fn mark_attachment_as_reported(&self, id: &str) -> Result<()>;
     async fn mark_attachment_as_deleted(&self, id: &str) -> Result<()>;
+    async fn mark_attachments_as_deleted(&self, ids: &[String]) -> Result<()>;
 }

crates/quark/src/util/result.rs

@@ -41,6 +41,7 @@ pub enum Error {
     TooManyAttachments,
     TooManyReplies,
     EmptyMessage,
+    PayloadTooLarge,
     CannotRemoveYourself,
     GroupTooLarge {
         max: usize,
@@ -145,6 +146,7 @@ impl<'r> Responder<'r, 'static> for Error {
             Error::TooManyAttachments => Status::BadRequest,
             Error::TooManyReplies => Status::BadRequest,
             Error::EmptyMessage => Status::UnprocessableEntity,
+            Error::PayloadTooLarge => Status::UnprocessableEntity,
             Error::CannotRemoveYourself => Status::BadRequest,
             Error::GroupTooLarge { .. } => Status::Forbidden,
             Error::AlreadyInGroup => Status::Conflict,

crates/quark/src/web/ratelimiter.rs

@@ -100,7 +100,13 @@ fn resolve_bucket<'r>(request: &'r rocket::Request<'_>) -> (&'r str, Option<&'r
     if let Some(segment) = request.routed_segment(0) {
         let resource = request.routed_segment(1);
         match (segment, resource) {
-            ("users", _) => ("users", None),
+            ("users", _) => {
+                if let Some("default_avatar") = request.routed_segment(2) {
+                    return ("default_avatar", None);
+                }
+
+                ("users", None)
+            }
             ("bots", _) => ("bots", None),
             ("channels", Some(id)) => {
                 if request.method() == Method::Post {
@@ -137,6 +143,7 @@ fn resolve_bucket_limit(bucket: &str) -> u8 {
         "servers" => 5,
         "auth" => 15,
         "auth_delete" => 255,
+        "default_avatar" => 255,
         "swagger" => 100,
         _ => 20,
     }

docker-compose.yml

@@ -1,18 +1,73 @@
 version: "3.3"
 services:
+  # Redis
   redis:
     image: eqalpha/keydb
     ports:
       - "6379:6379"
+
+  # MongoDB
   database:
     image: mongo
     ports:
       - "27017:27017"
     volumes:
       - ./.data/db:/data/db
+
+  # MinIO
+  minio:
+    image: minio/minio
+    command: server /data
+    env_file: .env
+    volumes:
+      - ./.data/minio:/data
+    ports:
+      - "10000:9000"
+    restart: always
+
+  # Mongo Express
   mongo-express:
     image: mongo-express
     ports:
       - "8081:8081"
     environment:
       - ME_CONFIG_MONGODB_SERVER=database
+    depends_on:
+      - database
+
+  # Create buckets for minio.
+  createbuckets:
+    image: minio/mc
+    depends_on:
+      - minio
+    env_file: .env
+    entrypoint: >
+      /bin/sh -c "
+      while ! curl -s --output /dev/null --connect-timeout 1 http://minio:9000; do echo 'Waiting minio...' && sleep 0.1; done;
+      /usr/bin/mc alias set minio http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD;
+      /usr/bin/mc mb minio/attachments;
+      /usr/bin/mc mb minio/avatars;
+      /usr/bin/mc mb minio/backgrounds;
+      /usr/bin/mc mb minio/icons;
+      /usr/bin/mc mb minio/banners;
+      exit 0;
+      "
+  # File server (autumn)
+  autumn:
+    image: ghcr.io/revoltchat/autumn:1.1.4
+    env_file: .env
+    depends_on:
+      - database
+      - createbuckets
+    environment:
+      - AUTUMN_MONGO_URI=mongodb://database
+    ports:
+      - "3000:3000"
+    restart: always
+
+  # Metadata and image proxy (january)
+  january:
+    image: ghcr.io/revoltchat/january:0.3.4
+    ports:
+      - "7000:7000"
+    restart: always