feat(services/autumn): authenticate the user on upload

feat(services/autumn): file size limits on upload
chore: add database migrations for new autumn things

Cargo.lock

@@ -5381,6 +5381,7 @@ dependencies = [
  "async-std",
  "async-trait",
  "authifier",
+ "axum",
  "base64 0.21.3",
  "bson",
  "deadqueue",

crates/core/config/Revolt.toml

@@ -100,6 +100,8 @@ blocked_mime_types = []
 clamd_host = ""
 
 [files.limit]
+# Minimum file size (in bytes)
+min_file_size = 1
 # Minimum image resolution
 min_resolution = [1, 1]
 # Maximum MP of images
@@ -153,42 +155,66 @@ server_emoji = 100
 server_roles = 200
 server_channels = 200
 
-# How many days since creation a user is considered new
+# How many hours since creation a user is considered new
-new_user_days = 3
+new_user_hours = 72
 
 # Maximum permissible body size in bytes for uploads
 # (should be greater than any one file upload limit)
 body_limit_size = 20_000_000
 
 [features.limits.new_user]
+# Limits imposed on new users
+
+# Number of outgoing friend requests permitted at any time
 outgoing_friend_requests = 5
 
+# Maximum number of owned bots
 bots = 2
-message_length = 2000
-message_attachments = 5
-servers = 100
 
-attachment_size = 20_000_000
+# Message content length
-avatar_size = 4_000_000
+message_length = 2000
-background_size = 6_000_000
+
-icon_size = 2_500_000
+# Number of attachments that can be included
-banner_size = 6_000_000
+message_attachments = 5
-emoji_size = 500_000
+
+# Maximum number of servers the user can create/join
+servers = 50
+
+[features.limits.new_user.file_upload_size_limit]
+# Maximum file size limits (in bytes)
+attachments = 20_000_000
+avatars = 4_000_000
+backgrounds = 6_000_000
+icons = 2_500_000
+banners = 6_000_000
+emojis = 500_000
 
 [features.limits.default]
+# Limits imposed on users by default
+
+# Number of outgoing friend requests permitted at any time
 outgoing_friend_requests = 10
 
+# Maximum number of owned bots
 bots = 5
+
+# Message content length
 message_length = 2000
+
+# Number of attachments that can be included
 message_attachments = 5
+
+# Maximum number of servers the user can create/join
 servers = 100
 
-attachment_size = 20_000_000
+[features.limits.default.file_upload_size_limit]
-avatar_size = 4_000_000
+# Maximum file size limits (in bytes)
-background_size = 6_000_000
+attachments = 20_000_000
-icon_size = 2_500_000
+avatars = 4_000_000
-banner_size = 6_000_000
+backgrounds = 6_000_000
-emoji_size = 500_000
+icons = 2_500_000
+banners = 6_000_000
+emojis = 500_000
 
 [sentry]
 # Configuration for Sentry error reporting

crates/core/config/src/lib.rs

@@ -136,6 +136,7 @@ pub struct Api {
 
 #[derive(Deserialize, Debug, Clone)]
 pub struct FilesLimit {
+    pub min_file_size: usize,
     pub min_resolution: [usize; 2],
     pub max_mega_pixels: usize,
     pub max_pixel_side: usize,
@@ -172,7 +173,7 @@ pub struct GlobalLimits {
     pub server_roles: usize,
     pub server_channels: usize,
 
-    pub new_user_days: usize,
+    pub new_user_hours: usize,
 
     pub body_limit_size: usize,
 }
@@ -186,12 +187,7 @@ pub struct FeaturesLimits {
     pub message_attachments: usize,
     pub servers: usize,
 
-    pub attachment_size: usize,
+    pub file_upload_size_limit: HashMap<String, usize>,
-    pub avatar_size: usize,
-    pub background_size: usize,
-    pub icon_size: usize,
-    pub banner_size: usize,
-    pub emoji_size: usize,
 }
 
 #[derive(Deserialize, Debug, Clone)]

crates/core/database/src/models/admin_migrations/ops/mongodb/init.rs

@@ -56,6 +56,10 @@ pub async fn create_database(db: &MongoDb) {
         .await
         .expect("Failed to create attachments collection.");
 
+    db.create_collection("attachment_hashes", None)
+        .await
+        .expect("Failed to create attachment_hashes collection.");
+
     db.create_collection("user_settings", None)
         .await
         .expect("Failed to create user_settings collection.");
@@ -209,6 +213,40 @@ pub async fn create_database(db: &MongoDb) {
     .await
     .expect("Failed to create server_members index.");
 
+    db.run_command(
+        doc! {
+            "createIndexes": "attachments",
+            "indexes": [
+                {
+                    "key": {
+                        "hash": 1_i32
+                    },
+                    "name": "hash"
+                }
+            ]
+        },
+        None,
+    )
+    .await
+    .expect("Failed to create attachments index.");
+
+    db.run_command(
+        doc! {
+            "createIndexes": "attachment_hashes",
+            "indexes": [
+                {
+                    "key": {
+                        "processed_hash": 1_i32
+                    },
+                    "name": "processed_hash"
+                }
+            ]
+        },
+        None,
+    )
+    .await
+    .expect("Failed to create attachment_hashes index.");
+
     db.collection("migrations")
         .insert_one(
             doc! {

crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs

@@ -20,7 +20,7 @@ struct MigrationInfo {
     revision: i32,
 }
 
-pub const LATEST_REVISION: i32 = 28;
+pub const LATEST_REVISION: i32 = 29;
 
 pub async fn migrate_database(db: &MongoDb) {
     let migrations = db.col::<Document>("migrations");
@@ -1094,6 +1094,51 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
             .expect("Failed to create message index.");
     }
 
+    if revision <= 28 {
+        info!("Running migration [revision 28 / 10-09-2024]: Add support for new Autumn.");
+
+        db.db()
+            .create_collection("attachment_hashes", None)
+            .await
+            .ok();
+
+        db.db()
+            .run_command(
+                doc! {
+                    "createIndexes": "attachments",
+                    "indexes": [
+                        {
+                            "key": {
+                                "hash": 1_i32
+                            },
+                            "name": "hash"
+                        }
+                    ]
+                },
+                None,
+            )
+            .await
+            .expect("Failed to create attachments index.");
+
+        db.db()
+            .run_command(
+                doc! {
+                    "createIndexes": "attachment_hashes",
+                    "indexes": [
+                        {
+                            "key": {
+                                "processed_hash": 1_i32
+                            },
+                            "name": "processed_hash"
+                        }
+                    ]
+                },
+                None,
+            )
+            .await
+            .expect("Failed to create attachment_hashes index.");
+    }
+
     // Need to migrate fields on attachments, change `user_id`, `object_id`, etc to `parent`.
 
     // Reminder to update LATEST_REVISION when adding new migrations.

crates/core/result/src/axum.rs

@@ -74,6 +74,9 @@ impl IntoResponse for Error {
             ErrorType::FailedValidation { .. } => StatusCode::BAD_REQUEST,
 
             ErrorType::ProxyError => StatusCode::BAD_REQUEST,
+            ErrorType::FileTooSmall => StatusCode::UNPROCESSABLE_ENTITY,
+            ErrorType::FileTooLarge { .. } => StatusCode::UNPROCESSABLE_ENTITY,
+            ErrorType::FileTypeNotAllowed => StatusCode::BAD_REQUEST,
         };
 
         (status, Json(&self)).into_response()

crates/core/result/src/lib.rs

@@ -156,6 +156,11 @@ pub enum ErrorType {
 
     // ? Micro-service errors
     ProxyError,
+    FileTooSmall,
+    FileTooLarge {
+        max: usize,
+    },
+    FileTypeNotAllowed,
 
     // ? Legacy errors
     VosoUnavailable,

crates/core/result/src/rocket.rs

@@ -80,6 +80,9 @@ impl<'r> Responder<'r, 'static> for Error {
             ErrorType::FailedValidation { .. } => Status::BadRequest,
 
             ErrorType::ProxyError => Status::BadRequest,
+            ErrorType::FileTooSmall => Status::UnprocessableEntity,
+            ErrorType::FileTooLarge { .. } => Status::UnprocessableEntity,
+            ErrorType::FileTypeNotAllowed => Status::BadRequest,
         };
 
         // Serialize the error data structure into JSON.

crates/services/autumn/Cargo.toml

@@ -37,7 +37,9 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 # Core crates
 revolt-files = { version = "0.7.16", path = "../../core/files" }
 revolt-config = { version = "0.7.16", path = "../../core/config" }
-revolt-database = { version = "0.7.16", path = "../../core/database" }
+revolt-database = { version = "0.7.16", path = "../../core/database", features = [
+    "axum-impl",
+] }
 revolt-result = { version = "0.7.16", path = "../../core/result", features = [
     "utoipa",
     "axum",

crates/services/autumn/src/api.rs

@@ -5,7 +5,7 @@ use std::{
 
 use axum::{
     extract::{DefaultBodyLimit, Path, State},
-    http::header,
+    http::{header, HeaderMap},
     response::{IntoResponse, Redirect, Response},
     routing::{get, post},
     Json, Router,
@@ -14,7 +14,7 @@ use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart};
 use image::ImageReader;
 use lazy_static::lazy_static;
 use revolt_config::config;
-use revolt_database::{Database, FileHash, Metadata};
+use revolt_database::{Database, FileHash, Metadata, User};
 use revolt_files::{fetch_from_s3, AUTHENTICATION_TAG_SIZE_BYTES};
 use revolt_result::{create_error, Result};
 use serde::{Deserialize, Serialize};
@@ -136,16 +136,23 @@ pub struct UploadResponse {
         ("tag" = Tag, Path, description = "Tag to upload to (e.g. attachments, icons, ...)")
     ),
     request_body(content_type = "multipart/form-data", content = UploadPayload),
+    security(
+        ("session_token" = []),
+        ("bot_token" = [])
+    )
 )]
 async fn upload_file(
+    State(db): State<Database>,
+    user: User,
     Path(tag): Path<Tag>,
     TypedMultipart(UploadPayload { mut file }): TypedMultipart<UploadPayload>,
 ) -> axum::response::Result<Json<UploadResponse>> {
+    // Fetch configuration
+    let config = config().await;
+
     // Keep track of processing time
     let now = Instant::now();
 
-    // TODO: authenticate the user
-
     // Extract the filename, or give it a generic name
     let file_name = file.metadata.file_name.unwrap_or("unnamed-file".to_owned());
 
@@ -158,7 +165,21 @@ async fn upload_file(
     // Take note of original file size
     let original_file_size = buf.len();
 
-    // TODO: check against tag
+    // Ensure the file is not empty
+    if original_file_size < config.files.limit.min_file_size {
+        return Err(create_error!(FileTooSmall).into());
+    }
+
+    // Get user's file upload limits
+    let limits = user.limits().await;
+    let size_limit = *limits
+        .file_upload_size_limit
+        .get(tag.into())
+        .expect("size limit");
+
+    if original_file_size > size_limit {
+        return Err(create_error!(FileTooLarge { max: size_limit }).into());
+    }
 
     // Generate sha256 hash
     let original_hash = {
@@ -167,13 +188,28 @@ async fn upload_file(
         hasher.finalize()
     };
 
-    // TODO: find existing `hash` (or match `processed`) and use that if possible
+    // Find an existing hash and use that if possible
-    // then: create attachment and return UploadResponse { id }
+    if let Ok(file_hash) = db
+        .fetch_attachment_hash(&format!("{original_hash:02x}"))
+        // or match processed (add to query TODO)
+        .await
+    {
+        unimplemented!() // TODO
+                         // then: create attachment and return UploadResponse { id }
+    }
 
     // Determine the mime type for the file
     let mime_type = determine_mime_type(&mut file.contents, &buf, &file_name);
 
-    // TODO: block mime types here
+    // Check blocklist for mime type
+    if config
+        .files
+        .blocked_mime_types
+        .iter()
+        .any(|m| m == mime_type)
+    {
+        return Err(create_error!(FileTypeNotAllowed).into());
+    }
 
     // Determine metadata for the file
     let metadata = generate_metadata(&file.contents, mime_type);
@@ -193,7 +229,7 @@ async fn upload_file(
     let process_ratio = new_file_size as f32 / original_file_size as f32;
     let time_to_process = Instant::now() - now;
 
-    tracing::info!("Received file {file_name}\nOriginal hash: {original_hash:02X}\nOriginal size: {original_file_size} bytes\nMime type: {mime_type}\nMetadata: {metadata:?}\nProcessed file size: {new_file_size} bytes ({:.2}%).\nProcessed hash: {processed_hash:02X}\nProcessing took {time_to_process:?}", process_ratio * 100.0);
+    tracing::info!("Received file {file_name}\nOriginal hash: {original_hash:02x}\nOriginal size: {original_file_size} bytes\nMime type: {mime_type}\nMetadata: {metadata:?}\nProcessed file size: {new_file_size} bytes ({:.2}%).\nProcessed hash: {processed_hash:02x}\nProcessing took {time_to_process:?}", process_ratio * 100.0);
 
     // TODO: encrypt the file and generate FileHash
     // note: file_size is processed file's size (incl. authTag in encrypted buffer)

crates/services/autumn/src/main.rs

@@ -52,9 +52,13 @@ async fn main() -> Result<(), std::io::Error> {
         fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) {
             if let Some(components) = openapi.components.as_mut() {
                 components.add_security_scheme(
-                    "api_key",
+                    "bot_token",
-                    SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::new("todo_apikey"))),
+                    SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::new("X-Bot-Token"))),
-                )
+                );
+                components.add_security_scheme(
+                    "session_token",
+                    SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::new("X-Session-Token"))),
+                );
             }
         }
     }
@@ -69,6 +73,7 @@ async fn main() -> Result<(), std::io::Error> {
         .with_state(db);
 
     // Configure TCP listener and bind
+    tracing::info!("hi vscode, please port forward http://127.0.0.1:14704, thank you!");
     let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, 14704));
     let listener = TcpListener::bind(&address).await?;
     axum::serve(listener, app.into_make_service()).await

crates/services/autumn/src/metadata.rs

@@ -1,7 +1,6 @@
-use std::{io::Cursor, os::unix::fs::MetadataExt};
+use std::io::Cursor;
 
 use revolt_database::Metadata;
-use revolt_result::{create_error, Result};
 use tempfile::NamedTempFile;
 
 /// Intersection of what infer can detect and what image-rs supports