diff --git a/Cargo.lock b/Cargo.lock index d5d82818..0ac9cf9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5381,6 +5381,7 @@ dependencies = [ "async-std", "async-trait", "authifier", + "axum", "base64 0.21.3", "bson", "deadqueue", diff --git a/crates/core/config/Revolt.toml b/crates/core/config/Revolt.toml index 23181762..178ad36d 100644 --- a/crates/core/config/Revolt.toml +++ b/crates/core/config/Revolt.toml @@ -100,6 +100,8 @@ blocked_mime_types = [] clamd_host = "" [files.limit] +# Minimum file size (in bytes) +min_file_size = 1 # Minimum image resolution min_resolution = [1, 1] # Maximum MP of images @@ -153,42 +155,66 @@ server_emoji = 100 server_roles = 200 server_channels = 200 -# How many days since creation a user is considered new -new_user_days = 3 +# How many hours since creation a user is considered new +new_user_hours = 72 # Maximum permissible body size in bytes for uploads # (should be greater than any one file upload limit) body_limit_size = 20_000_000 [features.limits.new_user] +# Limits imposed on new users + +# Number of outgoing friend requests permitted at any time outgoing_friend_requests = 5 +# Maximum number of owned bots bots = 2 -message_length = 2000 -message_attachments = 5 -servers = 100 -attachment_size = 20_000_000 -avatar_size = 4_000_000 -background_size = 6_000_000 -icon_size = 2_500_000 -banner_size = 6_000_000 -emoji_size = 500_000 +# Message content length +message_length = 2000 + +# Number of attachments that can be included +message_attachments = 5 + +# Maximum number of servers the user can create/join +servers = 50 + +[features.limits.new_user.file_upload_size_limit] +# Maximum file size limits (in bytes) +attachments = 20_000_000 +avatars = 4_000_000 +backgrounds = 6_000_000 +icons = 2_500_000 +banners = 6_000_000 +emojis = 500_000 [features.limits.default] +# Limits imposed on users by default + +# Number of outgoing friend requests permitted at any time outgoing_friend_requests = 10 +# Maximum number of owned bots bots = 5 + +# Message content length message_length = 2000 + +# Number of attachments that can be included message_attachments = 5 + +# Maximum number of servers the user can create/join servers = 100 -attachment_size = 20_000_000 -avatar_size = 4_000_000 -background_size = 6_000_000 -icon_size = 2_500_000 -banner_size = 6_000_000 -emoji_size = 500_000 +[features.limits.default.file_upload_size_limit] +# Maximum file size limits (in bytes) +attachments = 20_000_000 +avatars = 4_000_000 +backgrounds = 6_000_000 +icons = 2_500_000 +banners = 6_000_000 +emojis = 500_000 [sentry] # Configuration for Sentry error reporting diff --git a/crates/core/config/src/lib.rs b/crates/core/config/src/lib.rs index dceba725..33b8aeef 100644 --- a/crates/core/config/src/lib.rs +++ b/crates/core/config/src/lib.rs @@ -136,6 +136,7 @@ pub struct Api { #[derive(Deserialize, Debug, Clone)] pub struct FilesLimit { + pub min_file_size: usize, pub min_resolution: [usize; 2], pub max_mega_pixels: usize, pub max_pixel_side: usize, @@ -172,7 +173,7 @@ pub struct GlobalLimits { pub server_roles: usize, pub server_channels: usize, - pub new_user_days: usize, + pub new_user_hours: usize, pub body_limit_size: usize, } @@ -186,12 +187,7 @@ pub struct FeaturesLimits { pub message_attachments: usize, pub servers: usize, - pub attachment_size: usize, - pub avatar_size: usize, - pub background_size: usize, - pub icon_size: usize, - pub banner_size: usize, - pub emoji_size: usize, + pub file_upload_size_limit: HashMap, } #[derive(Deserialize, Debug, Clone)] diff --git a/crates/core/database/src/models/admin_migrations/ops/mongodb/init.rs b/crates/core/database/src/models/admin_migrations/ops/mongodb/init.rs index 7e6f3d7f..82ff3ee9 100644 --- a/crates/core/database/src/models/admin_migrations/ops/mongodb/init.rs +++ b/crates/core/database/src/models/admin_migrations/ops/mongodb/init.rs @@ -56,6 +56,10 @@ pub async fn create_database(db: &MongoDb) { .await .expect("Failed to create attachments collection."); + db.create_collection("attachment_hashes", None) + .await + .expect("Failed to create attachment_hashes collection."); + db.create_collection("user_settings", None) .await .expect("Failed to create user_settings collection."); @@ -209,6 +213,40 @@ pub async fn create_database(db: &MongoDb) { .await .expect("Failed to create server_members index."); + db.run_command( + doc! { + "createIndexes": "attachments", + "indexes": [ + { + "key": { + "hash": 1_i32 + }, + "name": "hash" + } + ] + }, + None, + ) + .await + .expect("Failed to create attachments index."); + + db.run_command( + doc! { + "createIndexes": "attachment_hashes", + "indexes": [ + { + "key": { + "processed_hash": 1_i32 + }, + "name": "processed_hash" + } + ] + }, + None, + ) + .await + .expect("Failed to create attachment_hashes index."); + db.collection("migrations") .insert_one( doc! { diff --git a/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs index 9898be7e..b236f7fb 100644 --- a/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs +++ b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs @@ -20,7 +20,7 @@ struct MigrationInfo { revision: i32, } -pub const LATEST_REVISION: i32 = 28; +pub const LATEST_REVISION: i32 = 29; pub async fn migrate_database(db: &MongoDb) { let migrations = db.col::("migrations"); @@ -1094,6 +1094,51 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 { .expect("Failed to create message index."); } + if revision <= 28 { + info!("Running migration [revision 28 / 10-09-2024]: Add support for new Autumn."); + + db.db() + .create_collection("attachment_hashes", None) + .await + .ok(); + + db.db() + .run_command( + doc! { + "createIndexes": "attachments", + "indexes": [ + { + "key": { + "hash": 1_i32 + }, + "name": "hash" + } + ] + }, + None, + ) + .await + .expect("Failed to create attachments index."); + + db.db() + .run_command( + doc! { + "createIndexes": "attachment_hashes", + "indexes": [ + { + "key": { + "processed_hash": 1_i32 + }, + "name": "processed_hash" + } + ] + }, + None, + ) + .await + .expect("Failed to create attachment_hashes index."); + } + // Need to migrate fields on attachments, change `user_id`, `object_id`, etc to `parent`. // Reminder to update LATEST_REVISION when adding new migrations. diff --git a/crates/core/result/src/axum.rs b/crates/core/result/src/axum.rs index e42348d6..29c52875 100644 --- a/crates/core/result/src/axum.rs +++ b/crates/core/result/src/axum.rs @@ -74,6 +74,9 @@ impl IntoResponse for Error { ErrorType::FailedValidation { .. } => StatusCode::BAD_REQUEST, ErrorType::ProxyError => StatusCode::BAD_REQUEST, + ErrorType::FileTooSmall => StatusCode::UNPROCESSABLE_ENTITY, + ErrorType::FileTooLarge { .. } => StatusCode::UNPROCESSABLE_ENTITY, + ErrorType::FileTypeNotAllowed => StatusCode::BAD_REQUEST, }; (status, Json(&self)).into_response() diff --git a/crates/core/result/src/lib.rs b/crates/core/result/src/lib.rs index 9e0579de..d5d093c7 100644 --- a/crates/core/result/src/lib.rs +++ b/crates/core/result/src/lib.rs @@ -156,6 +156,11 @@ pub enum ErrorType { // ? Micro-service errors ProxyError, + FileTooSmall, + FileTooLarge { + max: usize, + }, + FileTypeNotAllowed, // ? Legacy errors VosoUnavailable, diff --git a/crates/core/result/src/rocket.rs b/crates/core/result/src/rocket.rs index 58b54689..2b3a3fb2 100644 --- a/crates/core/result/src/rocket.rs +++ b/crates/core/result/src/rocket.rs @@ -80,6 +80,9 @@ impl<'r> Responder<'r, 'static> for Error { ErrorType::FailedValidation { .. } => Status::BadRequest, ErrorType::ProxyError => Status::BadRequest, + ErrorType::FileTooSmall => Status::UnprocessableEntity, + ErrorType::FileTooLarge { .. } => Status::UnprocessableEntity, + ErrorType::FileTypeNotAllowed => Status::BadRequest, }; // Serialize the error data structure into JSON. diff --git a/crates/services/autumn/Cargo.toml b/crates/services/autumn/Cargo.toml index d6739f69..c36354f2 100644 --- a/crates/services/autumn/Cargo.toml +++ b/crates/services/autumn/Cargo.toml @@ -37,7 +37,9 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } # Core crates revolt-files = { version = "0.7.16", path = "../../core/files" } revolt-config = { version = "0.7.16", path = "../../core/config" } -revolt-database = { version = "0.7.16", path = "../../core/database" } +revolt-database = { version = "0.7.16", path = "../../core/database", features = [ + "axum-impl", +] } revolt-result = { version = "0.7.16", path = "../../core/result", features = [ "utoipa", "axum", diff --git a/crates/services/autumn/src/api.rs b/crates/services/autumn/src/api.rs index a7063d27..455f5c2d 100644 --- a/crates/services/autumn/src/api.rs +++ b/crates/services/autumn/src/api.rs @@ -5,7 +5,7 @@ use std::{ use axum::{ extract::{DefaultBodyLimit, Path, State}, - http::header, + http::{header, HeaderMap}, response::{IntoResponse, Redirect, Response}, routing::{get, post}, Json, Router, @@ -14,7 +14,7 @@ use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart}; use image::ImageReader; use lazy_static::lazy_static; use revolt_config::config; -use revolt_database::{Database, FileHash, Metadata}; +use revolt_database::{Database, FileHash, Metadata, User}; use revolt_files::{fetch_from_s3, AUTHENTICATION_TAG_SIZE_BYTES}; use revolt_result::{create_error, Result}; use serde::{Deserialize, Serialize}; @@ -136,16 +136,23 @@ pub struct UploadResponse { ("tag" = Tag, Path, description = "Tag to upload to (e.g. attachments, icons, ...)") ), request_body(content_type = "multipart/form-data", content = UploadPayload), + security( + ("session_token" = []), + ("bot_token" = []) + ) )] async fn upload_file( + State(db): State, + user: User, Path(tag): Path, TypedMultipart(UploadPayload { mut file }): TypedMultipart, ) -> axum::response::Result> { + // Fetch configuration + let config = config().await; + // Keep track of processing time let now = Instant::now(); - // TODO: authenticate the user - // Extract the filename, or give it a generic name let file_name = file.metadata.file_name.unwrap_or("unnamed-file".to_owned()); @@ -158,7 +165,21 @@ async fn upload_file( // Take note of original file size let original_file_size = buf.len(); - // TODO: check against tag + // Ensure the file is not empty + if original_file_size < config.files.limit.min_file_size { + return Err(create_error!(FileTooSmall).into()); + } + + // Get user's file upload limits + let limits = user.limits().await; + let size_limit = *limits + .file_upload_size_limit + .get(tag.into()) + .expect("size limit"); + + if original_file_size > size_limit { + return Err(create_error!(FileTooLarge { max: size_limit }).into()); + } // Generate sha256 hash let original_hash = { @@ -167,13 +188,28 @@ async fn upload_file( hasher.finalize() }; - // TODO: find existing `hash` (or match `processed`) and use that if possible - // then: create attachment and return UploadResponse { id } + // Find an existing hash and use that if possible + if let Ok(file_hash) = db + .fetch_attachment_hash(&format!("{original_hash:02x}")) + // or match processed (add to query TODO) + .await + { + unimplemented!() // TODO + // then: create attachment and return UploadResponse { id } + } // Determine the mime type for the file let mime_type = determine_mime_type(&mut file.contents, &buf, &file_name); - // TODO: block mime types here + // Check blocklist for mime type + if config + .files + .blocked_mime_types + .iter() + .any(|m| m == mime_type) + { + return Err(create_error!(FileTypeNotAllowed).into()); + } // Determine metadata for the file let metadata = generate_metadata(&file.contents, mime_type); @@ -193,7 +229,7 @@ async fn upload_file( let process_ratio = new_file_size as f32 / original_file_size as f32; let time_to_process = Instant::now() - now; - tracing::info!("Received file {file_name}\nOriginal hash: {original_hash:02X}\nOriginal size: {original_file_size} bytes\nMime type: {mime_type}\nMetadata: {metadata:?}\nProcessed file size: {new_file_size} bytes ({:.2}%).\nProcessed hash: {processed_hash:02X}\nProcessing took {time_to_process:?}", process_ratio * 100.0); + tracing::info!("Received file {file_name}\nOriginal hash: {original_hash:02x}\nOriginal size: {original_file_size} bytes\nMime type: {mime_type}\nMetadata: {metadata:?}\nProcessed file size: {new_file_size} bytes ({:.2}%).\nProcessed hash: {processed_hash:02x}\nProcessing took {time_to_process:?}", process_ratio * 100.0); // TODO: encrypt the file and generate FileHash // note: file_size is processed file's size (incl. authTag in encrypted buffer) diff --git a/crates/services/autumn/src/main.rs b/crates/services/autumn/src/main.rs index cf741f79..7aff1c69 100644 --- a/crates/services/autumn/src/main.rs +++ b/crates/services/autumn/src/main.rs @@ -52,9 +52,13 @@ async fn main() -> Result<(), std::io::Error> { fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) { if let Some(components) = openapi.components.as_mut() { components.add_security_scheme( - "api_key", - SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::new("todo_apikey"))), - ) + "bot_token", + SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::new("X-Bot-Token"))), + ); + components.add_security_scheme( + "session_token", + SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::new("X-Session-Token"))), + ); } } } @@ -69,6 +73,7 @@ async fn main() -> Result<(), std::io::Error> { .with_state(db); // Configure TCP listener and bind + tracing::info!("hi vscode, please port forward http://127.0.0.1:14704, thank you!"); let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, 14704)); let listener = TcpListener::bind(&address).await?; axum::serve(listener, app.into_make_service()).await diff --git a/crates/services/autumn/src/metadata.rs b/crates/services/autumn/src/metadata.rs index 5203a520..e7526a7c 100644 --- a/crates/services/autumn/src/metadata.rs +++ b/crates/services/autumn/src/metadata.rs @@ -1,7 +1,6 @@ -use std::{io::Cursor, os::unix::fs::MetadataExt}; +use std::io::Cursor; use revolt_database::Metadata; -use revolt_result::{create_error, Result}; use tempfile::NamedTempFile; /// Intersection of what infer can detect and what image-rs supports