jmug/stoatchat
1
0
Fork 1
mirror of https://github.com/stoatchat/stoatchat.git synced 2026-07-14 13:36:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(services/autumn): authenticate the user on upload

Browse Source 
feat(services/autumn): file size limits on upload
chore: add database migrations for new autumn things
This commit is contained in:
Paul Makles
2024-09-10 16:53:58 +01:00
parent 1a6a8a809b
commit 6209bc7152
12 changed files with 199 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@@ -5381,6 +5381,7 @@ dependencies = [
"async-std",
"async-trait",
"authifier",
"axum",
"base64 0.21.3",
"bson",
"deadqueue",

60
crates/core/config/Revolt.toml
View File

@@ -100,6 +100,8 @@ blocked_mime_types = []
clamd_host = ""
[files.limit]
# Minimum file size (in bytes)
min_file_size = 1
# Minimum image resolution
min_resolution = [1, 1]
# Maximum MP of images
@@ -153,42 +155,66 @@ server_emoji = 100
server_roles = 200
server_channels = 200
# How many days since creation a user is considered new
new_user_days = 3
# How many hours since creation a user is considered new
new_user_hours = 72
# Maximum permissible body size in bytes for uploads
# (should be greater than any one file upload limit)
body_limit_size = 20_000_000
[features.limits.new_user]
# Limits imposed on new users
# Number of outgoing friend requests permitted at any time
outgoing_friend_requests = 5
# Maximum number of owned bots
bots = 2
message_length = 2000
message_attachments = 5
servers = 100
attachment_size = 20_000_000
avatar_size = 4_000_000
background_size = 6_000_000
icon_size = 2_500_000
banner_size = 6_000_000
emoji_size = 500_000
# Message content length
message_length = 2000
# Number of attachments that can be included
message_attachments = 5
# Maximum number of servers the user can create/join
servers = 50
[features.limits.new_user.file_upload_size_limit]
# Maximum file size limits (in bytes)
attachments = 20_000_000
avatars = 4_000_000
backgrounds = 6_000_000
icons = 2_500_000
banners = 6_000_000
emojis = 500_000
[features.limits.default]
# Limits imposed on users by default
# Number of outgoing friend requests permitted at any time
outgoing_friend_requests = 10
# Maximum number of owned bots
bots = 5
# Message content length
message_length = 2000
# Number of attachments that can be included
message_attachments = 5
# Maximum number of servers the user can create/join
servers = 100
attachment_size = 20_000_000
avatar_size = 4_000_000
background_size = 6_000_000
icon_size = 2_500_000
banner_size = 6_000_000
emoji_size = 500_000
[features.limits.default.file_upload_size_limit]
# Maximum file size limits (in bytes)
attachments = 20_000_000
avatars = 4_000_000
backgrounds = 6_000_000
icons = 2_500_000
banners = 6_000_000
emojis = 500_000
[sentry]
# Configuration for Sentry error reporting

10
crates/core/config/src/lib.rs
View File

@@ -136,6 +136,7 @@ pub struct Api {
#[derive(Deserialize, Debug, Clone)]
pub struct FilesLimit {
pub min_file_size: usize,
pub min_resolution: [usize; 2],
pub max_mega_pixels: usize,
pub max_pixel_side: usize,
@@ -172,7 +173,7 @@ pub struct GlobalLimits {
pub server_roles: usize,
pub server_channels: usize,
pub new_user_days: usize,
pub new_user_hours: usize,
pub body_limit_size: usize,
}
@@ -186,12 +187,7 @@ pub struct FeaturesLimits {
pub message_attachments: usize,
pub servers: usize,
pub attachment_size: usize,
pub avatar_size: usize,
pub background_size: usize,
pub icon_size: usize,
pub banner_size: usize,
pub emoji_size: usize,
pub file_upload_size_limit: HashMap<String, usize>,
}
#[derive(Deserialize, Debug, Clone)]

38
crates/core/database/src/models/admin_migrations/ops/mongodb/init.rs
View File

@@ -56,6 +56,10 @@ pub async fn create_database(db: &MongoDb) {
.await
.expect("Failed to create attachments collection.");
db.create_collection("attachment_hashes", None)
.await
.expect("Failed to create attachment_hashes collection.");
db.create_collection("user_settings", None)
.await
.expect("Failed to create user_settings collection.");
@@ -209,6 +213,40 @@ pub async fn create_database(db: &MongoDb) {
.await
.expect("Failed to create server_members index.");
db.run_command(
doc! {
"createIndexes": "attachments",
"indexes": [
{
"key": {
"hash": 1_i32
},
"name": "hash"
}
]
},
None,
)
.await
.expect("Failed to create attachments index.");
db.run_command(
doc! {
"createIndexes": "attachment_hashes",
"indexes": [
{
"key": {
"processed_hash": 1_i32
},
"name": "processed_hash"
}
]
},
None,
)
.await
.expect("Failed to create attachment_hashes index.");
db.collection("migrations")
.insert_one(
doc! {

47
crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs
View File

@@ -20,7 +20,7 @@ struct MigrationInfo {
revision: i32,
}
pub const LATEST_REVISION: i32 = 28;
pub const LATEST_REVISION: i32 = 29;
pub async fn migrate_database(db: &MongoDb) {
let migrations = db.col::<Document>("migrations");
@@ -1094,6 +1094,51 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
.expect("Failed to create message index.");
}
if revision <= 28 {
info!("Running migration [revision 28 / 10-09-2024]: Add support for new Autumn.");
db.db()
.create_collection("attachment_hashes", None)
.await
.ok();
db.db()
.run_command(
doc! {
"createIndexes": "attachments",
"indexes": [
{
"key": {
"hash": 1_i32
},
"name": "hash"
}
]
},
None,
)
.await
.expect("Failed to create attachments index.");
db.db()
.run_command(
doc! {
"createIndexes": "attachment_hashes",
"indexes": [
{
"key": {
"processed_hash": 1_i32
},
"name": "processed_hash"
}
]
},
None,
)
.await
.expect("Failed to create attachment_hashes index.");
}
// Need to migrate fields on attachments, change `user_id`, `object_id`, etc to `parent`.
// Reminder to update LATEST_REVISION when adding new migrations.

3
crates/core/result/src/axum.rs
View File

@@ -74,6 +74,9 @@ impl IntoResponse for Error {
ErrorType::FailedValidation { .. } => StatusCode::BAD_REQUEST,
ErrorType::ProxyError => StatusCode::BAD_REQUEST,
ErrorType::FileTooSmall => StatusCode::UNPROCESSABLE_ENTITY,
ErrorType::FileTooLarge { .. } => StatusCode::UNPROCESSABLE_ENTITY,
ErrorType::FileTypeNotAllowed => StatusCode::BAD_REQUEST,
};
(status, Json(&self)).into_response()

5
crates/core/result/src/lib.rs
View File

@@ -156,6 +156,11 @@ pub enum ErrorType {
// ? Micro-service errors
ProxyError,
FileTooSmall,
FileTooLarge {
max: usize,
},
FileTypeNotAllowed,
// ? Legacy errors
VosoUnavailable,

3
crates/core/result/src/rocket.rs
View File

@@ -80,6 +80,9 @@ impl<'r> Responder<'r, 'static> for Error {
ErrorType::FailedValidation { .. } => Status::BadRequest,
ErrorType::ProxyError => Status::BadRequest,
ErrorType::FileTooSmall => Status::UnprocessableEntity,
ErrorType::FileTooLarge { .. } => Status::UnprocessableEntity,
ErrorType::FileTypeNotAllowed => Status::BadRequest,
};
// Serialize the error data structure into JSON.

4
crates/services/autumn/Cargo.toml
View File

@@ -37,7 +37,9 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
# Core crates
revolt-files = { version = "0.7.16", path = "../../core/files" }
revolt-config = { version = "0.7.16", path = "../../core/config" }
revolt-database = { version = "0.7.16", path = "../../core/database" }
revolt-database = { version = "0.7.16", path = "../../core/database", features = [
"axum-impl",
] }
revolt-result = { version = "0.7.16", path = "../../core/result", features = [
"utoipa",
"axum",

54
crates/services/autumn/src/api.rs
View File

@@ -5,7 +5,7 @@ use std::{
use axum::{
extract::{DefaultBodyLimit, Path, State},
http::header,
http::{header, HeaderMap},
response::{IntoResponse, Redirect, Response},
routing::{get, post},
Json, Router,
@@ -14,7 +14,7 @@ use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart};
use image::ImageReader;
use lazy_static::lazy_static;
use revolt_config::config;
use revolt_database::{Database, FileHash, Metadata};
use revolt_database::{Database, FileHash, Metadata, User};
use revolt_files::{fetch_from_s3, AUTHENTICATION_TAG_SIZE_BYTES};
use revolt_result::{create_error, Result};
use serde::{Deserialize, Serialize};
@@ -136,16 +136,23 @@ pub struct UploadResponse {
("tag" = Tag, Path, description = "Tag to upload to (e.g. attachments, icons, ...)")
),
request_body(content_type = "multipart/form-data", content = UploadPayload),
security(
("session_token" = []),
("bot_token" = [])
)
)]
async fn upload_file(
State(db): State<Database>,
user: User,
Path(tag): Path<Tag>,
TypedMultipart(UploadPayload { mut file }): TypedMultipart<UploadPayload>,
) -> axum::response::Result<Json<UploadResponse>> {
// Fetch configuration
let config = config().await;
// Keep track of processing time
let now = Instant::now();
// TODO: authenticate the user
// Extract the filename, or give it a generic name
let file_name = file.metadata.file_name.unwrap_or("unnamed-file".to_owned());
@@ -158,7 +165,21 @@ async fn upload_file(
// Take note of original file size
let original_file_size = buf.len();
// TODO: check against tag
// Ensure the file is not empty
if original_file_size < config.files.limit.min_file_size {
return Err(create_error!(FileTooSmall).into());
}
// Get user's file upload limits
let limits = user.limits().await;
let size_limit = *limits
.file_upload_size_limit
.get(tag.into())
.expect("size limit");
if original_file_size > size_limit {
return Err(create_error!(FileTooLarge { max: size_limit }).into());
}
// Generate sha256 hash
let original_hash = {
@@ -167,13 +188,28 @@ async fn upload_file(
hasher.finalize()
};
// TODO: find existing `hash` (or match `processed`) and use that if possible
// then: create attachment and return UploadResponse { id }
// Find an existing hash and use that if possible
if let Ok(file_hash) = db
.fetch_attachment_hash(&format!("{original_hash:02x}"))
// or match processed (add to query TODO)
.await
{
unimplemented!() // TODO
// then: create attachment and return UploadResponse { id }
}
// Determine the mime type for the file
let mime_type = determine_mime_type(&mut file.contents, &buf, &file_name);
// TODO: block mime types here
// Check blocklist for mime type
if config
.files
.blocked_mime_types
.iter()
.any(|m| m == mime_type)
{
return Err(create_error!(FileTypeNotAllowed).into());
}
// Determine metadata for the file
let metadata = generate_metadata(&file.contents, mime_type);
@@ -193,7 +229,7 @@ async fn upload_file(
let process_ratio = new_file_size as f32 / original_file_size as f32;
let time_to_process = Instant::now() - now;
tracing::info!("Received file {file_name}\nOriginal hash: {original_hash:02X}\nOriginal size: {original_file_size} bytes\nMime type: {mime_type}\nMetadata: {metadata:?}\nProcessed file size: {new_file_size} bytes ({:.2}%).\nProcessed hash: {processed_hash:02X}\nProcessing took {time_to_process:?}", process_ratio * 100.0);
tracing::info!("Received file {file_name}\nOriginal hash: {original_hash:02x}\nOriginal size: {original_file_size} bytes\nMime type: {mime_type}\nMetadata: {metadata:?}\nProcessed file size: {new_file_size} bytes ({:.2}%).\nProcessed hash: {processed_hash:02x}\nProcessing took {time_to_process:?}", process_ratio * 100.0);
// TODO: encrypt the file and generate FileHash
// note: file_size is processed file's size (incl. authTag in encrypted buffer)

11
crates/services/autumn/src/main.rs
View File

@@ -52,9 +52,13 @@ async fn main() -> Result<(), std::io::Error> {
fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) {
if let Some(components) = openapi.components.as_mut() {
components.add_security_scheme(
"api_key",
SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::new("todo_apikey"))),
)
"bot_token",
SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::new("X-Bot-Token"))),
);
components.add_security_scheme(
"session_token",
SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::new("X-Session-Token"))),
);
}
}
}
@@ -69,6 +73,7 @@ async fn main() -> Result<(), std::io::Error> {
.with_state(db);
// Configure TCP listener and bind
tracing::info!("hi vscode, please port forward http://127.0.0.1:14704, thank you!");
let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, 14704));
let listener = TcpListener::bind(&address).await?;
axum::serve(listener, app.into_make_service()).await

3
crates/services/autumn/src/metadata.rs
View File

@@ -1,7 +1,6 @@
use std::{io::Cursor, os::unix::fs::MetadataExt};
use std::io::Cursor;
use revolt_database::Metadata;
use revolt_result::{create_error, Result};
use tempfile::NamedTempFile;
/// Intersection of what infer can detect and what image-rs supports