From 24dc96f80fd3421fcb04ee35c1162888588f759b Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 1 Sep 2024 16:42:28 +0100 Subject: [PATCH] feat(services/autumn): work towards upload API --- Cargo.lock | 181 ++++++++++++++++++ compose.yml | 43 +++++ .../core/database/src/models/files/model.rs | 29 ++- crates/core/database/src/util/bridge/v0.rs | 3 +- crates/services/autumn/Cargo.toml | 8 +- crates/services/autumn/Dockerfile | 1 + crates/services/autumn/src/api.rs | 56 +++++- crates/services/autumn/src/main.rs | 2 + crates/services/autumn/src/metadata.rs | 104 ++++++++++ crates/services/autumn/src/mime_type.rs | 36 ++++ docker-compose.db.yml | 15 -- docker-compose.yml | 72 ------- 12 files changed, 453 insertions(+), 97 deletions(-) create mode 100644 compose.yml create mode 100644 crates/services/autumn/Dockerfile create mode 100644 crates/services/autumn/src/metadata.rs create mode 100644 crates/services/autumn/src/mime_type.rs delete mode 100644 docker-compose.db.yml delete mode 100644 docker-compose.yml diff --git a/Cargo.lock b/Cargo.lock index d6d984a8..c754f4bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1333,6 +1333,17 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfb" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f" +dependencies = [ + "byteorder", + "fnv", + "uuid 1.4.1", +] + [[package]] name = "cfg-expr" version = "0.15.8" @@ -2216,6 +2227,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "ffprobe" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ffef835e1f9ac151db5bb2adbb95c9dfe1f315f987f011dd89cd655b4e9a52c" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "figment" version = "0.10.6" @@ -3081,6 +3102,12 @@ dependencies = [ "quick-error 2.0.1", ] +[[package]] +name = "imagesize" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edcd27d72f2f071c64249075f42e205ff93c9a4c5f6c6da53e79ed9f9832c285" + [[package]] name = "imgref" version = "1.10.1" @@ -3115,6 +3142,15 @@ dependencies = [ "serde", ] +[[package]] +name = "infer" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc150e5ce2330295b8616ce0e3f53250e53af31759a9dbedad1621ba29151847" +dependencies = [ + "cfb", +] + [[package]] name = "inlinable_string" version = "0.1.15" @@ -3296,6 +3332,147 @@ dependencies = [ "zeroize", ] +[[package]] +name = "jxl-bitstream" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703fcab2fcc410830ee2923e7b343e065ca3373d221917c4e1942c46976f82cf" +dependencies = [ + "tracing", +] + +[[package]] +name = "jxl-coding" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1fc775c864ae62965f39fd60de5582018d130727fc210a64dae47df1e1f085d" +dependencies = [ + "jxl-bitstream", + "tracing", +] + +[[package]] +name = "jxl-color" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb1c31e10054079df585633fc14fb9e4c96565c58c05b983c502e2472b57fa0" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-threadpool", + "tracing", +] + +[[package]] +name = "jxl-frame" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35b289aa0f24044167d83a1c29ae2b99210c5082ab7e3e90dacbcae818aa0a2" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-image", + "jxl-modular", + "jxl-threadpool", + "jxl-vardct", + "tracing", +] + +[[package]] +name = "jxl-grid" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70b96735a85299a6bce8664643fcb759f29ea73ce344a90b6f7de9b92a8e9b2d" +dependencies = [ + "tracing", +] + +[[package]] +name = "jxl-image" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b31b17ed3bd0e3b65e7b06628f5930e009dda6cd17638cf5159a20a3feedec6" +dependencies = [ + "jxl-bitstream", + "jxl-color", + "jxl-grid", + "tracing", +] + +[[package]] +name = "jxl-modular" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3b9fb8f46e63a14ecedefbd0f873b04162aaf8a09676b630c31bc8dadc4638" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-threadpool", + "tracing", +] + +[[package]] +name = "jxl-oxide" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba1ee3895e6c62b131994807b1ee6d179013613a86c01c203369af8d1e8d2f0" +dependencies = [ + "jxl-bitstream", + "jxl-color", + "jxl-frame", + "jxl-grid", + "jxl-image", + "jxl-render", + "jxl-threadpool", + "tracing", +] + +[[package]] +name = "jxl-render" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "203a79b3025b86f875cc97a6f39e1fcc6314f915b2f6b69f767fca45fd482a11" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-color", + "jxl-frame", + "jxl-grid", + "jxl-image", + "jxl-modular", + "jxl-threadpool", + "jxl-vardct", + "tracing", +] + +[[package]] +name = "jxl-threadpool" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d2860c68899a3c6266044fc26c6a0041e9f27145f58cc69b6eedc1b77f5ee13" +dependencies = [ + "rayon", + "rayon-core", + "tracing", +] + +[[package]] +name = "jxl-vardct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16af82a1ad770887cad720bfd3cc6a6d023faf377036989a24cf2c6538b649e0" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-modular", + "jxl-threadpool", + "tracing", +] + [[package]] name = "k256" version = "0.11.6" @@ -5104,7 +5281,11 @@ dependencies = [ "axum", "axum-macros", "axum_typed_multipart", + "ffprobe", "image", + "imagesize", + "infer", + "jxl-oxide", "lazy_static", "moka", "revolt-config", diff --git a/compose.yml b/compose.yml new file mode 100644 index 00000000..f8edcf7b --- /dev/null +++ b/compose.yml @@ -0,0 +1,43 @@ +services: + # Redis + redis: + image: eqalpha/keydb + ports: + - "14079:6379" + + # MongoDB + database: + image: mongo + ports: + - "14017:27017" + volumes: + - ./.data/db:/data/db + + # MinIO + minio: + image: minio/minio + command: server /data + environment: + MINIO_ROOT_USER: minioautumn + MINIO_ROOT_PASSWORD: minioautumn + volumes: + - ./.data/minio:/data + ports: + - "14009:9000" + restart: always + + # Create buckets for minio. + createbuckets: + image: minio/mc + depends_on: + - minio + environment: + MINIO_ROOT_USER: minioautumn + MINIO_ROOT_PASSWORD: minioautumn + entrypoint: > + /bin/sh -c " + /usr/bin/mc config host add minio http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD; + while ! /usr/bin/mc ready minio; do echo 'Waiting minio...' && sleep 1; done; + /usr/bin/mc mb minio/revolt-uploads; + exit 0; + " diff --git a/crates/core/database/src/models/files/model.rs b/crates/core/database/src/models/files/model.rs index 786d91bc..b04521a3 100644 --- a/crates/core/database/src/models/files/model.rs +++ b/crates/core/database/src/models/files/model.rs @@ -21,8 +21,9 @@ auto_derived_partial!( /// ID of user who uploaded this file pub uploaded_id: Option, // these are Option<>s to not break file uploads on legacy Autumn - // TODO: used_for: field - // + /// What the file was used for + pub used_for: Option, + /// Whether this file was deleted #[serde(skip_serializing_if = "Option::is_none")] pub deleted: Option, @@ -55,6 +56,30 @@ auto_derived_partial!( "PartialFile" ); +auto_derived!( + /// Type of object file was used for + pub enum FileUsedForType { + // TODO: changing this requires a db migration + message, + serverBanner, + emoji, + userAvatar, + userProfileBackground, + legacyGroupIcon, + channelIcon, + serverIcon, + } + + /// Information about what the file was used for + pub struct FileUsedFor { + /// Type of the object + #[serde(rename = "type")] + pub object_type: FileUsedForType, + /// ID of the object + pub id: String, + } +); + impl File { /// Get the hash entry for this file pub async fn as_hash(&self, db: &Database) -> Result { diff --git a/crates/core/database/src/util/bridge/v0.rs b/crates/core/database/src/util/bridge/v0.rs index a3911a8d..8c0ecb3e 100644 --- a/crates/core/database/src/util/bridge/v0.rs +++ b/crates/core/database/src/util/bridge/v0.rs @@ -1,7 +1,7 @@ use revolt_models::v0::*; use revolt_permissions::{calculate_user_permissions, UserPermission}; -use crate::{util::permissions::DatabasePermissionQuery, Database}; +use crate::{util::permissions::DatabasePermissionQuery, Database, FileUsedFor}; impl crate::Bot { pub fn into_public_bot(self, user: crate::User) -> PublicBot { @@ -422,6 +422,7 @@ impl From for crate::File { hash: None, uploaded_at: None, uploaded_id: None, + used_for: None, } } } diff --git a/crates/services/autumn/Cargo.toml b/crates/services/autumn/Cargo.toml index aaa78c08..f9a1a4a1 100644 --- a/crates/services/autumn/Cargo.toml +++ b/crates/services/autumn/Cargo.toml @@ -6,7 +6,13 @@ edition = "2021" [dependencies] # Media processing webp = "0.3.0" -image = "0.25.2" +jxl-oxide = "0.8.1" +image = { version = "0.25.2" } # requires dav1d system library: features = ["avif-native"] + +# Content type processing +infer = "0.16.0" +ffprobe = "0.4.0" +imagesize = "0.13.0" # Utility lazy_static = "1.5.0" diff --git a/crates/services/autumn/Dockerfile b/crates/services/autumn/Dockerfile new file mode 100644 index 00000000..70390857 --- /dev/null +++ b/crates/services/autumn/Dockerfile @@ -0,0 +1 @@ +# TODO: https://github.com/wader/static-ffmpeg \ No newline at end of file diff --git a/crates/services/autumn/src/api.rs b/crates/services/autumn/src/api.rs index b82b8232..e7a0483a 100644 --- a/crates/services/autumn/src/api.rs +++ b/crates/services/autumn/src/api.rs @@ -11,13 +11,18 @@ use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart}; use image::ImageReader; use lazy_static::lazy_static; use revolt_config::config; -use revolt_database::{Database, FileHash}; +use revolt_database::{Database, FileHash, Metadata}; use revolt_files::fetch_from_s3; use revolt_result::{create_error, Result}; use serde::{Deserialize, Serialize}; use tempfile::NamedTempFile; use utoipa::ToSchema; +use crate::{ + metadata::{generate_metadata, temp_file_size}, + mime_type::determine_mime_type, +}; + /// Build the API router pub async fn router() -> Router { let config = config().await; @@ -132,8 +137,36 @@ pub struct UploadResponse { )] async fn upload_file( Path(tag): Path, - TypedMultipart(UploadPayload { file }): TypedMultipart, + TypedMultipart(UploadPayload { mut file }): TypedMultipart, ) -> axum::response::Result> { + // Extract the filename, or give it a generic name. + let file_name = file.metadata.file_name.unwrap_or("unnamed-file".to_owned()); + + // TODO: find existing `hash` (or match `processed`) and use that if possible + // then: create attachment and return UploadResponse { id } + + // Determine file size + let original_file_size = temp_file_size(&file.contents)?; + + // Determine the mime type for the file + let mime_type = determine_mime_type(&mut file.contents, &file_name, original_file_size); + + // TODO: block mime types here + + // Determine metadata for the file + let metadata = generate_metadata(&file.contents, mime_type); + + // TODO: Re-encode MIMES_WITH_EXIF to remove EXIF data (this needs orientation data: https://github.com/revoltchat/autumn/blob/master/src/routes/upload.rs#L96) + // TODO: Re-encode videos to remove EXIF data (this may need orientation data?) + + // TODO: Virus scan Metadata::Text/File + + // TODO: encrypt the file and generate FileHash + // note: file_size is processed file's size (incl. authTag in encrypted buffer) + // TODO: insert FileHash + // TODO: upload to S3 + // TODO: create attachment + Ok(Json(UploadResponse { id: "aaa" })) } @@ -180,10 +213,15 @@ async fn fetch_preview( return Err(create_error!(NotFound)); } + // Ignore files that haven't been attached + if file.used_for.is_none() { + return Err(create_error!(NotFound)); + } + let hash = file.as_hash(&db).await?; // Only process image files - if !hash.content_type.starts_with("image/") { + if !matches!(hash.metadata, Metadata::Image { .. }) { return Ok( Redirect::permanent(&format!("/{tag}/{file_id}/{}", file.filename)).into_response(), ); @@ -197,15 +235,16 @@ async fn fetch_preview( let [w, h] = config.files.preview.get(tag).unwrap(); // Read the image and resize it + // TODO: use jxl_oxide to process image/jxl files let image = ImageReader::new(Cursor::new(data)) .with_guessed_format() .map_err(|_| create_error!(InternalError))? .decode() .map_err(|_| create_error!(InternalError))? + //.resize(width as u32, height as u32, image::imageops::FilterType::Gaussian) // resize is about 2.5x slower, - // thumb approximation doesn't have terrible quality so it's fine to stick with - // .resize(width as u32, height as u32, image::imageops::FilterType::Gaussian) - // aspect ratio is preserved when scaling + // thumbnail doesn't have terrible quality + // so we use thumbnail .thumbnail(*w as u32, *h as u32); // Encode it into WEBP @@ -253,6 +292,11 @@ async fn fetch_file( return Err(create_error!(NotFound)); } + // Ignore files that haven't been attached + if file.used_for.is_none() { + return Err(create_error!(NotFound)); + } + // Ensure filename is correct if file_name != file.filename { return Err(create_error!(NotFound)); diff --git a/crates/services/autumn/src/main.rs b/crates/services/autumn/src/main.rs index 5e6aa7db..76ead8df 100644 --- a/crates/services/autumn/src/main.rs +++ b/crates/services/autumn/src/main.rs @@ -11,6 +11,8 @@ use utoipa::{ use utoipa_scalar::{Scalar, Servable as ScalarServable}; mod api; +pub mod metadata; +pub mod mime_type; #[tokio::main] async fn main() -> Result<(), std::io::Error> { diff --git a/crates/services/autumn/src/metadata.rs b/crates/services/autumn/src/metadata.rs new file mode 100644 index 00000000..a34831c0 --- /dev/null +++ b/crates/services/autumn/src/metadata.rs @@ -0,0 +1,104 @@ +use std::{io::Cursor, os::unix::fs::MetadataExt}; + +use revolt_database::Metadata; +use revolt_result::{create_error, Result}; +use tempfile::NamedTempFile; + +/// Intersection of what infer can detect and what image-rs supports +/// +/// Note: imagesize crate also supports all of these, so we use that for quick size probing. +static SUPPORTED_IMAGE_MIME: [&str; 9] = [ + "image/avif", + "image/bmp", + "image/gif", + "image/vnd.microsoft.icon", + "image/jpeg", + "image/jxl", // not supported by image-rs but we shim it + "image/png", + "image/tiff", + "image/webp", +]; + +/// Image mime types that have EXIF data +static IMAGE_MIMES_WITH_EXIF: [&str; 4] = ["image/avif", "image/jpeg", "image/jxl", "image/tiff"]; + +/// Get the size of a temp file +pub fn temp_file_size(f: &NamedTempFile) -> Result { + // Check the size of the file + if let Ok(file) = f.reopen() { + if let Ok(metadata) = file.metadata() { + return Ok(metadata.size()); + } + } + + Err(create_error!(InternalError)) +} + +/// Generate metadata from file, using mime type as a hint +pub fn generate_metadata(f: &NamedTempFile, mime_type: &str) -> Metadata { + if SUPPORTED_IMAGE_MIME.contains(&mime_type) { + if let Ok(size) = imagesize::size(f.path()) + .inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}")) + { + if let (Ok(width), Ok(height)) = (size.width.try_into(), size.height.try_into()) { + return Metadata::Image { width, height }; + } + } + + Metadata::File + } else if mime_type.starts_with("video/") { + if let Ok(data) = ffprobe::ffprobe(f.path()) + .inspect_err(|err| tracing::error!("Failed to ffprobe file! {err:?}")) + { + // Use first valid stream + for stream in data.streams { + if let (Some(w), Some(h)) = (stream.width, stream.height) { + if let (Ok(width), Ok(height)) = (w.try_into(), h.try_into()) { + return Metadata::Video { width, height }; + } + } + } + + Metadata::File + } else { + Metadata::File + } + } else if mime_type.starts_with("audio/") { + Metadata::Audio + } else if mime_type == "plain/text" { + Metadata::Text + } else { + Metadata::File + } +} + +/// Subroutine to ensure data isn't corrupted +pub fn validate_from_metadata( + reader: Cursor>, + metadata: Metadata, + mime_type: &str, +) -> Metadata { + if let Metadata::Image { .. } = &metadata { + if mime_type == "image/jxl" { + // Check if we can read using jxl-oxide crate + if jxl_oxide::JxlImage::builder() + .read(reader) + .inspect_err(|err| tracing::error!("Failed to read JXL! {err:?}")) + .is_err() + { + return Metadata::File; + } + } else if matches!( + // Check if we can read using image-rs crate + image::ImageReader::new(reader) + .with_guessed_format() + .inspect_err(|err| tracing::error!("Failed to read image! {err:?}")) + .map(|f| f.decode()), + Err(_) | Ok(Err(_)) + ) { + return Metadata::File; + } + } + + metadata +} diff --git a/crates/services/autumn/src/mime_type.rs b/crates/services/autumn/src/mime_type.rs new file mode 100644 index 00000000..ad54e78c --- /dev/null +++ b/crates/services/autumn/src/mime_type.rs @@ -0,0 +1,36 @@ +use std::io::Read; + +use tempfile::NamedTempFile; + +/// Determine the mime type of the given temporary file and filename +pub fn determine_mime_type(f: &mut NamedTempFile, file_name: &str, file_size: u64) -> &'static str { + // Use magic signatures to determine mime type + let kind = infer::get_from_path(f.path()).expect("file read successfully"); + let mime_type = if let Some(kind) = kind { + kind.mime_type() + } else { + "application/octet-stream" + }; + + // Map any known conflicts where appropriate + let mime_type = if mime_type == "application/zip" && file_name.to_lowercase().ends_with(".apk") + { + "application/vnd.android.package-archive" + } else { + mime_type + }; + + // See if the file is actually just plain Unicode/ASCII text + if mime_type == "application/octet-stream" { + // don't check files over >= 500 kB + if file_size <= 500_000 { + let mut buf = String::new(); + if f.read_to_string(&mut buf).is_ok() { + // successfully read the file as UTF-8 + return "plain/text"; + } + } + } + + mime_type +} diff --git a/docker-compose.db.yml b/docker-compose.db.yml deleted file mode 100644 index 222a1790..00000000 --- a/docker-compose.db.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: "3.3" -services: - # Redis - redis: - image: eqalpha/keydb - ports: - - "6379:6379" - - # MongoDB - database: - image: mongo - ports: - - "27017:27017" - volumes: - - ./.data/db:/data/db diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index de56d7a9..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,72 +0,0 @@ -version: "3.3" -services: - # Redis - redis: - image: eqalpha/keydb - ports: - - "6379:6379" - - # MongoDB - database: - image: mongo - ports: - - "27017:27017" - volumes: - - ./.data/db:/data/db - - # MinIO - minio: - image: minio/minio - command: server /data - env_file: .env - volumes: - - ./.data/minio:/data - ports: - - "10000:9000" - restart: always - - # Mongo Express - mongo-express: - image: mongo-express - ports: - - "8081:8081" - environment: - - ME_CONFIG_MONGODB_SERVER=database - depends_on: - - database - - # Create buckets for minio. - createbuckets: - image: minio/mc - depends_on: - - minio - env_file: .env - entrypoint: > - /bin/sh -c " - /usr/bin/mc config host add minio http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD; - while ! /usr/bin/mc ready minio; do echo 'Waiting minio...' && sleep 1; done; - /usr/bin/mc mb minio/attachments; - /usr/bin/mc mb minio/avatars; - /usr/bin/mc mb minio/backgrounds; - /usr/bin/mc mb minio/icons; - /usr/bin/mc mb minio/banners; - /usr/bin/mc mb minio/emojis; - exit 0; - " - - # File server (autumn) - autumn: - image: ghcr.io/revoltchat/autumn:1.1.5 - env_file: .env - depends_on: - - createbuckets - ports: - - "3000:3000" - restart: always - - # Metadata and image proxy (january) - january: - image: ghcr.io/revoltchat/january:0.3.4 - ports: - - "7000:7000" - restart: always