feat(services/autumn): download and preview files

crates/services/autumn/Cargo.toml

@@ -4,9 +4,18 @@ version = "0.7.14"
 edition = "2021"
 
 [dependencies]
+# Media processing
+webp = "0.3.0"
+image = "0.25.2"
+
+# Utility
+lazy_static = "1.5.0"
+moka = { version = "0.12.8", features = ["future"] }
+
 # Serialisation
-serde = { version = "1.0", features = ["derive"] }
+strum_macros = "0.26.4"
 serde_json = "1.0.68"
+serde = { version = "1.0", features = ["derive"] }
 
 # Async runtime
 tokio = { version = "1.0", features = ["full"] }
@@ -16,7 +25,9 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 # Core crates
+revolt-files = { version = "0.7.16", path = "../../core/files" }
 revolt-config = { version = "0.7.16", path = "../../core/config" }
+revolt-database = { version = "0.7.16", path = "../../core/database" }
 revolt-result = { version = "0.7.16", path = "../../core/result", features = [
     "utoipa",
     "axum",
@@ -24,6 +35,7 @@ revolt-result = { version = "0.7.16", path = "../../core/result", features = [
 
 # Axum / web server
 tempfile = "3.12.0"
+axum-macros = "0.4.1"
 axum_typed_multipart = "0.12.1"
 axum = { version = "0.7.5", features = ["multipart"] }
 

crates/services/autumn/src/api.rs

@@ -1,26 +1,57 @@
+use std::{io::Cursor, time::Duration};
+
 use axum::{
-    extract::{DefaultBodyLimit, Multipart, Path},
-    http::StatusCode,
-    response::{IntoResponse, Response},
+    extract::{DefaultBodyLimit, Path, State},
+    http::header,
+    response::{IntoResponse, Redirect, Response},
     routing::{get, post},
     Json, Router,
 };
 use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart};
+use image::ImageReader;
+use lazy_static::lazy_static;
 use revolt_config::config;
+use revolt_database::{Database, FileHash};
+use revolt_files::fetch_from_s3;
+use revolt_result::{create_error, Result};
 use serde::{Deserialize, Serialize};
+use tempfile::NamedTempFile;
 use utoipa::ToSchema;
 
-use tempfile::NamedTempFile;
-
-pub async fn router() -> Router {
+/// Build the API router
+pub async fn router() -> Router<Database> {
     let config = config().await;
 
-    Router::new().route("/", get(root)).route(
-        "/:tag",
-        post(upload_file).layer(DefaultBodyLimit::max(
-            config.features.limits.global.body_limit_size,
-        )),
-    )
+    Router::new()
+        .route("/", get(root))
+        .route(
+            "/:tag",
+            post(upload_file).layer(DefaultBodyLimit::max(
+                config.features.limits.global.body_limit_size,
+            )),
+        )
+        .route("/:tag/:file_id", get(fetch_preview))
+        .route("/:tag/:file_id/:file_name", get(fetch_file))
+}
+
+lazy_static! {
+    /// Short-lived file cache to allow us to populate different CDN regions without increasing bandwidth to S3 provider
+    /// Uploads will also be stored here to prevent immediately queued downloads from doing the entire round-trip
+    static ref S3_CACHE: moka::future::Cache<String, Result<Vec<u8>>> = moka::future::Cache::builder()
+        .max_capacity(10_000) // TODO config
+        .time_to_live(Duration::from_secs(60)) // TODO config
+        .build();
+}
+
+/// Retrieve hash information and file data by given hash
+async fn retrieve_file_by_hash(hash: &FileHash) -> Result<Vec<u8>> {
+    if let Some(data) = S3_CACHE.get(&hash.id).await {
+        data
+    } else {
+        let data = fetch_from_s3(&hash.bucket_id, &hash.path, &hash.iv).await;
+        S3_CACHE.insert(hash.id.to_owned(), data.clone()).await;
+        data
+    }
 }
 
 /// Successful root response
@@ -49,7 +80,7 @@ async fn root() -> Json<RootResponse> {
 }
 
 /// Available tags to upload to
-#[derive(Deserialize, Debug, ToSchema)]
+#[derive(Deserialize, Debug, ToSchema, strum_macros::IntoStaticStr)]
 #[allow(non_camel_case_types)]
 pub enum Tag {
     attachments,
@@ -106,8 +137,14 @@ async fn upload_file(
     Ok(Json(UploadResponse { id: "aaa" }))
 }
 
+/// Header value used for cache control
+pub static CACHE_CONTROL: &str = "public, max-age=604800, must-revalidate";
+
 /// Fetch preview of file
 ///
+/// This route will only return image content.
+/// For all other file types, please use the fetch route (you will receive a redirect if you try to use this route anyways!).
+///
 /// Depending on the given tag, the file will be re-processed to fit the criteria:
 ///
 /// | Tag | Image Resolution <sup>†</sup> |
@@ -132,18 +169,72 @@ async fn upload_file(
     ),
 )]
 async fn fetch_preview(
-    Path(tag): Path<Tag>,
-    Path(file_id): Path<String>,
-) -> axum::response::Result<Response> {
-    todo!()
+    State(db): State<Database>,
+    Path((tag, file_id)): Path<(Tag, String)>,
+) -> Result<Response> {
+    let tag: &'static str = tag.into();
+    let file = db.fetch_attachment(tag, &file_id).await?;
+
+    // Ignore deleted files
+    if file.deleted.is_some_and(|v| v) {
+        return Err(create_error!(NotFound));
+    }
+
+    let hash = file.as_hash(&db).await?;
+
+    // Only process image files
+    if !hash.content_type.starts_with("image/") {
+        return Ok(
+            Redirect::permanent(&format!("/{tag}/{file_id}/{}", file.filename)).into_response(),
+        );
+    }
+
+    // Original image data
+    let data = retrieve_file_by_hash(&hash).await?;
+
+    // Dimensions we need to resize to
+    let config = config().await;
+    let [w, h] = config.files.preview.get(tag).unwrap();
+
+    // Read the image and resize it
+    let image = ImageReader::new(Cursor::new(data))
+        .with_guessed_format()
+        .map_err(|_| create_error!(InternalError))?
+        .decode()
+        .map_err(|_| create_error!(InternalError))?
+        // resize is about 2.5x slower,
+        //  thumb approximation doesn't have terrible quality so it's fine to stick with
+        // .resize(width as u32, height as u32, image::imageops::FilterType::Gaussian)
+        // aspect ratio is preserved when scaling
+        .thumbnail(*w as u32, *h as u32);
+
+    // Encode it into WEBP
+    let encoder = webp::Encoder::from_image(&image).expect("Could not create encoder.");
+    let data = if config.files.webp_quality != 100.0 {
+        encoder.encode(config.files.webp_quality).to_vec()
+    } else {
+        encoder.encode_lossless().to_vec()
+    };
+
+    Ok((
+        [
+            (header::CONTENT_TYPE, "image/webp"),
+            (header::CONTENT_DISPOSITION, "inline"),
+            (header::CACHE_CONTROL, CACHE_CONTROL),
+        ],
+        data,
+    )
+        .into_response())
 }
 
 /// Fetch original file
+///
+/// Content disposition header will be set to 'attachment' to prevent browser from rendering anything.
 #[utoipa::path(
     get,
     path = "/{tag}/{file_id}/{file_name}",
     responses(
-        (status = 200, description = "Generated preview", body = Vec<u8>)
+        (status = 200, description = "Original file", body = Vec<u8>)
     ),
     params(
         ("tag" = Tag, Path, description = "Tag to fetch from (e.g. attachments, icons, ...)"),
@@ -152,9 +243,31 @@ async fn fetch_preview(
     ),
 )]
 async fn fetch_file(
-    Path(tag): Path<Tag>,
-    Path(file_id): Path<String>,
-    Path(file_name): Path<String>,
-) -> axum::response::Result<Response> {
-    todo!()
+    State(db): State<Database>,
+    Path((tag, file_id, file_name)): Path<(Tag, String, String)>,
+) -> Result<Response> {
+    let file = db.fetch_attachment(tag.into(), &file_id).await?;
+
+    // Ignore deleted files
+    if file.deleted.is_some_and(|v| v) {
+        return Err(create_error!(NotFound));
+    }
+
+    // Ensure filename is correct
+    if file_name != file.filename {
+        return Err(create_error!(NotFound));
+    }
+
+    let hash = file.as_hash(&db).await?;
+    retrieve_file_by_hash(&hash).await.map(|data| {
+        (
+            [
+                (header::CONTENT_TYPE, hash.content_type),
+                (header::CONTENT_DISPOSITION, "attachment".to_owned()),
+                (header::CACHE_CONTROL, CACHE_CONTROL.to_owned()),
+            ],
+            data,
+        )
+            .into_response()
+    })
 }

crates/services/autumn/src/main.rs

@@ -2,6 +2,7 @@ use std::net::{Ipv4Addr, SocketAddr};
 
 use axum::Router;
 
+use revolt_database::DatabaseInfo;
 use tokio::net::TcpListener;
 use utoipa::{
     openapi::security::{ApiKey, ApiKeyValue, SecurityScheme},
@@ -55,10 +56,14 @@ async fn main() -> Result<(), std::io::Error> {
         }
     }
 
+    // Connect to the database
+    let db = DatabaseInfo::Auto.connect().await.unwrap();
+
     // Configure Axum and router
     let app = Router::new()
         .merge(Scalar::with_url("/scalar", ApiDoc::openapi()))
-        .nest("/", api::router().await);
+        .nest("/", api::router().await)
+        .with_state(db);
 
     // Configure TCP listener and bind
     let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, 3000));