feat: repository architecture for files crate w. added tests (#498)

.mise/tasks/test

@@ -2,4 +2,7 @@
 #MISE description="Test project"
 set -e
 
+: "${TEST_DB:=REFERENCE}"
+export TEST_DB
+
 cargo nextest run

Cargo.toml

@@ -14,9 +14,37 @@ redis23 = { package = "redis", version = "0.23.3", git = "https://github.com/rev
 #authifier = { package = "authifier", version = "1.0.10", path = "../authifier/crates/authifier" }
 #rocket_authifier = { package = "rocket_authifier", version = "1.0.10", path = "../authifier/crates/rocket_authifier" }
 
-# I'm 99% sure this is overloading the GitHub worker
-# hence builds have been failing since, let's just
-# disable it for now. In the future, we could use this
-# if we were rolling our own CI (that is now).
 [profile.release]
 lto = true
+
+[workspace.dependencies]
+# Async
+async-trait = "0.1.89"
+tokio = { version = "1.49.0", features = ["macros", "rt"] }
+
+# Error Handling
+anyhow = "1.0.100"
+thiserror = "2.0.18"
+
+# Other Utilities
+uuid = { version = "1.19.0", features = ["v4"] }
+
+# Axum (HTTP server)
+axum-macros = "0.4.1"
+axum_typed_multipart = "0.12.1"
+axum = { version = "0.7.5", features = ["multipart"] }
+tower-http = { version = "0.5.2", features = ["cors", "trace"] }
+
+# Image Processing
+jxl-oxide = { version = "0.12.5", features = ["image"] }
+image = "0.25.9"
+
+# OpenTelemetry
+tracing = "0.1.44"
+tracing-subscriber = { version = "0.3.22", features = [
+    "env-filter",
+] } # consider https://crates.io/crates/better-tracing
+opentelemetry = { version = "0.31.0", features = ["logs"] }
+opentelemetry_sdk = { version = "0.31.0", features = ["logs"] }
+opentelemetry-otlp = { version = "0.31.0", features = ["logs"] }
+opentelemetry-appender-tracing = { version = "0.31.1" }

crates/core/config/src/lib.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, path::Path};
 
 use cached::proc_macro::cached;
 use config::{Config, File, FileFormat};
@@ -94,10 +94,19 @@ static CONFIG_BUILDER: Lazy<RwLock<Config>> = Lazy::new(|| {
             }
         }
 
-        for path in CONFIG_SEARCH_PATHS {
-            if std::path::Path::new(path).exists() {
-                builder = builder.add_source(File::new(path, FileFormat::Toml));
+        let cwd = std::env::current_dir().unwrap();
+        let mut cwd: Option<&Path> = Some(&cwd);
+
+        while let Some(path) = cwd {
+            for config_path in CONFIG_SEARCH_PATHS {
+                let config_path = path.join(config_path);
+                if config_path.exists() {
+                    builder = builder
+                        .add_source(File::new(config_path.to_str().unwrap(), FileFormat::Toml));
+                }
             }
+
+            cwd = path.parent();
         }
 
         builder.build().unwrap()

crates/core/files/Cargo.toml

@@ -8,7 +8,13 @@ description = "Revolt Backend: S3 and encryption subroutines"
 repository = "https://github.com/stoatchat/stoatchat"
 
 [dependencies]
-tracing = "0.1"
+anyhow = { workspace = true }
+thiserror = { workspace = true }
+
+tracing = { workspace = true }
+
+tokio = { workspace = true }
+async-trait = { workspace = true }
 
 ffprobe = "0.4.0"
 imagesize = "0.13.0"
@@ -27,8 +33,8 @@ revolt-config = { version = "0.9.4", path = "../config", features = [
 revolt-result = { version = "0.9.4", path = "../result" }
 
 # image processing
-jxl-oxide = "0.8.1"
-image = { version = "0.25.2" }
+jxl-oxide = { workspace = true }
+image = { workspace = true }
 
 # svg rendering
 usvg = "0.44.0"
@@ -37,3 +43,6 @@ tiny-skia = "0.11.4"
 
 # encoding
 webp = "0.3.0"
+
+[dev-dependencies]
+uuid = { workspace = true }

crates/core/files/src/implementation/encryption_impl.rs

@@ -0,0 +1,75 @@
+use aes_gcm::{
+    aead::{Aead, AeadCore, AeadMutInPlace, OsRng},
+    Aes256Gcm, Key, KeyInit, Nonce,
+};
+use base64::{prelude::BASE64_STANDARD, Engine};
+
+use crate::EncryptionRepository;
+
+pub struct EncryptionKey {
+    key: String,
+}
+
+impl EncryptionKey {
+    pub async fn from_config() -> EncryptionKey {
+        EncryptionKey::new(revolt_config::config().await.files.encryption_key)
+    }
+
+    pub fn new(key: String) -> EncryptionKey {
+        EncryptionKey { key }
+    }
+
+    fn create_cipher(&self) -> Aes256Gcm {
+        let key = &BASE64_STANDARD
+            .decode(self.key.clone())
+            .expect("valid base64 string")[..];
+        let key: &Key<Aes256Gcm> = key.into();
+        Aes256Gcm::new(key)
+    }
+}
+
+impl EncryptionRepository for EncryptionKey {
+    fn decrypt_buffer(&self, mut buf: Vec<u8>, iv: &str) -> anyhow::Result<Vec<u8>> {
+        let iv = &BASE64_STANDARD.decode(iv).unwrap()[..];
+        let iv: &Nonce<typenum::consts::U12> = iv.into();
+
+        self.create_cipher()
+            .decrypt_in_place(iv, b"", &mut buf)
+            .map_err(|error| {
+                tracing::error!("{}", error);
+                anyhow::anyhow!("EncryptionRepository: decryption failed")
+            })?;
+
+        Ok(buf)
+    }
+
+    fn encrypt_buffer(&self, buf: &[u8]) -> anyhow::Result<(Vec<u8>, String)> {
+        let iv = Aes256Gcm::generate_nonce(&mut OsRng);
+
+        let buf = self.create_cipher().encrypt(&iv, buf).map_err(|error| {
+            tracing::error!("{}", error);
+            anyhow::anyhow!("EncryptionRepository: encryption failed")
+        })?;
+
+        Ok((buf, BASE64_STANDARD.encode(iv)))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_encrypt_and_decrypt() {
+        let encryption =
+            EncryptionKey::new("XkbJ8gBzrouQ+15Ri23xCC81+aZE26Z6+gXzglFxOD4=".to_string());
+
+        let buf: Vec<u8> = vec![67];
+        let (ciphertext, iv) = encryption.encrypt_buffer(&buf[..]).unwrap();
+        assert_eq!(ciphertext.len(), 17);
+
+        let plaintext = encryption.decrypt_buffer(ciphertext, &iv).unwrap();
+        assert_eq!(plaintext.len(), 1);
+        assert_eq!(plaintext[0], 67);
+    }
+}

crates/core/files/src/implementation/media_impl.rs

@@ -0,0 +1,294 @@
+use anyhow::Result;
+use image::{DynamicImage, ImageBuffer, ImageReader};
+use jxl_oxide::integration::JxlDecoder;
+use revolt_config::report_internal_error;
+use std::io::{BufRead, Read, Seek};
+use tempfile::NamedTempFile;
+use tiny_skia::Pixmap;
+
+use crate::{MediaError, MediaRepository};
+
+pub struct MediaImpl {
+    config: revolt_config::Files,
+}
+
+impl MediaImpl {
+    pub async fn from_config() -> MediaImpl {
+        MediaImpl {
+            config: revolt_config::config().await.files,
+        }
+    }
+
+    pub fn new(config: revolt_config::Files) -> MediaImpl {
+        MediaImpl { config }
+    }
+}
+
+impl MediaRepository for MediaImpl {
+    fn image_size(&self, f: &NamedTempFile) -> Option<(usize, usize)> {
+        if let Ok(size) = imagesize::size(f.path())
+            .inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}"))
+        {
+            Some((size.width, size.height))
+        } else {
+            None
+        }
+    }
+
+    fn image_size_vec(&self, v: &[u8], mime: &str) -> Option<(usize, usize)> {
+        match mime {
+            "image/svg+xml" => {
+                let tree =
+                    report_internal_error!(usvg::Tree::from_data(v, &Default::default())).ok()?;
+
+                let size = tree.size();
+                Some((size.width() as usize, size.height() as usize))
+            }
+            _ => {
+                if let Ok(size) = imagesize::blob_size(v)
+                    .inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}"))
+                {
+                    Some((size.width, size.height))
+                } else {
+                    None
+                }
+            }
+        }
+    }
+
+    fn decode_image<R: Read + BufRead + Seek>(
+        &self,
+        reader: &mut R,
+        mime: &str,
+    ) -> Result<DynamicImage, MediaError> {
+        match mime {
+            "image/jxl" => {
+                let decoder =
+                    JxlDecoder::new(reader).map_err(|e| MediaError::from(anyhow::anyhow!(e)))?;
+
+                DynamicImage::from_decoder(decoder)
+                    .map_err(|e| MediaError::from(anyhow::anyhow!(e)))
+            }
+            "image/svg+xml" => {
+                let mut buf = Vec::new();
+                reader
+                    .read_to_end(&mut buf)
+                    .map_err(|e| MediaError::from(anyhow::anyhow!(e)))?;
+
+                let tree: usvg::Tree = usvg::Tree::from_data(&buf, &Default::default())
+                    .map_err(|e| MediaError::from(anyhow::anyhow!(e)))?;
+
+                let size = tree.size();
+                let mut pixmap = Pixmap::new(size.width() as u32, size.height() as u32)
+                    .ok_or_else(|| MediaError::ImageProcessingFailed {
+                        cause: "failed to create Pixmap, likely zero sized".to_string(),
+                    })?;
+
+                let mut pixmap_mut = pixmap.as_mut();
+                resvg::render(&tree, Default::default(), &mut pixmap_mut);
+
+                Ok(DynamicImage::ImageRgba8(
+                    ImageBuffer::from_vec(
+                        size.width() as u32,
+                        size.height() as u32,
+                        pixmap.data().to_vec(),
+                    )
+                    .ok_or_else(|| MediaError::ImageProcessingFailed {
+                        cause: "buffer is not big enough".to_string(),
+                    })?,
+                ))
+            }
+            _ => {
+                let image: ImageReader<&mut R> = image::ImageReader::new(reader)
+                    .with_guessed_format()
+                    .map_err(|e| MediaError::from(anyhow::anyhow!(e)))?;
+
+                let image: Result<DynamicImage, MediaError> = image
+                    .decode()
+                    .map_err(|e| MediaError::from(anyhow::anyhow!(e)));
+
+                image
+            }
+        }
+    }
+
+    fn is_valid_image<R: Read + BufRead + Seek>(&self, reader: &mut R, mime: &str) -> bool {
+        match mime {
+            "image/jxl" => jxl_oxide::JxlImage::builder()
+                .read(reader)
+                .inspect_err(|err| tracing::error!("Failed to read JXL! {err:?}"))
+                .is_ok(),
+            _ => !matches!(
+                image::ImageReader::new(reader)
+                    .with_guessed_format()
+                    .inspect_err(|err| tracing::error!("Failed to read image! {err:?}"))
+                    .map(|f| f.decode()),
+                Err(_) | Ok(Err(_))
+            ),
+        }
+    }
+
+    fn create_thumbnail(&self, image: DynamicImage, tag: &str) -> Vec<u8> {
+        let [w, h] = self.config.preview.get(tag).unwrap();
+
+        let image = image.thumbnail(image.width().min(*w as u32), image.height().min(*h as u32));
+
+        let encoder = webp::Encoder::from_image(&image).expect("Could not create encoder.");
+        if self.config.webp_quality != 100.0 {
+            encoder.encode(self.config.webp_quality).to_vec()
+        } else {
+            encoder.encode_lossless().to_vec()
+        }
+    }
+
+    fn video_size(&self, f: &NamedTempFile) -> Option<(i64, i64)> {
+        if let Ok(data) = ffprobe::ffprobe(f.path())
+            .inspect_err(|err| tracing::error!("Failed to ffprobe file! {err:?}"))
+        {
+            for stream in data.streams {
+                if let (Some(w), Some(h)) = (stream.width, stream.height) {
+                    return Some((w, h));
+                }
+            }
+
+            None
+        } else {
+            None
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io::Cursor;
+
+    use crate::{MediaImpl, MediaRepository};
+
+    #[tokio::test]
+    async fn asset_test_jpeg() {
+        let media = MediaImpl::from_config().await;
+        let buf = include_bytes!("../../tests/assets/test.jpeg");
+        assert_eq!(media.image_size_vec(buf, "image/jpeg"), Some((655, 582)));
+
+        let mut reader = Cursor::new(buf);
+        let image = media.decode_image(&mut reader, "image/jpeg").unwrap();
+        media.create_thumbnail(image, "attachments");
+    }
+
+    #[tokio::test]
+    async fn asset_test_jpeg_extra_bytes() {
+        let media = MediaImpl::from_config().await;
+        let buf = [
+            &include_bytes!("../../tests/assets/test.jpeg")[..],
+            &[0u8; 16],
+        ]
+        .concat();
+        assert_eq!(media.image_size_vec(&buf, "image/jpeg"), Some((655, 582)));
+
+        let mut reader = Cursor::new(buf);
+        let image = media.decode_image(&mut reader, "image/jpeg").unwrap();
+        media.create_thumbnail(image, "emojis");
+    }
+
+    #[tokio::test]
+    async fn asset_test_png() {
+        let media = MediaImpl::from_config().await;
+        let buf = include_bytes!("../../tests/assets/test.png");
+        assert_eq!(media.image_size_vec(buf, "image/png"), Some((900, 900)));
+
+        let mut reader = Cursor::new(buf);
+        let image = media.decode_image(&mut reader, "image/png").unwrap();
+        media.create_thumbnail(image, "emojis");
+    }
+
+    #[tokio::test]
+    async fn asset_test_png_extra_bytes() {
+        let media = MediaImpl::from_config().await;
+        let buf = [
+            &include_bytes!("../../tests/assets/test.png")[..],
+            &[0u8; 16],
+        ]
+        .concat();
+        assert_eq!(media.image_size_vec(&buf, "image/png"), Some((900, 900)));
+
+        let mut reader = Cursor::new(buf);
+        let image = media.decode_image(&mut reader, "image/png").unwrap();
+        media.create_thumbnail(image, "emojis");
+    }
+
+    #[tokio::test]
+    async fn asset_test_corrupted_png() {
+        let media = MediaImpl::from_config().await;
+        let buf = include_bytes!("../../tests/assets/corrupted.png");
+        assert_eq!(media.image_size_vec(buf, "image/png"), Some((900, 900)));
+
+        let mut reader = Cursor::new(buf);
+        media.decode_image(&mut reader, "image/png").unwrap_err();
+    }
+
+    #[tokio::test]
+    async fn asset_test_animated_png() {
+        let media = MediaImpl::from_config().await;
+        let buf = include_bytes!("../../tests/assets/anim-icos.apng");
+        assert_eq!(media.image_size_vec(buf, "image/png"), Some((128, 128)));
+
+        let mut reader = Cursor::new(buf);
+        let image = media.decode_image(&mut reader, "image/png").unwrap();
+        media.create_thumbnail(image, "attachments");
+    }
+
+    #[tokio::test]
+    async fn asset_test_jxl() {
+        let media = MediaImpl::from_config().await;
+        let buf = include_bytes!("../../tests/assets/dice.jxl");
+        assert_eq!(media.image_size_vec(buf, "image/jxl"), Some((800, 600)));
+
+        let mut reader = Cursor::new(buf);
+        let image = media.decode_image(&mut reader, "image/jxl").unwrap();
+        media.create_thumbnail(image, "attachments");
+    }
+
+    #[tokio::test]
+    async fn asset_test_animated_jxl() {
+        let media = MediaImpl::from_config().await;
+        let buf = include_bytes!("../../tests/assets/anim-icos.jxl");
+        assert_eq!(media.image_size_vec(buf, "image/jxl"), Some((128, 128)));
+
+        let mut reader = Cursor::new(buf);
+        let image = media.decode_image(&mut reader, "image/jxl").unwrap();
+        media.create_thumbnail(image, "attachments");
+    }
+
+    #[tokio::test]
+    async fn asset_test_webp() {
+        let media = MediaImpl::from_config().await;
+        let buf = include_bytes!("../../tests/assets/dice.webp");
+        assert_eq!(media.image_size_vec(buf, "image/webp"), Some((800, 600)));
+
+        let mut reader = Cursor::new(buf);
+        let image = media.decode_image(&mut reader, "image/webp").unwrap();
+        media.create_thumbnail(image, "attachments");
+    }
+
+    #[tokio::test]
+    async fn asset_test_animated_webp() {
+        let media = MediaImpl::from_config().await;
+        let buf = include_bytes!("../../tests/assets/anim-icos.webp");
+        assert_eq!(media.image_size_vec(buf, "image/webp"), Some((128, 128)));
+
+        let mut reader = Cursor::new(buf);
+        let image = media.decode_image(&mut reader, "image/webp").unwrap();
+        media.create_thumbnail(image, "attachments");
+    }
+
+    #[tokio::test]
+    async fn asset_test_animated_gif() {
+        let media = MediaImpl::from_config().await;
+        let buf = include_bytes!("../../tests/assets/anim-icos.gif");
+        assert_eq!(media.image_size_vec(buf, "image/gif"), Some((128, 128)));
+
+        let mut reader = Cursor::new(buf);
+        let image = media.decode_image(&mut reader, "image/gif").unwrap();
+        media.create_thumbnail(image, "attachments");
+    }
+}

crates/core/files/src/implementation/mod.rs

@@ -0,0 +1,7 @@
+mod encryption_impl;
+mod media_impl;
+mod s3_impl;
+
+pub use encryption_impl::EncryptionKey;
+pub use media_impl::MediaImpl;
+pub use s3_impl::S3Storage;

crates/core/files/src/implementation/s3_impl.rs

@@ -0,0 +1,118 @@
+use std::io::Write;
+
+use anyhow::Context;
+use aws_sdk_s3::{
+    config::{Credentials, Region},
+    Client, Config,
+};
+use revolt_config::FilesS3;
+
+use crate::{EncryptionRepository, FileStorageRepository};
+
+pub struct S3Storage<ER: EncryptionRepository> {
+    client: Client,
+    encryption: ER,
+}
+
+impl<ER: EncryptionRepository> S3Storage<ER> {
+    pub async fn from_config(encryption: ER) -> S3Storage<ER> {
+        S3Storage::new(encryption, revolt_config::config().await.files.s3)
+    }
+
+    pub fn new(encryption: ER, s3_config: FilesS3) -> S3Storage<ER> {
+        let provider_name = "my-creds";
+        let creds = Credentials::new(
+            s3_config.access_key_id,
+            s3_config.secret_access_key,
+            None,
+            None,
+            provider_name,
+        );
+
+        let config = Config::builder()
+            .region(Region::new(s3_config.region))
+            .endpoint_url(s3_config.endpoint)
+            .force_path_style(s3_config.path_style_buckets)
+            .credentials_provider(creds)
+            .build();
+
+        S3Storage {
+            client: Client::from_conf(config),
+            encryption,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<ER: EncryptionRepository> FileStorageRepository for S3Storage<ER> {
+    async fn create_bucket(&self, bucket_id: &str) -> anyhow::Result<()> {
+        self.client
+            .create_bucket()
+            .bucket(bucket_id)
+            .send()
+            .await
+            .with_context(|| format!("failed to create bucket {bucket_id}"))?;
+
+        Ok(())
+    }
+
+    async fn fetch_and_decrypt_file(
+        &self,
+        bucket_id: &str,
+        path: &str,
+        iv: &str,
+    ) -> anyhow::Result<Vec<u8>> {
+        let mut object = self
+            .client
+            .get_object()
+            .bucket(bucket_id)
+            .key(path)
+            .send()
+            .await
+            .with_context(|| format!("failed to get object at {path} in {bucket_id}"))?;
+
+        let mut buf = vec![];
+        while let Some(bytes) = object.body.next().await {
+            let data = bytes?;
+            buf.write_all(&data)?;
+        }
+
+        if iv.is_empty() {
+            Ok(buf)
+        } else {
+            self.encryption.decrypt_buffer(buf, iv)
+        }
+    }
+
+    async fn encrypt_and_upload_file(
+        &self,
+        bucket_id: &str,
+        path: &str,
+        buf: &[u8],
+    ) -> anyhow::Result<String> {
+        let (buf, iv) = self.encryption.encrypt_buffer(buf)?;
+
+        self.client
+            .put_object()
+            .bucket(bucket_id)
+            .key(path)
+            .body(buf.into())
+            .send()
+            .await
+            .with_context(|| format!("failed to put object at {path} in {bucket_id}"))?;
+
+        Ok(iv)
+    }
+
+    async fn delete_file(&self, bucket_id: &str, path: &str) -> anyhow::Result<()> {
+        self.client
+            .delete_object()
+            .bucket(bucket_id)
+            .key(path)
+            .send()
+            .await
+            .with_context(|| format!("failed to delete object at {path} in {bucket_id}"))?;
+
+        Ok(())
+    }
+}

crates/core/files/src/lib.rs

@@ -1,293 +1,182 @@
-use std::io::{BufRead, Read, Seek, Write};
+mod implementation;
+mod repositories;
 
-use aes_gcm::{
-    aead::{AeadCore, AeadMutInPlace, OsRng},
-    Aes256Gcm, Key, KeyInit, Nonce,
-};
-use image::{DynamicImage, ImageBuffer};
-use revolt_config::{config, report_internal_error, FilesS3};
-use revolt_result::{create_error, Result};
+pub use implementation::*;
+pub use repositories::*;
 
-use aws_sdk_s3::{
-    config::{Credentials, Region},
-    Client, Config,
-};
+use std::io::{BufRead, Read, Seek};
+
+use image::DynamicImage;
+use revolt_config::{report_internal_error, Files, FilesLimit, FilesS3};
+use revolt_result::Result;
 
-use base64::prelude::*;
 use tempfile::NamedTempFile;
-use tiny_skia::Pixmap;
 
-/// Size of the authentication tag in the buffer
 pub const AUTHENTICATION_TAG_SIZE_BYTES: usize = 16;
 
-/// Create an S3 client
-pub fn create_client(s3_config: FilesS3) -> Client {
-    let provider_name = "my-creds";
-    let creds = Credentials::new(
-        s3_config.access_key_id,
-        s3_config.secret_access_key,
-        None,
-        None,
-        provider_name,
-    );
-
-    let config = Config::builder()
-        .region(Region::new(s3_config.region))
-        .endpoint_url(s3_config.endpoint)
-        .force_path_style(s3_config.path_style_buckets)
-        .credentials_provider(creds)
-        .build();
-
-    Client::from_conf(config)
-}
-
-/// Create an AES-256-GCM cipher
-pub fn create_cipher(key: &str) -> Aes256Gcm {
-    let key = &BASE64_STANDARD.decode(key).expect("valid base64 string")[..];
-    let key: &Key<Aes256Gcm> = key.into();
-    Aes256Gcm::new(key)
-}
-
 /// Fetch a file from S3 (and decrypt it)
-pub async fn fetch_from_s3(bucket_id: &str, path: &str, nonce: &str) -> Result<Vec<u8>> {
-    let config = config().await;
-    let client = create_client(config.files.s3);
-
-    // Send a request for the file
-    let mut obj =
-        report_internal_error!(client.get_object().bucket(bucket_id).key(path).send().await)?;
-
-    // Read the file from remote
-    let mut buf = vec![];
-    while let Some(bytes) = obj.body.next().await {
-        let data = report_internal_error!(bytes)?;
-        report_internal_error!(buf.write_all(&data))?;
-        // is there a more efficient way to do this?
-        // we just want the Vec<u8>
-    }
-
-    // File is not encrypted
-    if nonce.is_empty() {
-        return Ok(buf);
-    }
-
-    // Recover nonce as bytes
-    let nonce = &BASE64_STANDARD.decode(nonce).unwrap()[..];
-    let nonce: &Nonce<typenum::consts::U12> = nonce.into();
-
-    // Decrypt the file
-    create_cipher(&config.files.encryption_key)
-        .decrypt_in_place(nonce, b"", &mut buf)
-        .map_err(|_| create_error!(InternalError))?;
-
-    // Remove the authentication tag bytes that were added during encryption
-    buf.truncate(buf.len() - AUTHENTICATION_TAG_SIZE_BYTES);
-
-    Ok(buf)
+pub async fn fetch_from_s3(bucket_id: &str, path: &str, iv: &str) -> Result<Vec<u8>> {
+    let encryption = implementation::EncryptionKey::from_config().await;
+    let storage = implementation::S3Storage::from_config(encryption).await;
+    report_internal_error!(storage.fetch_and_decrypt_file(bucket_id, path, iv).await)
 }
 
 /// Encrypt and upload a file to S3 (returning its nonce/IV)
 pub async fn upload_to_s3(bucket_id: &str, path: &str, buf: &[u8]) -> Result<String> {
-    let config = config().await;
-    let client = create_client(config.files.s3);
-
-    // Generate a nonce
-    let nonce = Aes256Gcm::generate_nonce(&mut OsRng);
-
-    // Extend the buffer for in-place encryption
-    let mut buf = [buf, &[0; AUTHENTICATION_TAG_SIZE_BYTES]].concat();
-
-    // Encrypt the file in place
-    create_cipher(&config.files.encryption_key)
-        .encrypt_in_place(&nonce, b"", &mut buf)
-        .map_err(|_| create_error!(InternalError))?;
-
-    // Upload the file to remote
-    report_internal_error!(
-        client
-            .put_object()
-            .bucket(bucket_id)
-            .key(path)
-            .body(buf.into())
-            .send()
-            .await
-    )?;
-
-    Ok(BASE64_STANDARD.encode(nonce))
+    let encryption = implementation::EncryptionKey::from_config().await;
+    let storage = implementation::S3Storage::from_config(encryption).await;
+    report_internal_error!(storage.encrypt_and_upload_file(bucket_id, path, buf).await)
 }
 
 /// Delete a file from S3 by path
 pub async fn delete_from_s3(bucket_id: &str, path: &str) -> Result<()> {
-    let config = config().await;
-    let client = create_client(config.files.s3);
-
-    report_internal_error!(
-        client
-            .delete_object()
-            .bucket(bucket_id)
-            .key(path)
-            .send()
-            .await
-    )?;
-
-    Ok(())
+    let encryption = implementation::EncryptionKey::from_config().await;
+    let storage = implementation::S3Storage::from_config(encryption).await;
+    report_internal_error!(storage.delete_file(bucket_id, path).await)
 }
 
 /// Determine size of image at temp file
 pub fn image_size(f: &NamedTempFile) -> Option<(usize, usize)> {
-    if let Ok(size) = imagesize::size(f.path())
-        .inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}"))
-    {
-        Some((size.width, size.height))
-    } else {
-        None
-    }
+    let media = MediaImpl::new(Files {
+        blocked_mime_types: Default::default(),
+        clamd_host: Default::default(),
+        encryption_key: Default::default(),
+        limit: FilesLimit {
+            max_mega_pixels: 0,
+            max_pixel_side: 0,
+            min_file_size: 0,
+            min_resolution: [0, 0],
+        },
+        preview: Default::default(),
+        s3: FilesS3 {
+            access_key_id: Default::default(),
+            default_bucket: Default::default(),
+            endpoint: Default::default(),
+            path_style_buckets: Default::default(),
+            region: Default::default(),
+            secret_access_key: Default::default(),
+        },
+        scan_mime_types: Default::default(),
+        webp_quality: Default::default(),
+    });
+
+    media.image_size(f)
 }
 
 /// Determine size of image with buffer
 pub fn image_size_vec(v: &[u8], mime: &str) -> Option<(usize, usize)> {
-    match mime {
-        "image/svg+xml" => {
-            let tree =
-                report_internal_error!(usvg::Tree::from_data(v, &Default::default())).ok()?;
+    let media = MediaImpl::new(Files {
+        blocked_mime_types: Default::default(),
+        clamd_host: Default::default(),
+        encryption_key: Default::default(),
+        limit: FilesLimit {
+            max_mega_pixels: 0,
+            max_pixel_side: 0,
+            min_file_size: 0,
+            min_resolution: [0, 0],
+        },
+        preview: Default::default(),
+        s3: FilesS3 {
+            access_key_id: Default::default(),
+            default_bucket: Default::default(),
+            endpoint: Default::default(),
+            path_style_buckets: Default::default(),
+            region: Default::default(),
+            secret_access_key: Default::default(),
+        },
+        scan_mime_types: Default::default(),
+        webp_quality: Default::default(),
+    });
 
-            let size = tree.size();
-            Some((size.width() as usize, size.height() as usize))
-        }
-        _ => {
-            if let Ok(size) = imagesize::blob_size(v)
-                .inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}"))
-            {
-                Some((size.width, size.height))
-            } else {
-                None
-            }
-        }
-    }
+    media.image_size_vec(v, mime)
 }
 
 /// Determine size of video at temp file
 pub fn video_size(f: &NamedTempFile) -> Option<(i64, i64)> {
-    if let Ok(data) = ffprobe::ffprobe(f.path())
-        .inspect_err(|err| tracing::error!("Failed to ffprobe file! {err:?}"))
-    {
-        // Use first valid stream
-        for stream in data.streams {
-            if let (Some(w), Some(h)) = (stream.width, stream.height) {
-                return Some((w, h));
-            }
-        }
+    let media = MediaImpl::new(Files {
+        blocked_mime_types: Default::default(),
+        clamd_host: Default::default(),
+        encryption_key: Default::default(),
+        limit: FilesLimit {
+            max_mega_pixels: 0,
+            max_pixel_side: 0,
+            min_file_size: 0,
+            min_resolution: [0, 0],
+        },
+        preview: Default::default(),
+        s3: FilesS3 {
+            access_key_id: Default::default(),
+            default_bucket: Default::default(),
+            endpoint: Default::default(),
+            path_style_buckets: Default::default(),
+            region: Default::default(),
+            secret_access_key: Default::default(),
+        },
+        scan_mime_types: Default::default(),
+        webp_quality: Default::default(),
+    });
 
-        None
-    } else {
-        None
-    }
+    media.video_size(f)
 }
 
 /// Decode image from reader
 pub fn decode_image<R: Read + BufRead + Seek>(reader: &mut R, mime: &str) -> Result<DynamicImage> {
-    match mime {
-        // Read image using jxl-oxide crate
-        "image/jxl" => {
-            let jxl_image = report_internal_error!(jxl_oxide::JxlImage::builder().read(reader))?;
-            if let Ok(frame) = jxl_image.render_frame(0) {
-                match frame.color_channels().len() {
-                    3 => Ok(DynamicImage::ImageRgb8(
-                        DynamicImage::ImageRgb32F(
-                            ImageBuffer::from_vec(
-                                jxl_image.width(),
-                                jxl_image.height(),
-                                frame.image().buf().to_vec(),
-                            )
-                            .ok_or_else(|| create_error!(ImageProcessingFailed))?,
-                        )
-                        .to_rgb8(),
-                    )),
-                    4 => Ok(DynamicImage::ImageRgba8(
-                        DynamicImage::ImageRgba32F(
-                            ImageBuffer::from_vec(
-                                jxl_image.width(),
-                                jxl_image.height(),
-                                frame.image().buf().to_vec(),
-                            )
-                            .ok_or_else(|| create_error!(ImageProcessingFailed))?,
-                        )
-                        .to_rgba8(),
-                    )),
-                    _ => Err(create_error!(ImageProcessingFailed)),
-                }
-            } else {
-                Err(create_error!(ImageProcessingFailed))
-            }
-        }
-        // Read image using resvg
-        "image/svg+xml" => {
-            // usvg doesn't support Read trait so copy to buffer
-            let mut buf = Vec::new();
-            report_internal_error!(reader.read_to_end(&mut buf))?;
+    let media = MediaImpl::new(Files {
+        blocked_mime_types: Default::default(),
+        clamd_host: Default::default(),
+        encryption_key: Default::default(),
+        limit: FilesLimit {
+            max_mega_pixels: 0,
+            max_pixel_side: 0,
+            min_file_size: 0,
+            min_resolution: [0, 0],
+        },
+        preview: Default::default(),
+        s3: FilesS3 {
+            access_key_id: Default::default(),
+            default_bucket: Default::default(),
+            endpoint: Default::default(),
+            path_style_buckets: Default::default(),
+            region: Default::default(),
+            secret_access_key: Default::default(),
+        },
+        scan_mime_types: Default::default(),
+        webp_quality: Default::default(),
+    });
 
-            let tree = report_internal_error!(usvg::Tree::from_data(&buf, &Default::default()))?;
-            let size = tree.size();
-            let mut pixmap = Pixmap::new(size.width() as u32, size.height() as u32)
-                .ok_or_else(|| create_error!(ImageProcessingFailed))?;
-
-            let mut pixmap_mut = pixmap.as_mut();
-            resvg::render(&tree, Default::default(), &mut pixmap_mut);
-
-            Ok(DynamicImage::ImageRgba8(
-                ImageBuffer::from_vec(
-                    size.width() as u32,
-                    size.height() as u32,
-                    pixmap.data().to_vec(),
-                )
-                .ok_or_else(|| create_error!(ImageProcessingFailed))?,
-            ))
-        }
-        // Check if we can read using image-rs crate
-        _ => report_internal_error!(report_internal_error!(
-            image::ImageReader::new(reader).with_guessed_format()
-        )?
-        .decode()),
-    }
+    report_internal_error!(media.decode_image(reader, mime))
 }
 
 /// Check whether given reader has a valid image
 pub fn is_valid_image<R: Read + BufRead + Seek>(reader: &mut R, mime: &str) -> bool {
-    match mime {
-        // Check if we can read using jxl-oxide crate
-        "image/jxl" => jxl_oxide::JxlImage::builder()
-            .read(reader)
-            .inspect_err(|err| tracing::error!("Failed to read JXL! {err:?}"))
-            .is_ok(),
-        // Check if we can read using image-rs crate
-        _ => !matches!(
-            image::ImageReader::new(reader)
-                .with_guessed_format()
-                .inspect_err(|err| tracing::error!("Failed to read image! {err:?}"))
-                .map(|f| f.decode()),
-            Err(_) | Ok(Err(_))
-        ),
-    }
+    let media = MediaImpl::new(Files {
+        blocked_mime_types: Default::default(),
+        clamd_host: Default::default(),
+        encryption_key: Default::default(),
+        limit: FilesLimit {
+            max_mega_pixels: 0,
+            max_pixel_side: 0,
+            min_file_size: 0,
+            min_resolution: [0, 0],
+        },
+        preview: Default::default(),
+        s3: FilesS3 {
+            access_key_id: Default::default(),
+            default_bucket: Default::default(),
+            endpoint: Default::default(),
+            path_style_buckets: Default::default(),
+            region: Default::default(),
+            secret_access_key: Default::default(),
+        },
+        scan_mime_types: Default::default(),
+        webp_quality: Default::default(),
+    });
+
+    media.is_valid_image(reader, mime)
 }
 
 /// Create thumbnail from given image
 pub async fn create_thumbnail(image: DynamicImage, tag: &str) -> Vec<u8> {
-    // Load configuration
-    let config = config().await;
-    let [w, h] = config.files.preview.get(tag).unwrap();
-
-    // Create thumbnail
-    //.resize(width as u32, height as u32, image::imageops::FilterType::Gaussian)
-    // resize is about 2.5x slower,
-    // thumbnail doesn't have terrible quality
-    // so we use thumbnail
-    let image = image.thumbnail(image.width().min(*w as u32), image.height().min(*h as u32));
-
-    // Encode it into WEBP
-    let encoder = webp::Encoder::from_image(&image).expect("Could not create encoder.");
-    if config.files.webp_quality != 100.0 {
-        encoder.encode(config.files.webp_quality).to_vec()
-    } else {
-        encoder.encode_lossless().to_vec()
-    }
+    let media = MediaImpl::from_config().await;
+    media.create_thumbnail(image, tag)
 }

crates/core/files/src/repositories/encryption_repository.rs

@@ -0,0 +1,6 @@
+use anyhow::Result;
+
+pub trait EncryptionRepository: Send + Sync + 'static {
+    fn decrypt_buffer(&self, buf: Vec<u8>, iv: &str) -> anyhow::Result<Vec<u8>>;
+    fn encrypt_buffer(&self, buf: &[u8]) -> Result<(Vec<u8>, String)>;
+}

crates/core/files/src/repositories/file_storage_repository.rs

@@ -0,0 +1,22 @@
+use anyhow::Result;
+
+#[async_trait::async_trait]
+pub trait FileStorageRepository: Send + Sync + 'static {
+    async fn create_bucket(&self, bucket_id: &str) -> anyhow::Result<()>;
+
+    async fn fetch_and_decrypt_file(
+        &self,
+        bucket_id: &str,
+        path: &str,
+        iv: &str,
+    ) -> Result<Vec<u8>>;
+
+    async fn encrypt_and_upload_file(
+        &self,
+        bucket_id: &str,
+        path: &str,
+        buf: &[u8],
+    ) -> Result<String>;
+
+    async fn delete_file(&self, bucket_id: &str, path: &str) -> Result<()>;
+}

crates/core/files/src/repositories/media_repository.rs

@@ -0,0 +1,30 @@
+use anyhow::Result;
+use image::DynamicImage;
+use std::io::{BufRead, Read, Seek};
+use tempfile::NamedTempFile;
+use thiserror::Error;
+
+pub trait MediaRepository: Send + Sync + 'static {
+    fn image_size(&self, f: &NamedTempFile) -> Option<(usize, usize)>;
+    fn image_size_vec(&self, v: &[u8], mime: &str) -> Option<(usize, usize)>;
+
+    fn decode_image<R: Read + BufRead + Seek>(
+        &self,
+        reader: &mut R,
+        mime: &str,
+    ) -> Result<DynamicImage, MediaError>;
+
+    fn is_valid_image<R: Read + BufRead + Seek>(&self, reader: &mut R, mime: &str) -> bool;
+
+    fn create_thumbnail(&self, image: DynamicImage, tag: &str) -> Vec<u8>;
+
+    fn video_size(&self, f: &NamedTempFile) -> Option<(i64, i64)>;
+}
+
+#[derive(Debug, Error)]
+pub enum MediaError {
+    #[error("image processing failed because {cause}")]
+    ImageProcessingFailed { cause: String },
+    #[error(transparent)]
+    Unknown(#[from] anyhow::Error),
+}

crates/core/files/src/repositories/mod.rs

@@ -0,0 +1,7 @@
+mod encryption_repository;
+mod file_storage_repository;
+mod media_repository;
+
+pub use encryption_repository::EncryptionRepository;
+pub use file_storage_repository::FileStorageRepository;
+pub use media_repository::{MediaError, MediaRepository};

crates/core/files/tests/integration_test.rs

@@ -0,0 +1,31 @@
+use std::io::Cursor;
+
+use revolt_files::{EncryptionKey, FileStorageRepository, MediaImpl, MediaRepository, S3Storage};
+
+#[tokio::test]
+async fn test_image_roundtrip_png() {
+    let media = MediaImpl::from_config().await;
+    let encryption = EncryptionKey::from_config().await;
+    let s3 = S3Storage::from_config(encryption).await;
+
+    let buf = include_bytes!("./assets/test.png");
+    let bucket_id = uuid::Uuid::new_v4().to_string();
+
+    s3.create_bucket(&bucket_id).await.unwrap();
+
+    let mut reader = Cursor::new(buf);
+    media.decode_image(&mut reader, "image/png").unwrap();
+
+    let iv = s3
+        .encrypt_and_upload_file(&bucket_id, "/my-file", buf)
+        .await
+        .unwrap();
+
+    let buf = s3
+        .fetch_and_decrypt_file(&bucket_id, "/my-file", &iv)
+        .await
+        .unwrap();
+
+    let mut reader = Cursor::new(buf);
+    media.decode_image(&mut reader, "image/png").unwrap();
+}

crates/core/files/tests/s3_test.rs

@@ -0,0 +1,42 @@
+use revolt_files::{EncryptionKey, FileStorageRepository, S3Storage};
+
+#[tokio::test]
+async fn test_upload_and_download() {
+    let encryption = EncryptionKey::from_config().await;
+    let s3 = S3Storage::from_config(encryption).await;
+
+    let buf = [67];
+    let bucket_id = uuid::Uuid::new_v4().to_string();
+
+    s3.create_bucket(&bucket_id).await.unwrap();
+
+    let iv = s3
+        .encrypt_and_upload_file(&bucket_id, "/my-file", &buf)
+        .await
+        .unwrap();
+
+    let buf = s3
+        .fetch_and_decrypt_file(&bucket_id, "/my-file", &iv)
+        .await
+        .unwrap();
+
+    assert_eq!(buf.len(), 1);
+    assert_eq!(buf[0], 67);
+}
+
+#[tokio::test]
+async fn test_upload_and_delete() {
+    let encryption = EncryptionKey::from_config().await;
+    let s3 = S3Storage::from_config(encryption).await;
+
+    let buf = [67];
+    let bucket_id = uuid::Uuid::new_v4().to_string();
+
+    s3.create_bucket(&bucket_id).await.unwrap();
+
+    s3.encrypt_and_upload_file(&bucket_id, "/my-file", &buf)
+        .await
+        .unwrap();
+
+    s3.delete_file(&bucket_id, "/my-file").await.unwrap();
+}

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "stable"
+channel = "1.92.0"