feat(services/autumn): download and preview files

This commit is contained in:
Paul Makles
2024-09-01 13:58:11 +01:00
parent 78757ac7f1
commit ebbbb5e174
21 changed files with 1119 additions and 81 deletions

View File

@@ -1,6 +1,5 @@
{
"editor.formatOnSave": true,
"rust-analyzer.checkOnSave.command": "clippy",
"nixEnvSelector.suggestion": false,
"nixEnvSelector.nixFile": "${workspaceRoot}/default.nix"
"nixEnvSelector.suggestion": false
}

694
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -88,6 +88,8 @@ max_concurrent_connections = 50
# Encryption key for stored files
# Generate your own key using `openssl rand -base64 32`
encryption_key = "qcuMA+ssxhMyKaNAKBGFfryfFtUH8NDlamQyDwGW6fU="
# Quality used for lossy WebP previews (set to 100 for lossless)
webp_quality = 80.0
[files.limit]
# Minimum image resolution
@@ -118,13 +120,13 @@ emojis = [128, 128]
# - default_bucket matches the name of the bucket you've created
# S3 protocol endpoint
endpoint = ""
endpoint = "http://minio:9000"
# S3 region name
region = ""
region = "minio"
# S3 protocol key ID
access_key_id = ""
access_key_id = "minioautumn"
# S3 protocol access key
secret_access_key = ""
secret_access_key = "minioautumn"
# Bucket to upload to by default
default_bucket = "revolt-uploads"

View File

@@ -126,6 +126,31 @@ pub struct Api {
pub workers: ApiWorkers,
}
#[derive(Deserialize, Debug, Clone)]
pub struct FilesLimit {
pub min_resolution: [usize; 2],
pub max_mega_pixels: usize,
pub max_pixel_side: usize,
}
#[derive(Deserialize, Debug, Clone)]
pub struct FilesS3 {
pub endpoint: String,
pub region: String,
pub access_key_id: String,
pub secret_access_key: String,
pub default_bucket: String,
}
#[derive(Deserialize, Debug, Clone)]
pub struct Files {
pub encryption_key: String,
pub webp_quality: f32,
pub limit: FilesLimit,
pub preview: HashMap<String, [usize; 2]>,
pub s3: FilesS3,
}
#[derive(Deserialize, Debug, Clone)]
pub struct GlobalLimits {
pub group_size: usize,
@@ -186,6 +211,7 @@ pub struct Settings {
pub database: Database,
pub hosts: Hosts,
pub api: Api,
pub files: Files,
pub features: Features,
pub sentry: Sentry,
}

View File

@@ -3,7 +3,7 @@ use std::{collections::HashMap, sync::Arc};
use futures::lock::Mutex;
use crate::{
Bot, Channel, ChannelCompositeKey, ChannelUnread, Emoji, File, Invite, Member,
Bot, Channel, ChannelCompositeKey, ChannelUnread, Emoji, File, FileHash, Invite, Member,
MemberCompositeKey, Message, RatelimitEvent, Report, Server, ServerBan, Snapshot, User,
UserSettings, Webhook,
};
@@ -18,6 +18,7 @@ database_derived!(
pub channel_unreads: Arc<Mutex<HashMap<ChannelCompositeKey, ChannelUnread>>>,
pub channel_webhooks: Arc<Mutex<HashMap<String, Webhook>>>,
pub emojis: Arc<Mutex<HashMap<String, Emoji>>>,
pub file_hashes: Arc<Mutex<HashMap<String, FileHash>>>,
pub files: Arc<Mutex<HashMap<String, File>>>,
pub messages: Arc<Mutex<HashMap<String, Message>>>,
pub ratelimit_events: Arc<Mutex<HashMap<String, RatelimitEvent>>>,

View File

@@ -0,0 +1,5 @@
mod model;
mod ops;
pub use model::*;
pub use ops::*;

View File

@@ -0,0 +1,49 @@
use iso8601_timestamp::Timestamp;
auto_derived_partial!(
/// File hash
pub struct FileHash {
/// Sha256 hash of the file
#[serde(rename = "_id")]
pub id: String,
/// Sha256 hash of file after it has been processed
pub processed_hash: String,
/// When this file was created in system
pub created_at: Timestamp,
/// The bucket this file is stored in
pub bucket_id: String,
/// The path at which this file exists in
pub path: String,
/// Cryptographic nonce used to encrypt this file
pub iv: String,
/// Parsed metadata of this file
pub metadata: Metadata,
/// Raw content type of this file
pub content_type: String,
/// Size of this file (in bytes)
pub size: isize,
},
"PartialFile"
);
auto_derived!(
/// Metadata associated with a file
#[serde(tag = "type")]
#[derive(Default)]
pub enum Metadata {
/// File is just a generic uncategorised file
#[default]
File,
/// File contains textual data and should be displayed as such
Text,
/// File is an image with specific dimensions
Image { width: isize, height: isize },
/// File is a video with specific dimensions
Video { width: isize, height: isize },
/// File is audio
Audio,
}
);

View File

@@ -0,0 +1,12 @@
use revolt_result::Result;
use crate::FileHash;
mod mongodb;
mod reference;
#[async_trait]
pub trait AbstractAttachmentHashes: Sync + Send {
/// Fetch an attachment hash entry by sha256 hash.
async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash>;
}

View File

@@ -0,0 +1,24 @@
use revolt_result::Result;
use crate::FileHash;
use crate::MongoDb;
use super::AbstractAttachmentHashes;
static COL: &str = "attachment_hashes";
#[async_trait]
impl AbstractAttachmentHashes for MongoDb {
/// Fetch an attachment hash entry by sha256 hash.
async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash> {
query!(
self,
find_one,
COL,
doc! {
"_id": hash
}
)?
.ok_or_else(|| create_error!(NotFound))
}
}

View File

@@ -0,0 +1,23 @@
use revolt_result::Result;
use crate::FileHash;
use crate::ReferenceDb;
use super::AbstractAttachmentHashes;
#[async_trait]
impl AbstractAttachmentHashes for ReferenceDb {
/// Fetch an attachment hash entry by sha256 hash.
async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash> {
let hashes = self.file_hashes.lock().await;
if let Some(file) = hashes.get(hash) {
if file.id == hash {
Ok(file.clone())
} else {
Err(create_error!(NotFound))
}
} else {
Err(create_error!(NotFound))
}
}
}

View File

@@ -1,5 +1,6 @@
use crate::Database;
use crate::{Database, FileHash, Metadata};
use iso8601_timestamp::Timestamp;
use revolt_result::Result;
auto_derived_partial!(
@@ -12,13 +13,16 @@ auto_derived_partial!(
pub tag: String,
/// Original filename
pub filename: String,
/// Parsed metadata of this file
pub metadata: Metadata,
/// Raw content type of this file
pub content_type: String,
/// Size of this file (in bytes)
pub size: isize,
/// Hash of this file
pub hash: Option<String>, // these are Option<>s to not break file uploads on legacy Autumn
/// When this file was uploaded
pub uploaded_at: Option<Timestamp>, // these are Option<>s to not break file uploads on legacy Autumn
/// ID of user who uploaded this file
pub uploaded_id: Option<String>, // these are Option<>s to not break file uploads on legacy Autumn
// TODO: used_for: field
//
/// Whether this file was deleted
#[serde(skip_serializing_if = "Option::is_none")]
pub deleted: Option<bool>,
@@ -26,6 +30,14 @@ auto_derived_partial!(
#[serde(skip_serializing_if = "Option::is_none")]
pub reported: Option<bool>,
// !!! DEPRECATED:
/// Parsed metadata of this file
pub metadata: Metadata,
/// Raw content type of this file
pub content_type: String,
/// Size of this file (in bytes)
pub size: isize,
// TODO: migrate this mess to having:
// - author_id
// - parent: Parent { Message(id), User(id), etc }
@@ -43,26 +55,12 @@ auto_derived_partial!(
"PartialFile"
);
auto_derived!(
/// Metadata associated with a file
#[serde(tag = "type")]
#[derive(Default)]
pub enum Metadata {
/// File is just a generic uncategorised file
#[default]
File,
/// File contains textual data and should be displayed as such
Text,
/// File is an image with specific dimensions
Image { width: isize, height: isize },
/// File is a video with specific dimensions
Video { width: isize, height: isize },
/// File is audio
Audio,
}
);
impl File {
/// Get the hash entry for this file
pub async fn as_hash(&self, db: &Database) -> Result<FileHash> {
db.fetch_attachment_hash(self.hash.as_ref().unwrap()).await
}
/// Use a file for a message attachment
pub async fn use_attachment(db: &Database, id: &str, parent: &str) -> Result<File> {
db.find_and_use_attachment(id, "attachments", "message", parent)

View File

@@ -10,6 +10,9 @@ pub trait AbstractAttachments: Sync + Send {
/// Insert attachment into database.
async fn insert_attachment(&self, attachment: &File) -> Result<()>;
/// Fetch an attachment by its id.
async fn fetch_attachment(&self, tag: &str, file_id: &str) -> Result<File>;
/// Find an attachment by its details and mark it as used by a given parent.
async fn find_and_use_attachment(
&self,

View File

@@ -15,6 +15,20 @@ impl AbstractAttachments for MongoDb {
query!(self, insert_one, COL, &attachment).map(|_| ())
}
/// Fetch an attachment by its id.
async fn fetch_attachment(&self, tag: &str, file_id: &str) -> Result<File> {
query!(
self,
find_one,
COL,
doc! {
"_id": file_id,
"tag": tag
}
)?
.ok_or_else(|| create_error!(NotFound))
}
/// Find an attachment by its details and mark it as used by a given parent.
async fn find_and_use_attachment(
&self,

View File

@@ -18,6 +18,20 @@ impl AbstractAttachments for ReferenceDb {
}
}
/// Fetch an attachment by its id.
async fn fetch_attachment(&self, tag: &str, file_id: &str) -> Result<File> {
let files = self.files.lock().await;
if let Some(file) = files.get(file_id) {
if file.tag == tag {
Ok(file.clone())
} else {
Err(create_error!(NotFound))
}
} else {
Err(create_error!(NotFound))
}
}
/// Find an attachment by its details and mark it as used by a given parent.
async fn find_and_use_attachment(
&self,

View File

@@ -5,6 +5,7 @@ mod channel_unreads;
mod channel_webhooks;
mod channels;
mod emojis;
mod file_hashes;
mod files;
mod messages;
mod ratelimit_events;
@@ -23,6 +24,7 @@ pub use channel_unreads::*;
pub use channel_webhooks::*;
pub use channels::*;
pub use emojis::*;
pub use file_hashes::*;
pub use files::*;
pub use messages::*;
pub use ratelimit_events::*;
@@ -46,6 +48,7 @@ pub trait AbstractDatabase:
+ channel_unreads::AbstractChannelUnreads
+ channel_webhooks::AbstractWebhooks
+ emojis::AbstractEmojis
+ file_hashes::AbstractAttachmentHashes
+ files::AbstractAttachments
+ messages::AbstractMessages
+ ratelimit_events::AbstractRatelimitEvents

View File

@@ -419,6 +419,9 @@ impl From<File> for crate::File {
user_id: value.user_id,
server_id: value.server_id,
object_id: value.object_id,
hash: None,
uploaded_at: None,
uploaded_id: None,
}
}
}

View File

@@ -7,7 +7,12 @@ authors = ["Paul Makles <me@insrt.uk>"]
description = "Revolt Backend: S3 and encryption subroutines"
[dependencies]
base64 = "0.22.1"
aes-gcm = "0.10.3"
typenum = "1.17.0"
aws-config = "1.5.5"
aws-sdk-s3 = { version = "1.46.0", features = ["behavior-version-latest"] }
revolt-config = { version = "0.7.16", path = "../config" }
revolt-result = { version = "0.7.16", path = "../result" }

View File

@@ -1,3 +1,78 @@
// pub fn
use std::io::Write;
pub fn decrypt(data: &mut [u8]) {}
use aes_gcm::{aead::AeadMutInPlace, Aes256Gcm, Key, KeyInit, Nonce};
use revolt_config::{config, FilesS3};
use revolt_result::{create_error, Result};
use aws_sdk_s3::{
config::{Credentials, Region},
Client, Config,
};
use base64::prelude::*;
/// Create an S3 client
pub fn create_client(s3_config: FilesS3) -> Client {
let provider_name = "my-creds";
let creds = Credentials::new(
s3_config.access_key_id,
s3_config.secret_access_key,
None,
None,
provider_name,
);
let config = Config::builder()
.region(Region::new(s3_config.region))
.endpoint_url(s3_config.endpoint)
.credentials_provider(creds)
.build();
Client::from_conf(config)
}
/// Create an AES-256-GCM cipher
pub fn create_cipher(key: &str) -> Aes256Gcm {
let key = &BASE64_STANDARD.decode(key).unwrap()[..];
let key: &Key<Aes256Gcm> = key.into();
Aes256Gcm::new(key)
}
/// Fetch a file from S3 (and decrypt it)
pub async fn fetch_from_s3(bucket_id: &str, path: &str, nonce: &str) -> Result<Vec<u8>> {
let config = config().await;
let client = create_client(config.files.s3);
// Send a request for the file
let mut obj = client
.get_object()
.bucket(bucket_id)
.key(path)
.send()
.await
.inspect_err(|err| {
revolt_config::capture_error(err);
})
.map_err(|_| create_error!(InternalError))?;
// Read the file from remote
let mut buf = vec![];
while let Some(bytes) = obj.body.next().await {
let data = bytes.map_err(|_| create_error!(InternalError))?;
buf.write_all(&data)
.map_err(|_| create_error!(InternalError))?;
// is there a more efficient way to do this?
// we just want the Vec<u8>
}
// Recover nonce as bytes
let nonce = &BASE64_STANDARD.decode(nonce).unwrap()[..];
let nonce: &Nonce<typenum::consts::U12> = nonce.into();
// Decrypt the file
create_cipher(&config.files.encryption_key)
.decrypt_in_place(nonce, b"", &mut buf)
.map_err(|_| create_error!(InternalError))?;
Ok(buf)
}

View File

@@ -4,9 +4,18 @@ version = "0.7.14"
edition = "2021"
[dependencies]
# Media processing
webp = "0.3.0"
image = "0.25.2"
# Utility
lazy_static = "1.5.0"
moka = { version = "0.12.8", features = ["future"] }
# Serialisation
serde = { version = "1.0", features = ["derive"] }
strum_macros = "0.26.4"
serde_json = "1.0.68"
serde = { version = "1.0", features = ["derive"] }
# Async runtime
tokio = { version = "1.0", features = ["full"] }
@@ -16,7 +25,9 @@ tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
# Core crates
revolt-files = { version = "0.7.16", path = "../../core/files" }
revolt-config = { version = "0.7.16", path = "../../core/config" }
revolt-database = { version = "0.7.16", path = "../../core/database" }
revolt-result = { version = "0.7.16", path = "../../core/result", features = [
"utoipa",
"axum",
@@ -24,6 +35,7 @@ revolt-result = { version = "0.7.16", path = "../../core/result", features = [
# Axum / web server
tempfile = "3.12.0"
axum-macros = "0.4.1"
axum_typed_multipart = "0.12.1"
axum = { version = "0.7.5", features = ["multipart"] }

View File

@@ -1,26 +1,57 @@
use std::{io::Cursor, time::Duration};
use axum::{
extract::{DefaultBodyLimit, Multipart, Path},
http::StatusCode,
response::{IntoResponse, Response},
extract::{DefaultBodyLimit, Path, State},
http::header,
response::{IntoResponse, Redirect, Response},
routing::{get, post},
Json, Router,
};
use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart};
use image::ImageReader;
use lazy_static::lazy_static;
use revolt_config::config;
use revolt_database::{Database, FileHash};
use revolt_files::fetch_from_s3;
use revolt_result::{create_error, Result};
use serde::{Deserialize, Serialize};
use tempfile::NamedTempFile;
use utoipa::ToSchema;
use tempfile::NamedTempFile;
pub async fn router() -> Router {
/// Build the API router
pub async fn router() -> Router<Database> {
let config = config().await;
Router::new().route("/", get(root)).route(
"/:tag",
post(upload_file).layer(DefaultBodyLimit::max(
config.features.limits.global.body_limit_size,
)),
)
Router::new()
.route("/", get(root))
.route(
"/:tag",
post(upload_file).layer(DefaultBodyLimit::max(
config.features.limits.global.body_limit_size,
)),
)
.route("/:tag/:file_id", get(fetch_preview))
.route("/:tag/:file_id/:file_name", get(fetch_file))
}
lazy_static! {
/// Short-lived file cache to allow us to populate different CDN regions without increasing bandwidth to S3 provider
/// Uploads will also be stored here to prevent immediately queued downloads from doing the entire round-trip
static ref S3_CACHE: moka::future::Cache<String, Result<Vec<u8>>> = moka::future::Cache::builder()
.max_capacity(10_000) // TODO config
.time_to_live(Duration::from_secs(60)) // TODO config
.build();
}
/// Retrieve hash information and file data by given hash
async fn retrieve_file_by_hash(hash: &FileHash) -> Result<Vec<u8>> {
if let Some(data) = S3_CACHE.get(&hash.id).await {
data
} else {
let data = fetch_from_s3(&hash.bucket_id, &hash.path, &hash.iv).await;
S3_CACHE.insert(hash.id.to_owned(), data.clone()).await;
data
}
}
/// Successful root response
@@ -49,7 +80,7 @@ async fn root() -> Json<RootResponse> {
}
/// Available tags to upload to
#[derive(Deserialize, Debug, ToSchema)]
#[derive(Deserialize, Debug, ToSchema, strum_macros::IntoStaticStr)]
#[allow(non_camel_case_types)]
pub enum Tag {
attachments,
@@ -106,8 +137,14 @@ async fn upload_file(
Ok(Json(UploadResponse { id: "aaa" }))
}
/// Header value used for cache control
pub static CACHE_CONTROL: &str = "public, max-age=604800, must-revalidate";
/// Fetch preview of file
///
/// This route will only return image content.
/// For all other file types, please use the fetch route (you will receive a redirect if you try to use this route anyways!).
///
/// Depending on the given tag, the file will be re-processed to fit the criteria:
///
/// | Tag | Image Resolution <sup>†</sup> |
@@ -132,18 +169,72 @@ async fn upload_file(
),
)]
async fn fetch_preview(
Path(tag): Path<Tag>,
Path(file_id): Path<String>,
) -> axum::response::Result<Response> {
todo!()
State(db): State<Database>,
Path((tag, file_id)): Path<(Tag, String)>,
) -> Result<Response> {
let tag: &'static str = tag.into();
let file = db.fetch_attachment(tag, &file_id).await?;
// Ignore deleted files
if file.deleted.is_some_and(|v| v) {
return Err(create_error!(NotFound));
}
let hash = file.as_hash(&db).await?;
// Only process image files
if !hash.content_type.starts_with("image/") {
return Ok(
Redirect::permanent(&format!("/{tag}/{file_id}/{}", file.filename)).into_response(),
);
}
// Original image data
let data = retrieve_file_by_hash(&hash).await?;
// Dimensions we need to resize to
let config = config().await;
let [w, h] = config.files.preview.get(tag).unwrap();
// Read the image and resize it
let image = ImageReader::new(Cursor::new(data))
.with_guessed_format()
.map_err(|_| create_error!(InternalError))?
.decode()
.map_err(|_| create_error!(InternalError))?
// resize is about 2.5x slower,
// thumb approximation doesn't have terrible quality so it's fine to stick with
// .resize(width as u32, height as u32, image::imageops::FilterType::Gaussian)
// aspect ratio is preserved when scaling
.thumbnail(*w as u32, *h as u32);
// Encode it into WEBP
let encoder = webp::Encoder::from_image(&image).expect("Could not create encoder.");
let data = if config.files.webp_quality != 100.0 {
encoder.encode(config.files.webp_quality).to_vec()
} else {
encoder.encode_lossless().to_vec()
};
Ok((
[
(header::CONTENT_TYPE, "image/webp"),
(header::CONTENT_DISPOSITION, "inline"),
(header::CACHE_CONTROL, CACHE_CONTROL),
],
data,
)
.into_response())
}
/// Fetch original file
///
/// Content disposition header will be set to 'attachment' to prevent browser from rendering anything.
#[utoipa::path(
get,
path = "/{tag}/{file_id}/{file_name}",
responses(
(status = 200, description = "Generated preview", body = Vec<u8>)
(status = 200, description = "Original file", body = Vec<u8>)
),
params(
("tag" = Tag, Path, description = "Tag to fetch from (e.g. attachments, icons, ...)"),
@@ -152,9 +243,31 @@ async fn fetch_preview(
),
)]
async fn fetch_file(
Path(tag): Path<Tag>,
Path(file_id): Path<String>,
Path(file_name): Path<String>,
) -> axum::response::Result<Response> {
todo!()
State(db): State<Database>,
Path((tag, file_id, file_name)): Path<(Tag, String, String)>,
) -> Result<Response> {
let file = db.fetch_attachment(tag.into(), &file_id).await?;
// Ignore deleted files
if file.deleted.is_some_and(|v| v) {
return Err(create_error!(NotFound));
}
// Ensure filename is correct
if file_name != file.filename {
return Err(create_error!(NotFound));
}
let hash = file.as_hash(&db).await?;
retrieve_file_by_hash(&hash).await.map(|data| {
(
[
(header::CONTENT_TYPE, hash.content_type),
(header::CONTENT_DISPOSITION, "attachment".to_owned()),
(header::CACHE_CONTROL, CACHE_CONTROL.to_owned()),
],
data,
)
.into_response()
})
}

View File

@@ -2,6 +2,7 @@ use std::net::{Ipv4Addr, SocketAddr};
use axum::Router;
use revolt_database::DatabaseInfo;
use tokio::net::TcpListener;
use utoipa::{
openapi::security::{ApiKey, ApiKeyValue, SecurityScheme},
@@ -55,10 +56,14 @@ async fn main() -> Result<(), std::io::Error> {
}
}
// Connect to the database
let db = DatabaseInfo::Auto.connect().await.unwrap();
// Configure Axum and router
let app = Router::new()
.merge(Scalar::with_url("/scalar", ApiDoc::openapi()))
.nest("/", api::router().await);
.nest("/", api::router().await)
.with_state(db);
// Configure TCP listener and bind
let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, 3000));