forked from jmug/stoatchat
feat(services/autumn): download and preview files
This commit is contained in:
3
.vscode/settings.json
vendored
3
.vscode/settings.json
vendored
@@ -1,6 +1,5 @@
|
|||||||
{
|
{
|
||||||
"editor.formatOnSave": true,
|
"editor.formatOnSave": true,
|
||||||
"rust-analyzer.checkOnSave.command": "clippy",
|
"rust-analyzer.checkOnSave.command": "clippy",
|
||||||
"nixEnvSelector.suggestion": false,
|
"nixEnvSelector.suggestion": false
|
||||||
"nixEnvSelector.nixFile": "${workspaceRoot}/default.nix"
|
|
||||||
}
|
}
|
||||||
|
|||||||
694
Cargo.lock
generated
694
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -88,6 +88,8 @@ max_concurrent_connections = 50
|
|||||||
# Encryption key for stored files
|
# Encryption key for stored files
|
||||||
# Generate your own key using `openssl rand -base64 32`
|
# Generate your own key using `openssl rand -base64 32`
|
||||||
encryption_key = "qcuMA+ssxhMyKaNAKBGFfryfFtUH8NDlamQyDwGW6fU="
|
encryption_key = "qcuMA+ssxhMyKaNAKBGFfryfFtUH8NDlamQyDwGW6fU="
|
||||||
|
# Quality used for lossy WebP previews (set to 100 for lossless)
|
||||||
|
webp_quality = 80.0
|
||||||
|
|
||||||
[files.limit]
|
[files.limit]
|
||||||
# Minimum image resolution
|
# Minimum image resolution
|
||||||
@@ -118,13 +120,13 @@ emojis = [128, 128]
|
|||||||
# - default_bucket matches the name of the bucket you've created
|
# - default_bucket matches the name of the bucket you've created
|
||||||
|
|
||||||
# S3 protocol endpoint
|
# S3 protocol endpoint
|
||||||
endpoint = ""
|
endpoint = "http://minio:9000"
|
||||||
# S3 region name
|
# S3 region name
|
||||||
region = ""
|
region = "minio"
|
||||||
# S3 protocol key ID
|
# S3 protocol key ID
|
||||||
access_key_id = ""
|
access_key_id = "minioautumn"
|
||||||
# S3 protocol access key
|
# S3 protocol access key
|
||||||
secret_access_key = ""
|
secret_access_key = "minioautumn"
|
||||||
# Bucket to upload to by default
|
# Bucket to upload to by default
|
||||||
default_bucket = "revolt-uploads"
|
default_bucket = "revolt-uploads"
|
||||||
|
|
||||||
|
|||||||
@@ -126,6 +126,31 @@ pub struct Api {
|
|||||||
pub workers: ApiWorkers,
|
pub workers: ApiWorkers,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
|
pub struct FilesLimit {
|
||||||
|
pub min_resolution: [usize; 2],
|
||||||
|
pub max_mega_pixels: usize,
|
||||||
|
pub max_pixel_side: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
|
pub struct FilesS3 {
|
||||||
|
pub endpoint: String,
|
||||||
|
pub region: String,
|
||||||
|
pub access_key_id: String,
|
||||||
|
pub secret_access_key: String,
|
||||||
|
pub default_bucket: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
|
pub struct Files {
|
||||||
|
pub encryption_key: String,
|
||||||
|
pub webp_quality: f32,
|
||||||
|
pub limit: FilesLimit,
|
||||||
|
pub preview: HashMap<String, [usize; 2]>,
|
||||||
|
pub s3: FilesS3,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Debug, Clone)]
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
pub struct GlobalLimits {
|
pub struct GlobalLimits {
|
||||||
pub group_size: usize,
|
pub group_size: usize,
|
||||||
@@ -186,6 +211,7 @@ pub struct Settings {
|
|||||||
pub database: Database,
|
pub database: Database,
|
||||||
pub hosts: Hosts,
|
pub hosts: Hosts,
|
||||||
pub api: Api,
|
pub api: Api,
|
||||||
|
pub files: Files,
|
||||||
pub features: Features,
|
pub features: Features,
|
||||||
pub sentry: Sentry,
|
pub sentry: Sentry,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ use std::{collections::HashMap, sync::Arc};
|
|||||||
use futures::lock::Mutex;
|
use futures::lock::Mutex;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
Bot, Channel, ChannelCompositeKey, ChannelUnread, Emoji, File, Invite, Member,
|
Bot, Channel, ChannelCompositeKey, ChannelUnread, Emoji, File, FileHash, Invite, Member,
|
||||||
MemberCompositeKey, Message, RatelimitEvent, Report, Server, ServerBan, Snapshot, User,
|
MemberCompositeKey, Message, RatelimitEvent, Report, Server, ServerBan, Snapshot, User,
|
||||||
UserSettings, Webhook,
|
UserSettings, Webhook,
|
||||||
};
|
};
|
||||||
@@ -18,6 +18,7 @@ database_derived!(
|
|||||||
pub channel_unreads: Arc<Mutex<HashMap<ChannelCompositeKey, ChannelUnread>>>,
|
pub channel_unreads: Arc<Mutex<HashMap<ChannelCompositeKey, ChannelUnread>>>,
|
||||||
pub channel_webhooks: Arc<Mutex<HashMap<String, Webhook>>>,
|
pub channel_webhooks: Arc<Mutex<HashMap<String, Webhook>>>,
|
||||||
pub emojis: Arc<Mutex<HashMap<String, Emoji>>>,
|
pub emojis: Arc<Mutex<HashMap<String, Emoji>>>,
|
||||||
|
pub file_hashes: Arc<Mutex<HashMap<String, FileHash>>>,
|
||||||
pub files: Arc<Mutex<HashMap<String, File>>>,
|
pub files: Arc<Mutex<HashMap<String, File>>>,
|
||||||
pub messages: Arc<Mutex<HashMap<String, Message>>>,
|
pub messages: Arc<Mutex<HashMap<String, Message>>>,
|
||||||
pub ratelimit_events: Arc<Mutex<HashMap<String, RatelimitEvent>>>,
|
pub ratelimit_events: Arc<Mutex<HashMap<String, RatelimitEvent>>>,
|
||||||
|
|||||||
5
crates/core/database/src/models/file_hashes/mod.rs
Normal file
5
crates/core/database/src/models/file_hashes/mod.rs
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
mod model;
|
||||||
|
mod ops;
|
||||||
|
|
||||||
|
pub use model::*;
|
||||||
|
pub use ops::*;
|
||||||
49
crates/core/database/src/models/file_hashes/model.rs
Normal file
49
crates/core/database/src/models/file_hashes/model.rs
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
use iso8601_timestamp::Timestamp;
|
||||||
|
|
||||||
|
auto_derived_partial!(
|
||||||
|
/// File hash
|
||||||
|
pub struct FileHash {
|
||||||
|
/// Sha256 hash of the file
|
||||||
|
#[serde(rename = "_id")]
|
||||||
|
pub id: String,
|
||||||
|
/// Sha256 hash of file after it has been processed
|
||||||
|
pub processed_hash: String,
|
||||||
|
|
||||||
|
/// When this file was created in system
|
||||||
|
pub created_at: Timestamp,
|
||||||
|
|
||||||
|
/// The bucket this file is stored in
|
||||||
|
pub bucket_id: String,
|
||||||
|
/// The path at which this file exists in
|
||||||
|
pub path: String,
|
||||||
|
/// Cryptographic nonce used to encrypt this file
|
||||||
|
pub iv: String,
|
||||||
|
|
||||||
|
/// Parsed metadata of this file
|
||||||
|
pub metadata: Metadata,
|
||||||
|
/// Raw content type of this file
|
||||||
|
pub content_type: String,
|
||||||
|
/// Size of this file (in bytes)
|
||||||
|
pub size: isize,
|
||||||
|
},
|
||||||
|
"PartialFile"
|
||||||
|
);
|
||||||
|
|
||||||
|
auto_derived!(
|
||||||
|
/// Metadata associated with a file
|
||||||
|
#[serde(tag = "type")]
|
||||||
|
#[derive(Default)]
|
||||||
|
pub enum Metadata {
|
||||||
|
/// File is just a generic uncategorised file
|
||||||
|
#[default]
|
||||||
|
File,
|
||||||
|
/// File contains textual data and should be displayed as such
|
||||||
|
Text,
|
||||||
|
/// File is an image with specific dimensions
|
||||||
|
Image { width: isize, height: isize },
|
||||||
|
/// File is a video with specific dimensions
|
||||||
|
Video { width: isize, height: isize },
|
||||||
|
/// File is audio
|
||||||
|
Audio,
|
||||||
|
}
|
||||||
|
);
|
||||||
12
crates/core/database/src/models/file_hashes/ops.rs
Normal file
12
crates/core/database/src/models/file_hashes/ops.rs
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
use revolt_result::Result;
|
||||||
|
|
||||||
|
use crate::FileHash;
|
||||||
|
|
||||||
|
mod mongodb;
|
||||||
|
mod reference;
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
pub trait AbstractAttachmentHashes: Sync + Send {
|
||||||
|
/// Fetch an attachment hash entry by sha256 hash.
|
||||||
|
async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash>;
|
||||||
|
}
|
||||||
24
crates/core/database/src/models/file_hashes/ops/mongodb.rs
Normal file
24
crates/core/database/src/models/file_hashes/ops/mongodb.rs
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
use revolt_result::Result;
|
||||||
|
|
||||||
|
use crate::FileHash;
|
||||||
|
use crate::MongoDb;
|
||||||
|
|
||||||
|
use super::AbstractAttachmentHashes;
|
||||||
|
|
||||||
|
static COL: &str = "attachment_hashes";
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl AbstractAttachmentHashes for MongoDb {
|
||||||
|
/// Fetch an attachment hash entry by sha256 hash.
|
||||||
|
async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash> {
|
||||||
|
query!(
|
||||||
|
self,
|
||||||
|
find_one,
|
||||||
|
COL,
|
||||||
|
doc! {
|
||||||
|
"_id": hash
|
||||||
|
}
|
||||||
|
)?
|
||||||
|
.ok_or_else(|| create_error!(NotFound))
|
||||||
|
}
|
||||||
|
}
|
||||||
23
crates/core/database/src/models/file_hashes/ops/reference.rs
Normal file
23
crates/core/database/src/models/file_hashes/ops/reference.rs
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
use revolt_result::Result;
|
||||||
|
|
||||||
|
use crate::FileHash;
|
||||||
|
use crate::ReferenceDb;
|
||||||
|
|
||||||
|
use super::AbstractAttachmentHashes;
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl AbstractAttachmentHashes for ReferenceDb {
|
||||||
|
/// Fetch an attachment hash entry by sha256 hash.
|
||||||
|
async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash> {
|
||||||
|
let hashes = self.file_hashes.lock().await;
|
||||||
|
if let Some(file) = hashes.get(hash) {
|
||||||
|
if file.id == hash {
|
||||||
|
Ok(file.clone())
|
||||||
|
} else {
|
||||||
|
Err(create_error!(NotFound))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Err(create_error!(NotFound))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
use crate::Database;
|
use crate::{Database, FileHash, Metadata};
|
||||||
|
|
||||||
|
use iso8601_timestamp::Timestamp;
|
||||||
use revolt_result::Result;
|
use revolt_result::Result;
|
||||||
|
|
||||||
auto_derived_partial!(
|
auto_derived_partial!(
|
||||||
@@ -12,13 +13,16 @@ auto_derived_partial!(
|
|||||||
pub tag: String,
|
pub tag: String,
|
||||||
/// Original filename
|
/// Original filename
|
||||||
pub filename: String,
|
pub filename: String,
|
||||||
/// Parsed metadata of this file
|
/// Hash of this file
|
||||||
pub metadata: Metadata,
|
pub hash: Option<String>, // these are Option<>s to not break file uploads on legacy Autumn
|
||||||
/// Raw content type of this file
|
|
||||||
pub content_type: String,
|
|
||||||
/// Size of this file (in bytes)
|
|
||||||
pub size: isize,
|
|
||||||
|
|
||||||
|
/// When this file was uploaded
|
||||||
|
pub uploaded_at: Option<Timestamp>, // these are Option<>s to not break file uploads on legacy Autumn
|
||||||
|
/// ID of user who uploaded this file
|
||||||
|
pub uploaded_id: Option<String>, // these are Option<>s to not break file uploads on legacy Autumn
|
||||||
|
|
||||||
|
// TODO: used_for: field
|
||||||
|
//
|
||||||
/// Whether this file was deleted
|
/// Whether this file was deleted
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub deleted: Option<bool>,
|
pub deleted: Option<bool>,
|
||||||
@@ -26,6 +30,14 @@ auto_derived_partial!(
|
|||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub reported: Option<bool>,
|
pub reported: Option<bool>,
|
||||||
|
|
||||||
|
// !!! DEPRECATED:
|
||||||
|
/// Parsed metadata of this file
|
||||||
|
pub metadata: Metadata,
|
||||||
|
/// Raw content type of this file
|
||||||
|
pub content_type: String,
|
||||||
|
/// Size of this file (in bytes)
|
||||||
|
pub size: isize,
|
||||||
|
|
||||||
// TODO: migrate this mess to having:
|
// TODO: migrate this mess to having:
|
||||||
// - author_id
|
// - author_id
|
||||||
// - parent: Parent { Message(id), User(id), etc }
|
// - parent: Parent { Message(id), User(id), etc }
|
||||||
@@ -43,26 +55,12 @@ auto_derived_partial!(
|
|||||||
"PartialFile"
|
"PartialFile"
|
||||||
);
|
);
|
||||||
|
|
||||||
auto_derived!(
|
|
||||||
/// Metadata associated with a file
|
|
||||||
#[serde(tag = "type")]
|
|
||||||
#[derive(Default)]
|
|
||||||
pub enum Metadata {
|
|
||||||
/// File is just a generic uncategorised file
|
|
||||||
#[default]
|
|
||||||
File,
|
|
||||||
/// File contains textual data and should be displayed as such
|
|
||||||
Text,
|
|
||||||
/// File is an image with specific dimensions
|
|
||||||
Image { width: isize, height: isize },
|
|
||||||
/// File is a video with specific dimensions
|
|
||||||
Video { width: isize, height: isize },
|
|
||||||
/// File is audio
|
|
||||||
Audio,
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
impl File {
|
impl File {
|
||||||
|
/// Get the hash entry for this file
|
||||||
|
pub async fn as_hash(&self, db: &Database) -> Result<FileHash> {
|
||||||
|
db.fetch_attachment_hash(self.hash.as_ref().unwrap()).await
|
||||||
|
}
|
||||||
|
|
||||||
/// Use a file for a message attachment
|
/// Use a file for a message attachment
|
||||||
pub async fn use_attachment(db: &Database, id: &str, parent: &str) -> Result<File> {
|
pub async fn use_attachment(db: &Database, id: &str, parent: &str) -> Result<File> {
|
||||||
db.find_and_use_attachment(id, "attachments", "message", parent)
|
db.find_and_use_attachment(id, "attachments", "message", parent)
|
||||||
|
|||||||
@@ -10,6 +10,9 @@ pub trait AbstractAttachments: Sync + Send {
|
|||||||
/// Insert attachment into database.
|
/// Insert attachment into database.
|
||||||
async fn insert_attachment(&self, attachment: &File) -> Result<()>;
|
async fn insert_attachment(&self, attachment: &File) -> Result<()>;
|
||||||
|
|
||||||
|
/// Fetch an attachment by its id.
|
||||||
|
async fn fetch_attachment(&self, tag: &str, file_id: &str) -> Result<File>;
|
||||||
|
|
||||||
/// Find an attachment by its details and mark it as used by a given parent.
|
/// Find an attachment by its details and mark it as used by a given parent.
|
||||||
async fn find_and_use_attachment(
|
async fn find_and_use_attachment(
|
||||||
&self,
|
&self,
|
||||||
|
|||||||
@@ -15,6 +15,20 @@ impl AbstractAttachments for MongoDb {
|
|||||||
query!(self, insert_one, COL, &attachment).map(|_| ())
|
query!(self, insert_one, COL, &attachment).map(|_| ())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fetch an attachment by its id.
|
||||||
|
async fn fetch_attachment(&self, tag: &str, file_id: &str) -> Result<File> {
|
||||||
|
query!(
|
||||||
|
self,
|
||||||
|
find_one,
|
||||||
|
COL,
|
||||||
|
doc! {
|
||||||
|
"_id": file_id,
|
||||||
|
"tag": tag
|
||||||
|
}
|
||||||
|
)?
|
||||||
|
.ok_or_else(|| create_error!(NotFound))
|
||||||
|
}
|
||||||
|
|
||||||
/// Find an attachment by its details and mark it as used by a given parent.
|
/// Find an attachment by its details and mark it as used by a given parent.
|
||||||
async fn find_and_use_attachment(
|
async fn find_and_use_attachment(
|
||||||
&self,
|
&self,
|
||||||
|
|||||||
@@ -18,6 +18,20 @@ impl AbstractAttachments for ReferenceDb {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fetch an attachment by its id.
|
||||||
|
async fn fetch_attachment(&self, tag: &str, file_id: &str) -> Result<File> {
|
||||||
|
let files = self.files.lock().await;
|
||||||
|
if let Some(file) = files.get(file_id) {
|
||||||
|
if file.tag == tag {
|
||||||
|
Ok(file.clone())
|
||||||
|
} else {
|
||||||
|
Err(create_error!(NotFound))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Err(create_error!(NotFound))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Find an attachment by its details and mark it as used by a given parent.
|
/// Find an attachment by its details and mark it as used by a given parent.
|
||||||
async fn find_and_use_attachment(
|
async fn find_and_use_attachment(
|
||||||
&self,
|
&self,
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ mod channel_unreads;
|
|||||||
mod channel_webhooks;
|
mod channel_webhooks;
|
||||||
mod channels;
|
mod channels;
|
||||||
mod emojis;
|
mod emojis;
|
||||||
|
mod file_hashes;
|
||||||
mod files;
|
mod files;
|
||||||
mod messages;
|
mod messages;
|
||||||
mod ratelimit_events;
|
mod ratelimit_events;
|
||||||
@@ -23,6 +24,7 @@ pub use channel_unreads::*;
|
|||||||
pub use channel_webhooks::*;
|
pub use channel_webhooks::*;
|
||||||
pub use channels::*;
|
pub use channels::*;
|
||||||
pub use emojis::*;
|
pub use emojis::*;
|
||||||
|
pub use file_hashes::*;
|
||||||
pub use files::*;
|
pub use files::*;
|
||||||
pub use messages::*;
|
pub use messages::*;
|
||||||
pub use ratelimit_events::*;
|
pub use ratelimit_events::*;
|
||||||
@@ -46,6 +48,7 @@ pub trait AbstractDatabase:
|
|||||||
+ channel_unreads::AbstractChannelUnreads
|
+ channel_unreads::AbstractChannelUnreads
|
||||||
+ channel_webhooks::AbstractWebhooks
|
+ channel_webhooks::AbstractWebhooks
|
||||||
+ emojis::AbstractEmojis
|
+ emojis::AbstractEmojis
|
||||||
|
+ file_hashes::AbstractAttachmentHashes
|
||||||
+ files::AbstractAttachments
|
+ files::AbstractAttachments
|
||||||
+ messages::AbstractMessages
|
+ messages::AbstractMessages
|
||||||
+ ratelimit_events::AbstractRatelimitEvents
|
+ ratelimit_events::AbstractRatelimitEvents
|
||||||
|
|||||||
@@ -419,6 +419,9 @@ impl From<File> for crate::File {
|
|||||||
user_id: value.user_id,
|
user_id: value.user_id,
|
||||||
server_id: value.server_id,
|
server_id: value.server_id,
|
||||||
object_id: value.object_id,
|
object_id: value.object_id,
|
||||||
|
hash: None,
|
||||||
|
uploaded_at: None,
|
||||||
|
uploaded_id: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,12 @@ authors = ["Paul Makles <me@insrt.uk>"]
|
|||||||
description = "Revolt Backend: S3 and encryption subroutines"
|
description = "Revolt Backend: S3 and encryption subroutines"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
base64 = "0.22.1"
|
||||||
aes-gcm = "0.10.3"
|
aes-gcm = "0.10.3"
|
||||||
|
typenum = "1.17.0"
|
||||||
|
|
||||||
aws-config = "1.5.5"
|
aws-config = "1.5.5"
|
||||||
aws-sdk-s3 = { version = "1.46.0", features = ["behavior-version-latest"] }
|
aws-sdk-s3 = { version = "1.46.0", features = ["behavior-version-latest"] }
|
||||||
|
|
||||||
|
revolt-config = { version = "0.7.16", path = "../config" }
|
||||||
|
revolt-result = { version = "0.7.16", path = "../result" }
|
||||||
|
|||||||
@@ -1,3 +1,78 @@
|
|||||||
// pub fn
|
use std::io::Write;
|
||||||
|
|
||||||
pub fn decrypt(data: &mut [u8]) {}
|
use aes_gcm::{aead::AeadMutInPlace, Aes256Gcm, Key, KeyInit, Nonce};
|
||||||
|
use revolt_config::{config, FilesS3};
|
||||||
|
use revolt_result::{create_error, Result};
|
||||||
|
|
||||||
|
use aws_sdk_s3::{
|
||||||
|
config::{Credentials, Region},
|
||||||
|
Client, Config,
|
||||||
|
};
|
||||||
|
|
||||||
|
use base64::prelude::*;
|
||||||
|
|
||||||
|
/// Create an S3 client
|
||||||
|
pub fn create_client(s3_config: FilesS3) -> Client {
|
||||||
|
let provider_name = "my-creds";
|
||||||
|
let creds = Credentials::new(
|
||||||
|
s3_config.access_key_id,
|
||||||
|
s3_config.secret_access_key,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
provider_name,
|
||||||
|
);
|
||||||
|
|
||||||
|
let config = Config::builder()
|
||||||
|
.region(Region::new(s3_config.region))
|
||||||
|
.endpoint_url(s3_config.endpoint)
|
||||||
|
.credentials_provider(creds)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
Client::from_conf(config)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create an AES-256-GCM cipher
|
||||||
|
pub fn create_cipher(key: &str) -> Aes256Gcm {
|
||||||
|
let key = &BASE64_STANDARD.decode(key).unwrap()[..];
|
||||||
|
let key: &Key<Aes256Gcm> = key.into();
|
||||||
|
Aes256Gcm::new(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch a file from S3 (and decrypt it)
|
||||||
|
pub async fn fetch_from_s3(bucket_id: &str, path: &str, nonce: &str) -> Result<Vec<u8>> {
|
||||||
|
let config = config().await;
|
||||||
|
let client = create_client(config.files.s3);
|
||||||
|
|
||||||
|
// Send a request for the file
|
||||||
|
let mut obj = client
|
||||||
|
.get_object()
|
||||||
|
.bucket(bucket_id)
|
||||||
|
.key(path)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.inspect_err(|err| {
|
||||||
|
revolt_config::capture_error(err);
|
||||||
|
})
|
||||||
|
.map_err(|_| create_error!(InternalError))?;
|
||||||
|
|
||||||
|
// Read the file from remote
|
||||||
|
let mut buf = vec![];
|
||||||
|
while let Some(bytes) = obj.body.next().await {
|
||||||
|
let data = bytes.map_err(|_| create_error!(InternalError))?;
|
||||||
|
buf.write_all(&data)
|
||||||
|
.map_err(|_| create_error!(InternalError))?;
|
||||||
|
// is there a more efficient way to do this?
|
||||||
|
// we just want the Vec<u8>
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recover nonce as bytes
|
||||||
|
let nonce = &BASE64_STANDARD.decode(nonce).unwrap()[..];
|
||||||
|
let nonce: &Nonce<typenum::consts::U12> = nonce.into();
|
||||||
|
|
||||||
|
// Decrypt the file
|
||||||
|
create_cipher(&config.files.encryption_key)
|
||||||
|
.decrypt_in_place(nonce, b"", &mut buf)
|
||||||
|
.map_err(|_| create_error!(InternalError))?;
|
||||||
|
|
||||||
|
Ok(buf)
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,9 +4,18 @@ version = "0.7.14"
|
|||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
# Media processing
|
||||||
|
webp = "0.3.0"
|
||||||
|
image = "0.25.2"
|
||||||
|
|
||||||
|
# Utility
|
||||||
|
lazy_static = "1.5.0"
|
||||||
|
moka = { version = "0.12.8", features = ["future"] }
|
||||||
|
|
||||||
# Serialisation
|
# Serialisation
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
strum_macros = "0.26.4"
|
||||||
serde_json = "1.0.68"
|
serde_json = "1.0.68"
|
||||||
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
|
|
||||||
# Async runtime
|
# Async runtime
|
||||||
tokio = { version = "1.0", features = ["full"] }
|
tokio = { version = "1.0", features = ["full"] }
|
||||||
@@ -16,7 +25,9 @@ tracing = "0.1"
|
|||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
|
|
||||||
# Core crates
|
# Core crates
|
||||||
|
revolt-files = { version = "0.7.16", path = "../../core/files" }
|
||||||
revolt-config = { version = "0.7.16", path = "../../core/config" }
|
revolt-config = { version = "0.7.16", path = "../../core/config" }
|
||||||
|
revolt-database = { version = "0.7.16", path = "../../core/database" }
|
||||||
revolt-result = { version = "0.7.16", path = "../../core/result", features = [
|
revolt-result = { version = "0.7.16", path = "../../core/result", features = [
|
||||||
"utoipa",
|
"utoipa",
|
||||||
"axum",
|
"axum",
|
||||||
@@ -24,6 +35,7 @@ revolt-result = { version = "0.7.16", path = "../../core/result", features = [
|
|||||||
|
|
||||||
# Axum / web server
|
# Axum / web server
|
||||||
tempfile = "3.12.0"
|
tempfile = "3.12.0"
|
||||||
|
axum-macros = "0.4.1"
|
||||||
axum_typed_multipart = "0.12.1"
|
axum_typed_multipart = "0.12.1"
|
||||||
axum = { version = "0.7.5", features = ["multipart"] }
|
axum = { version = "0.7.5", features = ["multipart"] }
|
||||||
|
|
||||||
|
|||||||
@@ -1,26 +1,57 @@
|
|||||||
|
use std::{io::Cursor, time::Duration};
|
||||||
|
|
||||||
use axum::{
|
use axum::{
|
||||||
extract::{DefaultBodyLimit, Multipart, Path},
|
extract::{DefaultBodyLimit, Path, State},
|
||||||
http::StatusCode,
|
http::header,
|
||||||
response::{IntoResponse, Response},
|
response::{IntoResponse, Redirect, Response},
|
||||||
routing::{get, post},
|
routing::{get, post},
|
||||||
Json, Router,
|
Json, Router,
|
||||||
};
|
};
|
||||||
use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart};
|
use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart};
|
||||||
|
use image::ImageReader;
|
||||||
|
use lazy_static::lazy_static;
|
||||||
use revolt_config::config;
|
use revolt_config::config;
|
||||||
|
use revolt_database::{Database, FileHash};
|
||||||
|
use revolt_files::fetch_from_s3;
|
||||||
|
use revolt_result::{create_error, Result};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use tempfile::NamedTempFile;
|
||||||
use utoipa::ToSchema;
|
use utoipa::ToSchema;
|
||||||
|
|
||||||
use tempfile::NamedTempFile;
|
/// Build the API router
|
||||||
|
pub async fn router() -> Router<Database> {
|
||||||
pub async fn router() -> Router {
|
|
||||||
let config = config().await;
|
let config = config().await;
|
||||||
|
|
||||||
Router::new().route("/", get(root)).route(
|
Router::new()
|
||||||
"/:tag",
|
.route("/", get(root))
|
||||||
post(upload_file).layer(DefaultBodyLimit::max(
|
.route(
|
||||||
config.features.limits.global.body_limit_size,
|
"/:tag",
|
||||||
)),
|
post(upload_file).layer(DefaultBodyLimit::max(
|
||||||
)
|
config.features.limits.global.body_limit_size,
|
||||||
|
)),
|
||||||
|
)
|
||||||
|
.route("/:tag/:file_id", get(fetch_preview))
|
||||||
|
.route("/:tag/:file_id/:file_name", get(fetch_file))
|
||||||
|
}
|
||||||
|
|
||||||
|
lazy_static! {
|
||||||
|
/// Short-lived file cache to allow us to populate different CDN regions without increasing bandwidth to S3 provider
|
||||||
|
/// Uploads will also be stored here to prevent immediately queued downloads from doing the entire round-trip
|
||||||
|
static ref S3_CACHE: moka::future::Cache<String, Result<Vec<u8>>> = moka::future::Cache::builder()
|
||||||
|
.max_capacity(10_000) // TODO config
|
||||||
|
.time_to_live(Duration::from_secs(60)) // TODO config
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retrieve hash information and file data by given hash
|
||||||
|
async fn retrieve_file_by_hash(hash: &FileHash) -> Result<Vec<u8>> {
|
||||||
|
if let Some(data) = S3_CACHE.get(&hash.id).await {
|
||||||
|
data
|
||||||
|
} else {
|
||||||
|
let data = fetch_from_s3(&hash.bucket_id, &hash.path, &hash.iv).await;
|
||||||
|
S3_CACHE.insert(hash.id.to_owned(), data.clone()).await;
|
||||||
|
data
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Successful root response
|
/// Successful root response
|
||||||
@@ -49,7 +80,7 @@ async fn root() -> Json<RootResponse> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Available tags to upload to
|
/// Available tags to upload to
|
||||||
#[derive(Deserialize, Debug, ToSchema)]
|
#[derive(Deserialize, Debug, ToSchema, strum_macros::IntoStaticStr)]
|
||||||
#[allow(non_camel_case_types)]
|
#[allow(non_camel_case_types)]
|
||||||
pub enum Tag {
|
pub enum Tag {
|
||||||
attachments,
|
attachments,
|
||||||
@@ -106,8 +137,14 @@ async fn upload_file(
|
|||||||
Ok(Json(UploadResponse { id: "aaa" }))
|
Ok(Json(UploadResponse { id: "aaa" }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Header value used for cache control
|
||||||
|
pub static CACHE_CONTROL: &str = "public, max-age=604800, must-revalidate";
|
||||||
|
|
||||||
/// Fetch preview of file
|
/// Fetch preview of file
|
||||||
///
|
///
|
||||||
|
/// This route will only return image content.
|
||||||
|
/// For all other file types, please use the fetch route (you will receive a redirect if you try to use this route anyways!).
|
||||||
|
///
|
||||||
/// Depending on the given tag, the file will be re-processed to fit the criteria:
|
/// Depending on the given tag, the file will be re-processed to fit the criteria:
|
||||||
///
|
///
|
||||||
/// | Tag | Image Resolution <sup>†</sup> |
|
/// | Tag | Image Resolution <sup>†</sup> |
|
||||||
@@ -132,18 +169,72 @@ async fn upload_file(
|
|||||||
),
|
),
|
||||||
)]
|
)]
|
||||||
async fn fetch_preview(
|
async fn fetch_preview(
|
||||||
Path(tag): Path<Tag>,
|
State(db): State<Database>,
|
||||||
Path(file_id): Path<String>,
|
Path((tag, file_id)): Path<(Tag, String)>,
|
||||||
) -> axum::response::Result<Response> {
|
) -> Result<Response> {
|
||||||
todo!()
|
let tag: &'static str = tag.into();
|
||||||
|
let file = db.fetch_attachment(tag, &file_id).await?;
|
||||||
|
|
||||||
|
// Ignore deleted files
|
||||||
|
if file.deleted.is_some_and(|v| v) {
|
||||||
|
return Err(create_error!(NotFound));
|
||||||
|
}
|
||||||
|
|
||||||
|
let hash = file.as_hash(&db).await?;
|
||||||
|
|
||||||
|
// Only process image files
|
||||||
|
if !hash.content_type.starts_with("image/") {
|
||||||
|
return Ok(
|
||||||
|
Redirect::permanent(&format!("/{tag}/{file_id}/{}", file.filename)).into_response(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Original image data
|
||||||
|
let data = retrieve_file_by_hash(&hash).await?;
|
||||||
|
|
||||||
|
// Dimensions we need to resize to
|
||||||
|
let config = config().await;
|
||||||
|
let [w, h] = config.files.preview.get(tag).unwrap();
|
||||||
|
|
||||||
|
// Read the image and resize it
|
||||||
|
let image = ImageReader::new(Cursor::new(data))
|
||||||
|
.with_guessed_format()
|
||||||
|
.map_err(|_| create_error!(InternalError))?
|
||||||
|
.decode()
|
||||||
|
.map_err(|_| create_error!(InternalError))?
|
||||||
|
// resize is about 2.5x slower,
|
||||||
|
// thumb approximation doesn't have terrible quality so it's fine to stick with
|
||||||
|
// .resize(width as u32, height as u32, image::imageops::FilterType::Gaussian)
|
||||||
|
// aspect ratio is preserved when scaling
|
||||||
|
.thumbnail(*w as u32, *h as u32);
|
||||||
|
|
||||||
|
// Encode it into WEBP
|
||||||
|
let encoder = webp::Encoder::from_image(&image).expect("Could not create encoder.");
|
||||||
|
let data = if config.files.webp_quality != 100.0 {
|
||||||
|
encoder.encode(config.files.webp_quality).to_vec()
|
||||||
|
} else {
|
||||||
|
encoder.encode_lossless().to_vec()
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok((
|
||||||
|
[
|
||||||
|
(header::CONTENT_TYPE, "image/webp"),
|
||||||
|
(header::CONTENT_DISPOSITION, "inline"),
|
||||||
|
(header::CACHE_CONTROL, CACHE_CONTROL),
|
||||||
|
],
|
||||||
|
data,
|
||||||
|
)
|
||||||
|
.into_response())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch original file
|
/// Fetch original file
|
||||||
|
///
|
||||||
|
/// Content disposition header will be set to 'attachment' to prevent browser from rendering anything.
|
||||||
#[utoipa::path(
|
#[utoipa::path(
|
||||||
get,
|
get,
|
||||||
path = "/{tag}/{file_id}/{file_name}",
|
path = "/{tag}/{file_id}/{file_name}",
|
||||||
responses(
|
responses(
|
||||||
(status = 200, description = "Generated preview", body = Vec<u8>)
|
(status = 200, description = "Original file", body = Vec<u8>)
|
||||||
),
|
),
|
||||||
params(
|
params(
|
||||||
("tag" = Tag, Path, description = "Tag to fetch from (e.g. attachments, icons, ...)"),
|
("tag" = Tag, Path, description = "Tag to fetch from (e.g. attachments, icons, ...)"),
|
||||||
@@ -152,9 +243,31 @@ async fn fetch_preview(
|
|||||||
),
|
),
|
||||||
)]
|
)]
|
||||||
async fn fetch_file(
|
async fn fetch_file(
|
||||||
Path(tag): Path<Tag>,
|
State(db): State<Database>,
|
||||||
Path(file_id): Path<String>,
|
Path((tag, file_id, file_name)): Path<(Tag, String, String)>,
|
||||||
Path(file_name): Path<String>,
|
) -> Result<Response> {
|
||||||
) -> axum::response::Result<Response> {
|
let file = db.fetch_attachment(tag.into(), &file_id).await?;
|
||||||
todo!()
|
|
||||||
|
// Ignore deleted files
|
||||||
|
if file.deleted.is_some_and(|v| v) {
|
||||||
|
return Err(create_error!(NotFound));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure filename is correct
|
||||||
|
if file_name != file.filename {
|
||||||
|
return Err(create_error!(NotFound));
|
||||||
|
}
|
||||||
|
|
||||||
|
let hash = file.as_hash(&db).await?;
|
||||||
|
retrieve_file_by_hash(&hash).await.map(|data| {
|
||||||
|
(
|
||||||
|
[
|
||||||
|
(header::CONTENT_TYPE, hash.content_type),
|
||||||
|
(header::CONTENT_DISPOSITION, "attachment".to_owned()),
|
||||||
|
(header::CACHE_CONTROL, CACHE_CONTROL.to_owned()),
|
||||||
|
],
|
||||||
|
data,
|
||||||
|
)
|
||||||
|
.into_response()
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ use std::net::{Ipv4Addr, SocketAddr};
|
|||||||
|
|
||||||
use axum::Router;
|
use axum::Router;
|
||||||
|
|
||||||
|
use revolt_database::DatabaseInfo;
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
use utoipa::{
|
use utoipa::{
|
||||||
openapi::security::{ApiKey, ApiKeyValue, SecurityScheme},
|
openapi::security::{ApiKey, ApiKeyValue, SecurityScheme},
|
||||||
@@ -55,10 +56,14 @@ async fn main() -> Result<(), std::io::Error> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Connect to the database
|
||||||
|
let db = DatabaseInfo::Auto.connect().await.unwrap();
|
||||||
|
|
||||||
// Configure Axum and router
|
// Configure Axum and router
|
||||||
let app = Router::new()
|
let app = Router::new()
|
||||||
.merge(Scalar::with_url("/scalar", ApiDoc::openapi()))
|
.merge(Scalar::with_url("/scalar", ApiDoc::openapi()))
|
||||||
.nest("/", api::router().await);
|
.nest("/", api::router().await)
|
||||||
|
.with_state(db);
|
||||||
|
|
||||||
// Configure TCP listener and bind
|
// Configure TCP listener and bind
|
||||||
let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, 3000));
|
let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, 3000));
|
||||||
|
|||||||
Reference in New Issue
Block a user