feat(services/autumn): file uploads

feat(services/autumn): deduplicate uploads
feat(services/autumn): ClamAV support

crates/core/database/src/models/file_hashes/model.rs

@@ -1,5 +1,7 @@
 use iso8601_timestamp::Timestamp;
 
+use crate::File;
+
 auto_derived_partial!(
     /// File hash
     pub struct FileHash {
@@ -51,3 +53,40 @@ auto_derived!(
         Audio,
     }
 );
+
+impl FileHash {
+    /// Create a file from a file hash
+    pub fn into_file(
+        &self,
+        id: String,
+        tag: String,
+        filename: String,
+        uploader_id: String,
+    ) -> File {
+        File {
+            id,
+            tag,
+            filename,
+            hash: Some(self.id.clone()),
+
+            uploaded_at: Some(Timestamp::now_utc()),
+            uploader_id: Some(uploader_id),
+
+            used_for: None,
+
+            deleted: None,
+            reported: None,
+
+            // TODO: remove this data
+            metadata: self.metadata.clone(),
+            content_type: self.content_type.clone(),
+            size: self.size,
+
+            // TODO: superseded by "used_for"
+            message_id: None,
+            object_id: None,
+            server_id: None,
+            user_id: None,
+        }
+    }
+}

crates/core/database/src/models/file_hashes/ops.rs

@@ -7,6 +7,12 @@ mod reference;
 
 #[async_trait]
 pub trait AbstractAttachmentHashes: Sync + Send {
+    /// Insert a new attachment hash into the database.
+    async fn insert_attachment_hash(&self, hash: &FileHash) -> Result<()>;
+
     /// Fetch an attachment hash entry by sha256 hash.
     async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash>;
+
+    /// Update an attachment hash nonce value.
+    async fn set_attachment_hash_nonce(&self, hash: &str, nonce: &str) -> Result<()>;
 }

crates/core/database/src/models/file_hashes/ops/mongodb.rs

@@ -9,6 +9,11 @@ static COL: &str = "attachment_hashes";
 
 #[async_trait]
 impl AbstractAttachmentHashes for MongoDb {
+    /// Insert a new attachment hash into the database.
+    async fn insert_attachment_hash(&self, hash: &FileHash) -> Result<()> {
+        query!(self, insert_one, COL, &hash).map(|_| ())
+    }
+
     /// Fetch an attachment hash entry by sha256 hash.
     async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash> {
         query!(
@@ -16,9 +21,31 @@ impl AbstractAttachmentHashes for MongoDb {
             find_one,
             COL,
             doc! {
-                "_id": hash
+                "$or": [
+                    {"_id": hash},
+                    {"processed_hash": hash}
+                ]
             }
         )?
         .ok_or_else(|| create_error!(NotFound))
     }
+
+    /// Update an attachment hash nonce value.
+    async fn set_attachment_hash_nonce(&self, hash: &str, nonce: &str) -> Result<()> {
+        self.col::<FileHash>(COL)
+            .update_one(
+                doc! {
+                    "_id": hash
+                },
+                doc! {
+                    "$set": {
+                        "iv": nonce
+                    }
+                },
+                None,
+            )
+            .await
+            .map(|_| ())
+            .map_err(|_| create_database_error!("update_one", COL))
+    }
 }

crates/core/database/src/models/file_hashes/ops/reference.rs

@@ -7,15 +7,33 @@ use super::AbstractAttachmentHashes;
 
 #[async_trait]
 impl AbstractAttachmentHashes for ReferenceDb {
+    /// Insert a new attachment hash into the database.
+    async fn insert_attachment_hash(&self, hash: &FileHash) -> Result<()> {
+        let mut hashes = self.file_hashes.lock().await;
+        if hashes.contains_key(&hash.id) {
+            Err(create_database_error!("insert", "attachment"))
+        } else {
+            hashes.insert(hash.id.to_string(), hash.clone());
+            Ok(())
+        }
+    }
+
     /// Fetch an attachment hash entry by sha256 hash.
-    async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash> {
+    async fn fetch_attachment_hash(&self, hash_value: &str) -> Result<FileHash> {
         let hashes = self.file_hashes.lock().await;
-        if let Some(file) = hashes.get(hash) {
-            if file.id == hash {
-                Ok(file.clone())
-            } else {
-                Err(create_error!(NotFound))
-            }
+        hashes
+            .values()
+            .cloned()
+            .find(|hash| hash.id == hash_value || hash.processed_hash == hash_value)
+            .ok_or(create_error!(NotFound))
+    }
+
+    /// Update an attachment hash nonce value.
+    async fn set_attachment_hash_nonce(&self, hash: &str, nonce: &str) -> Result<()> {
+        let mut hashes = self.file_hashes.lock().await;
+        if let Some(file) = hashes.get_mut(hash) {
+            file.iv = nonce.to_owned();
+            Ok(())
         } else {
             Err(create_error!(NotFound))
         }

crates/core/database/src/models/files/model.rs

@@ -19,7 +19,7 @@ auto_derived_partial!(
         /// When this file was uploaded
         pub uploaded_at: Option<Timestamp>, // these are Option<>s to not break file uploads on legacy Autumn
         /// ID of user who uploaded this file
-        pub uploaded_id: Option<String>, // these are Option<>s to not break file uploads on legacy Autumn
+        pub uploader_id: Option<String>, // these are Option<>s to not break file uploads on legacy Autumn
 
         /// What the file was used for
         pub used_for: Option<FileUsedFor>,
@@ -59,15 +59,14 @@ auto_derived_partial!(
 auto_derived!(
     /// Type of object file was used for
     pub enum FileUsedForType {
-        // TODO: changing this requires a db migration
-        message,
-        serverBanner,
-        emoji,
-        userAvatar,
-        userProfileBackground,
-        legacyGroupIcon,
-        channelIcon,
-        serverIcon,
+        Message,
+        ServerBanner,
+        Emoji,
+        UserAvatar,
+        UserProfileBackground,
+        LegacyGroupIcon,
+        ChannelIcon,
+        ServerIcon,
     }
 
     /// Information about what the file was used for