chore: strip dotenv (unmaintained) from project

It is no longer needed as configuration is loaded via. TOML or direct env if appropriate.

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
   "editor.formatOnSave": true,
   "rust-analyzer.checkOnSave.command": "clippy",
-  "nixEnvSelector.suggestion": false
-}
+  "nixEnvSelector.suggestion": false,
+  "nixEnvSelector.nixFile": "${workspaceFolder}/default.nix"
+}

Cargo.lock

@@ -2004,12 +2004,6 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
 
-[[package]]
-name = "dotenv"
-version = "0.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
-
 [[package]]
 name = "dtoa"
 version = "1.0.9"
@@ -5580,7 +5574,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-autumn"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "axum",
  "axum-macros",
@@ -5617,7 +5611,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-bonfire"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "async-channel 2.3.1",
  "async-std",
@@ -5647,12 +5641,11 @@ dependencies = [
 
 [[package]]
 name = "revolt-config"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "async-std",
  "cached",
  "config",
- "dotenv",
  "futures-locks",
  "log",
  "once_cell",
@@ -5664,7 +5657,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-database"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "async-lock 2.8.0",
  "async-recursion",
@@ -5712,7 +5705,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-delta"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "async-channel 1.6.1",
  "async-std",
@@ -5720,7 +5713,6 @@ dependencies = [
  "bitfield",
  "chrono",
  "dashmap",
- "dotenv",
  "env_logger",
  "futures",
  "impl_ops",
@@ -5759,7 +5751,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-files"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "aes-gcm 0.10.3",
  "aws-config",
@@ -5782,7 +5774,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-january"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "async-recursion",
  "axum",
@@ -5810,7 +5802,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-models"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "indexmap 1.9.3",
  "iso8601-timestamp 0.2.11",
@@ -5828,7 +5820,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-nodejs-bindings"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "async-std",
  "neon",
@@ -5841,7 +5833,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-permissions"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "async-std",
  "async-trait",
@@ -5856,7 +5848,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-presence"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "async-std",
  "log",
@@ -5867,7 +5859,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-result"
-version = "0.7.18"
+version = "0.7.19"
 dependencies = [
  "axum",
  "revolt_okapi",

README.md

@@ -95,17 +95,21 @@ Then continue:
 # start other necessary services
 docker compose up -d
 
+# run everything together
+./scripts/start.sh
+# .. or individually
 # run the API server
 cargo run --bin revolt-delta
 # run the events server
 cargo run --bin revolt-bonfire
 # run the file server
 cargo run --bin revolt-autumn
-# run th proxy server
+# run the proxy server
 cargo run --bin revolt-january
 
 # hint:
 # mold -run <cargo build, cargo run, etc...>
+# mold -run ./scripts/start.sh
 ```
 
 You can start a web client by doing the following:

compose.yml

@@ -32,13 +32,12 @@ services:
     image: minio/mc
     depends_on:
       - minio
-    environment:
-      MINIO_ROOT_USER: minioautumn
-      MINIO_ROOT_PASSWORD: minioautumn
     entrypoint: >
       /bin/sh -c "
-      /usr/bin/mc config host add minio http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD;
-      while ! /usr/bin/mc ready minio; do echo 'Waiting minio...' && sleep 1; done;
+      while ! /usr/bin/mc ready minio; do
+        /usr/bin/mc config host add minio http://minio:9000 minioautumn minioautumn;
+        echo 'Waiting minio...' && sleep 1;
+      done;
       /usr/bin/mc mb minio/revolt-uploads;
       exit 0;
       "

crates/bindings/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-nodejs-bindings"
-version = "0.7.18"
+version = "0.7.19"
 description = "Node.js bindings for the Revolt software"
 authors = ["Paul Makles <me@insrt.uk>"]
 license = "MIT"
@@ -20,6 +20,6 @@ serde = { version = "1", features = ["derive"] }
 
 async-std = "1.12.0"
 
-revolt-config = { version = "0.7.18", path = "../../core/config" }
-revolt-result = { version = "0.7.18", path = "../../core/result" }
-revolt-database = { version = "0.7.18", path = "../../core/database" }
+revolt-config = { version = "0.7.19", path = "../../core/config" }
+revolt-result = { version = "0.7.19", path = "../../core/result" }
+revolt-database = { version = "0.7.19", path = "../../core/database" }

crates/bonfire/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-bonfire"
-version = "0.7.18"
+version = "0.7.19"
 license = "AGPL-3.0-or-later"
 edition = "2021"
 
@@ -41,7 +41,7 @@ revolt-result = { path = "../core/result" }
 revolt-models = { path = "../core/models" }
 revolt-config = { path = "../core/config" }
 revolt-database = { path = "../core/database" }
-revolt-permissions = { version = "0.7.18", path = "../core/permissions" }
+revolt-permissions = { version = "0.7.19", path = "../core/permissions" }
 revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] }
 
 # redis

crates/bonfire/src/websocket.rs

@@ -79,7 +79,9 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr)
     // Try to authenticate the user.
     let Some(token) = config.get_session_token().as_ref() else {
         write
-            .send(config.encode(&create_error!(InvalidSession)))
+            .send(config.encode(&EventV1::Error {
+                data: create_error!(InvalidSession),
+            }))
             .await
             .ok();
         return;
@@ -88,7 +90,10 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr)
     let (user, session_id) = match User::from_token(db, token, UserHint::Any).await {
         Ok(user) => user,
         Err(err) => {
-            write.send(config.encode(&err)).await.ok();
+            write
+                .send(config.encode(&EventV1::Error { data: err }))
+                .await
+                .ok();
             return;
         }
     };
@@ -301,25 +306,23 @@ async fn listener(
                     PayloadType::Json => message
                         .value
                         .as_str()
-                        .and_then(|s| serde_json::from_str::<EventV1>(s.as_ref()).ok()),
+                        .and_then(|s| report_internal_error!(serde_json::from_str::<EventV1>(s.as_ref())).ok()),
                     PayloadType::Msgpack => message
                         .value
                         .as_bytes()
-                        .and_then(|b| rmp_serde::from_slice::<EventV1>(b).ok()),
+                        .and_then(|b| report_internal_error!(rmp_serde::from_slice::<EventV1>(b)).ok()),
                     PayloadType::Bincode => message
                         .value
                         .as_bytes()
-                        .and_then(|b| bincode::deserialize::<EventV1>(b).ok()),
+                        .and_then(|b| report_internal_error!(bincode::deserialize::<EventV1>(b)).ok()),
                 };
 
                 let Some(mut event) = event else {
                     let err = format!(
-                        "Failed to deserialise an event for {}! Introspection: `{:?}`",
+                        "Failed to deserialise event for {}: `{:?}`",
                         message.channel,
                         message
                             .value
-                            .as_string()
-                            .map(|x| x.chars().take(32).collect::<String>())
                     );
 
                     error!("{}", err);

crates/core/config/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-config"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -15,7 +15,6 @@ default = ["test"]
 
 [dependencies]
 # Utility
-dotenv = "0.15.0"
 config = "0.13.3"
 cached = "0.44.0"
 once_cell = "1.18.0"
@@ -35,4 +34,4 @@ pretty_env_logger = "0.4.0"
 sentry = "0.31.5"
 
 # Core
-revolt-result = { version = "0.7.18", path = "../result", optional = true }
+revolt-result = { version = "0.7.19", path = "../result", optional = true }

crates/core/config/Revolt.toml

@@ -132,6 +132,7 @@ emojis = [128, 128]
 #
 # Backblaze B2:
 # - endpoint is listed on the "Buckets" page
+# - path_style_buckets is set to true
 # - region is `eu-central-003` string from endpoint URL
 # - access_key_id is keyID generated on the "Application Keys" page
 # - secret_access_key is token generated on the "Application Keys" page
@@ -139,6 +140,9 @@ emojis = [128, 128]
 
 # S3 protocol endpoint
 endpoint = "http://minio:9000"
+# Whether to use path-style buckets
+# Generally true, except for MinIO
+path_style_buckets = false
 # S3 region name
 region = "minio"
 # S3 protocol key ID

crates/core/config/src/lib.rs

@@ -175,6 +175,7 @@ pub struct FilesLimit {
 #[derive(Deserialize, Debug, Clone)]
 pub struct FilesS3 {
     pub endpoint: String,
+    pub path_style_buckets: bool,
     pub region: String,
     pub access_key_id: String,
     pub secret_access_key: String,
@@ -286,8 +287,6 @@ pub async fn config() -> Settings {
 
 /// Configure logging and common Rust variables
 pub async fn setup_logging(release: &'static str, dsn: String) -> Option<sentry::ClientInitGuard> {
-    dotenv::dotenv().ok();
-
     if std::env::var("RUST_LOG").is_err() {
         std::env::set_var("RUST_LOG", "info");
     }

crates/core/database/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-database"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -24,15 +24,15 @@ default = ["mongodb", "async-std-runtime", "tasks"]
 
 [dependencies]
 # Core
-revolt-config = { version = "0.7.18", path = "../config", features = [
+revolt-config = { version = "0.7.19", path = "../config", features = [
     "report-macros",
 ] }
-revolt-result = { version = "0.7.18", path = "../result" }
-revolt-models = { version = "0.7.18", path = "../models", features = [
+revolt-result = { version = "0.7.19", path = "../result" }
+revolt-models = { version = "0.7.19", path = "../models", features = [
     "validator",
 ] }
-revolt-presence = { version = "0.7.18", path = "../presence" }
-revolt-permissions = { version = "0.7.18", path = "../permissions", features = [
+revolt-presence = { version = "0.7.19", path = "../presence" }
+revolt-permissions = { version = "0.7.19", path = "../permissions", features = [
     "serde",
     "bson",
 ] }

crates/core/database/src/events/client.rs

@@ -1,25 +1,16 @@
 use authifier::AuthifierEvent;
+use revolt_result::Error;
 use serde::{Deserialize, Serialize};
 
 use revolt_models::v0::{
-    AppendMessage, Channel, ChannelUnread, Emoji, FieldsChannel, FieldsMember, FieldsMessage, FieldsRole, FieldsServer, FieldsUser, FieldsWebhook, Member, MemberCompositeKey, Message, PartialChannel, PartialMember, PartialMessage, PartialRole, PartialServer, PartialUser, PartialWebhook, RemovalIntention, Report, Server, User, UserSettings, Webhook
+    AppendMessage, Channel, ChannelUnread, Emoji, FieldsChannel, FieldsMember, FieldsMessage,
+    FieldsRole, FieldsServer, FieldsUser, FieldsWebhook, Member, MemberCompositeKey, Message,
+    PartialChannel, PartialMember, PartialMessage, PartialRole, PartialServer, PartialUser,
+    PartialWebhook, RemovalIntention, Report, Server, User, UserSettings, Webhook,
 };
-use revolt_result::Error;
 
 use crate::Database;
 
-/// WebSocket Client Errors
-#[derive(Serialize, Deserialize, Debug, Clone)]
-#[serde(tag = "error")]
-pub enum WebSocketError {
-    LabelMe,
-    InternalError { at: String },
-    InvalidSession,
-    OnboardingNotFinished,
-    AlreadyAuthenticated,
-    MalformedData { msg: String },
-}
-
 /// Ping Packet
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(untagged)]
@@ -28,14 +19,6 @@ pub enum Ping {
     Number(usize),
 }
 
-/// Untagged Error
-#[derive(Serialize)]
-#[serde(untagged)]
-pub enum ErrorEvent {
-    Error(WebSocketError),
-    APIError(Error),
-}
-
 /// Fields provided in Ready payload
 #[derive(PartialEq)]
 pub enum ReadyPayloadFields {
@@ -55,6 +38,8 @@ pub enum ReadyPayloadFields {
 pub enum EventV1 {
     /// Multiple events
     Bulk { v: Vec<EventV1> },
+    /// Error event
+    Error { data: Error },
 
     /// Successfully authenticated
     Authenticated,
@@ -89,6 +74,7 @@ pub enum EventV1 {
         id: String,
         channel: String,
         data: PartialMessage,
+        #[serde(default)]
         clear: Vec<FieldsMessage>,
     },
 
@@ -140,6 +126,7 @@ pub enum EventV1 {
     ServerUpdate {
         id: String,
         data: PartialServer,
+        #[serde(default)]
         clear: Vec<FieldsServer>,
     },
 
@@ -150,6 +137,7 @@ pub enum EventV1 {
     ServerMemberUpdate {
         id: MemberCompositeKey,
         data: PartialMember,
+        #[serde(default)]
         clear: Vec<FieldsMember>,
     },
 
@@ -168,6 +156,7 @@ pub enum EventV1 {
         id: String,
         role_id: String,
         data: PartialRole,
+        #[serde(default)]
         clear: Vec<FieldsRole>,
     },
 
@@ -178,6 +167,7 @@ pub enum EventV1 {
     UserUpdate {
         id: String,
         data: PartialUser,
+        #[serde(default)]
         clear: Vec<FieldsUser>,
         event_id: Option<String>,
     },
@@ -212,6 +202,7 @@ pub enum EventV1 {
     ChannelUpdate {
         id: String,
         data: PartialChannel,
+        #[serde(default)]
         clear: Vec<FieldsChannel>,
     },
 

crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs

@@ -11,6 +11,7 @@ use bson::oid::ObjectId;
 use futures::StreamExt;
 use rand::seq::SliceRandom;
 use revolt_permissions::DEFAULT_WEBHOOK_PERMISSIONS;
+use revolt_result::{Error, ErrorType};
 use serde::{Deserialize, Serialize};
 use unicode_segmentation::UnicodeSegmentation;
 
@@ -20,7 +21,7 @@ struct MigrationInfo {
     revision: i32,
 }
 
-pub const LATEST_REVISION: i32 = 30;
+pub const LATEST_REVISION: i32 = 31;
 
 pub async fn migrate_database(db: &MongoDb) {
     let migrations = db.col::<Document>("migrations");
@@ -1139,53 +1140,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
             .expect("Failed to create attachment_hashes index.");
     }
 
-    if revision <= 29 {
-        info!("Running migration [revision 29 / 29-09-2024]: Add creator_id to webhooks.");
-
-        #[derive(serde::Serialize, serde::Deserialize)]
-        struct WebhookShell {
-            _id: String,
-            channel_id: String,
-        }
-
-        let invites = db
-            .db()
-            .collection::<WebhookShell>("channel_webhooks")
-            .find(doc! {}, None)
-            .await
-            .expect("webhooks")
-            .filter_map(|s| async { s.ok() })
-            .collect::<Vec<WebhookShell>>()
-            .await;
-
-        for invite in invites {
-            let channel = db.fetch_channel(&invite.channel_id).await.expect("channel");
-            let creator_id = match channel {
-                Channel::Group { owner, .. } => owner,
-                Channel::TextChannel { server, .. } | Channel::VoiceChannel { server, .. } => {
-                    let server = db.fetch_server(&server).await.expect("server");
-                    server.owner
-                }
-                _ => unreachable!("not server or group channel!"),
-            };
-
-            db.db()
-                .collection::<Document>("channel_webhooks")
-                .update_one(
-                    doc! {
-                        "_id": invite._id,
-                    },
-                    doc! {
-                        "$set" : {
-                            "creator_id": creator_id
-                        }
-                    },
-                    None,
-                )
-                .await
-                .expect("update webhook");
-        }
-    }
+    // Revision 29 omitted due to bug.
 
     if revision <= 30 {
         info!("Running migration [revision 30 / 29-09-2024]: Add index for used_for.id to attachments.");
@@ -1209,7 +1164,68 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
             .expect("Failed to create attachments index.");
     }
 
-    // Need to migrate fields on attachments, change `user_id`, `object_id`, etc to `parent`.
+    if revision <= 31 {
+        info!("Running migration [revision 31 / 31-10-2024]: Add creator_id to webhooks and delete those whose channels don't exist.");
+
+        #[derive(serde::Serialize, serde::Deserialize)]
+        struct WebhookShell {
+            _id: String,
+            channel_id: String,
+        }
+
+        let webhooks = db
+            .db()
+            .collection::<WebhookShell>("channel_webhooks")
+            .find(doc! {}, None)
+            .await
+            .expect("webhooks")
+            .filter_map(|s| async { s.ok() })
+            .collect::<Vec<WebhookShell>>()
+            .await;
+
+        for webhook in webhooks {
+            match db.fetch_channel(&webhook.channel_id).await {
+                Ok(channel) => {
+                    let creator_id = match channel {
+                        Channel::Group { owner, .. } => owner,
+                        Channel::TextChannel { server, .. }
+                        | Channel::VoiceChannel { server, .. } => {
+                            let server = db.fetch_server(&server).await.expect("server");
+                            server.owner
+                        }
+                        _ => unreachable!("not server or group channel!"),
+                    };
+
+                    db.db()
+                        .collection::<Document>("channel_webhooks")
+                        .update_one(
+                            doc! {
+                                "_id": webhook._id,
+                            },
+                            doc! {
+                                "$set" : {
+                                    "creator_id": creator_id
+                                }
+                            },
+                            None,
+                        )
+                        .await
+                        .expect("update webhook");
+                }
+                Err(Error {
+                    error_type: ErrorType::NotFound,
+                    ..
+                }) => {
+                    db.db()
+                        .collection::<WebhookShell>("channel_webhooks")
+                        .delete_one(doc! { "_id": webhook._id }, None)
+                        .await
+                        .expect("failed to delete invalid webhook");
+                }
+                Err(err) => panic!("{err:?}"),
+            }
+        }
+    }
 
     // Reminder to update LATEST_REVISION when adding new migrations.
     LATEST_REVISION.max(revision)

crates/core/database/src/models/messages/model.rs

@@ -1,4 +1,4 @@
-use std::collections::HashSet;
+use std::{collections::HashSet, hash::RandomState};
 
 use indexmap::{IndexMap, IndexSet};
 use iso8601_timestamp::Timestamp;
@@ -338,7 +338,34 @@ impl Message {
         }
 
         if !mentions.is_empty() {
-            message.mentions.replace(mentions.into_iter().collect());
+            // FIXME: temp fix to stop spam attacks
+            match channel {
+                Channel::DirectMessage { ref recipients, .. }
+                | Channel::Group { ref recipients, .. } => {
+                    let recipients_hash: HashSet<&String, RandomState> =
+                        HashSet::from_iter(recipients.iter());
+
+                    mentions.retain(|m| recipients_hash.contains(m));
+                }
+                Channel::TextChannel { ref server, .. }
+                | Channel::VoiceChannel { ref server, .. } => {
+                    let mentions_vec = Vec::from_iter(mentions.iter().cloned());
+                    let valid_members = db.fetch_members(server.as_str(), &mentions_vec[..]).await;
+                    if let Ok(valid_members) = valid_members {
+                        let valid_ids: HashSet<String, RandomState> = HashSet::from_iter(
+                            valid_members.iter().map(|member| member.id.user.clone()),
+                        );
+                        mentions.retain(|m| valid_ids.contains(m));
+                    } else {
+                        revolt_config::capture_error(&valid_members.unwrap_err());
+                    }
+                }
+                Channel::SavedMessages { .. } => mentions.clear(),
+            }
+
+            if !mentions.is_empty() {
+                message.mentions.replace(mentions.into_iter().collect());
+            }
         }
 
         if !replies.is_empty() {

crates/core/files/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-files"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -20,10 +20,10 @@ typenum = "1.17.0"
 aws-config = "1.5.5"
 aws-sdk-s3 = { version = "1.46.0", features = ["behavior-version-latest"] }
 
-revolt-config = { version = "0.7.18", path = "../config", features = [
+revolt-config = { version = "0.7.19", path = "../config", features = [
     "report-macros",
 ] }
-revolt-result = { version = "0.7.18", path = "../result" }
+revolt-result = { version = "0.7.19", path = "../result" }
 
 # image processing
 jxl-oxide = "0.8.1"

crates/core/files/src/lib.rs

@@ -34,6 +34,7 @@ pub fn create_client(s3_config: FilesS3) -> Client {
     let config = Config::builder()
         .region(Region::new(s3_config.region))
         .endpoint_url(s3_config.endpoint)
+        .force_path_style(s3_config.path_style_buckets)
         .credentials_provider(creds)
         .build();
 
@@ -124,13 +125,24 @@ pub fn image_size(f: &NamedTempFile) -> Option<(usize, usize)> {
 }
 
 /// Determine size of image with buffer
-pub fn image_size_vec(v: &[u8]) -> Option<(usize, usize)> {
-    if let Ok(size) = imagesize::blob_size(v)
-        .inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}"))
-    {
-        Some((size.width, size.height))
-    } else {
-        None
+pub fn image_size_vec(v: &[u8], mime: &str) -> Option<(usize, usize)> {
+    match mime {
+        "image/svg+xml" => {
+            let tree =
+                report_internal_error!(usvg::Tree::from_data(v, &Default::default())).ok()?;
+
+            let size = tree.size();
+            Some((size.width() as usize, size.height() as usize))
+        }
+        _ => {
+            if let Ok(size) = imagesize::blob_size(v)
+                .inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}"))
+            {
+                Some((size.width, size.height))
+            } else {
+                None
+            }
+        }
     }
 }
 
@@ -167,7 +179,7 @@ pub fn decode_image<R: Read + BufRead + Seek>(reader: &mut R, mime: &str) -> Res
                                 jxl_image.height(),
                                 frame.image().buf().to_vec(),
                             )
-                            .ok_or_else(|| create_error!(LabelMe))?,
+                            .ok_or_else(|| create_error!(ImageProcessingFailed))?,
                         )
                         .to_rgb8(),
                     )),
@@ -178,14 +190,14 @@ pub fn decode_image<R: Read + BufRead + Seek>(reader: &mut R, mime: &str) -> Res
                                 jxl_image.height(),
                                 frame.image().buf().to_vec(),
                             )
-                            .ok_or_else(|| create_error!(LabelMe))?,
+                            .ok_or_else(|| create_error!(ImageProcessingFailed))?,
                         )
                         .to_rgba8(),
                     )),
-                    _ => Err(create_error!(LabelMe)),
+                    _ => Err(create_error!(ImageProcessingFailed)),
                 }
             } else {
-                Err(create_error!(LabelMe))
+                Err(create_error!(ImageProcessingFailed))
             }
         }
         // Read image using resvg
@@ -197,7 +209,7 @@ pub fn decode_image<R: Read + BufRead + Seek>(reader: &mut R, mime: &str) -> Res
             let tree = report_internal_error!(usvg::Tree::from_data(&buf, &Default::default()))?;
             let size = tree.size();
             let mut pixmap = Pixmap::new(size.width() as u32, size.height() as u32)
-                .ok_or_else(|| create_error!(LabelMe))?;
+                .ok_or_else(|| create_error!(ImageProcessingFailed))?;
 
             let mut pixmap_mut = pixmap.as_mut();
             resvg::render(&tree, Default::default(), &mut pixmap_mut);
@@ -208,7 +220,7 @@ pub fn decode_image<R: Read + BufRead + Seek>(reader: &mut R, mime: &str) -> Res
                     size.height() as u32,
                     pixmap.data().to_vec(),
                 )
-                .ok_or_else(|| create_error!(LabelMe))?,
+                .ok_or_else(|| create_error!(ImageProcessingFailed))?,
             ))
         }
         // Check if we can read using image-rs crate

crates/core/models/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-models"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -20,8 +20,8 @@ default = ["serde", "partials", "rocket"]
 
 [dependencies]
 # Core
-revolt-config = { version = "0.7.18", path = "../config" }
-revolt-permissions = { version = "0.7.18", path = "../permissions" }
+revolt-config = { version = "0.7.19", path = "../config" }
+revolt-permissions = { version = "0.7.19", path = "../permissions" }
 
 # Utility
 regex = "1"

crates/core/permissions/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-permissions"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] }
 
 [dependencies]
 # Core
-revolt-result = { version = "0.7.18", path = "../result" }
+revolt-result = { version = "0.7.19", path = "../result" }
 
 # Utility
 auto_ops = "0.3.0"

crates/core/presence/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-presence"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <me@insrt.uk>"]

crates/core/result/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-result"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]

crates/core/result/src/axum.rs

@@ -1,4 +1,5 @@
 use axum::{http::StatusCode, response::IntoResponse, Json};
+use rocket::http::Status;
 
 use crate::{Error, ErrorType};
 
@@ -78,6 +79,8 @@ impl IntoResponse for Error {
             ErrorType::FileTooSmall => StatusCode::UNPROCESSABLE_ENTITY,
             ErrorType::FileTooLarge { .. } => StatusCode::UNPROCESSABLE_ENTITY,
             ErrorType::FileTypeNotAllowed => StatusCode::BAD_REQUEST,
+            ErrorType::ImageProcessingFailed => StatusCode::INTERNAL_SERVER_ERROR,
+            ErrorType::NoEmbedData => StatusCode::BAD_REQUEST,
         };
 
         (status, Json(&self)).into_response()

crates/core/result/src/lib.rs

@@ -162,6 +162,8 @@ pub enum ErrorType {
         max: usize,
     },
     FileTypeNotAllowed,
+    ImageProcessingFailed,
+    NoEmbedData,
 
     // ? Legacy errors
     VosoUnavailable,

crates/core/result/src/rocket.rs

@@ -84,6 +84,8 @@ impl<'r> Responder<'r, 'static> for Error {
             ErrorType::FileTooSmall => Status::UnprocessableEntity,
             ErrorType::FileTooLarge { .. } => Status::UnprocessableEntity,
             ErrorType::FileTypeNotAllowed => Status::BadRequest,
+            ErrorType::ImageProcessingFailed => Status::InternalServerError,
+            ErrorType::NoEmbedData => Status::BadRequest,
         };
 
         // Serialize the error data structure into JSON.

crates/delta/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-delta"
-version = "0.7.18"
+version = "0.7.19"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <paulmakles@gmail.com>"]
 edition = "2018"
@@ -16,7 +16,6 @@ redis-kiss = "0.1.4"
 lru = "0.7.0"
 url = "2.2.2"
 log = "0.4.11"
-dotenv = "0.15.0"
 dashmap = "5.2.0"
 linkify = "0.6.0"
 once_cell = "1.17.1"

crates/delta/src/util/test.rs

@@ -19,8 +19,6 @@ pub struct TestHarness {
 
 impl TestHarness {
     pub async fn new() -> TestHarness {
-        dotenv::dotenv().ok();
-
         let client = Client::tracked(crate::web().await)
             .await
             .expect("valid rocket instance");

crates/services/autumn/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-autumn"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 
 [dependencies]
@@ -13,7 +13,7 @@ webp = "0.3.0"
 sha2 = "0.10.8"
 jxl-oxide = "0.8.1"
 kamadak-exif = "0.5.4"
-# revolt_little_exif = "0.4.0"
+# revolt_little_exif = "0.5.1"
 image = { version = "0.25.2" } # avif encode requires dav1d system library: features = ["avif-native"]
 
 # File processing
@@ -42,12 +42,12 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 # Core crates
-revolt-files = { version = "0.7.18", path = "../../core/files" }
-revolt-config = { version = "0.7.18", path = "../../core/config" }
-revolt-database = { version = "0.7.18", path = "../../core/database", features = [
+revolt-files = { version = "0.7.19", path = "../../core/files" }
+revolt-config = { version = "0.7.19", path = "../../core/config" }
+revolt-database = { version = "0.7.19", path = "../../core/database", features = [
     "axum-impl",
 ] }
-revolt-result = { version = "0.7.18", path = "../../core/result", features = [
+revolt-result = { version = "0.7.19", path = "../../core/result", features = [
     "utoipa",
     "axum",
 ] }

crates/services/autumn/src/api.rs

@@ -207,16 +207,27 @@ async fn upload_file(
     };
 
     // Find an existing hash and use that if possible
-    if let Ok(file_hash) = db
+    let file_hash_exists = if let Ok(file_hash) = db
         .fetch_attachment_hash(&format!("{original_hash:02x}"))
         .await
     {
-        let tag: &'static str = tag.into();
-        db.insert_attachment(&file_hash.into_file(id.clone(), tag.to_owned(), filename, user.id))
+        if !file_hash.iv.is_empty() {
+            let tag: &'static str = tag.into();
+            db.insert_attachment(&file_hash.into_file(
+                id.clone(),
+                tag.to_owned(),
+                filename,
+                user.id,
+            ))
             .await?;
 
-        return Ok(Json(UploadResponse { id }));
-    }
+            return Ok(Json(UploadResponse { id }));
+        }
+
+        true
+    } else {
+        false
+    };
 
     // Determine the mime type for the file
     let mime_type = determine_mime_type(&mut file.contents, &buf, &filename);
@@ -274,7 +285,10 @@ async fn upload_file(
         size: new_file_size as isize,
     };
 
-    db.insert_attachment_hash(&file_hash).await?;
+    // Add attachment hash if it doesn't exist
+    if !file_hash_exists {
+        db.insert_attachment_hash(&file_hash).await?;
+    }
 
     // Upload the file to S3 and commit nonce to database
     let upload_start = Instant::now();

crates/services/autumn/src/exif.rs

@@ -17,23 +17,24 @@ pub async fn strip_metadata(
 ) -> Result<(Vec<u8>, Metadata)> {
     match &metadata {
         Metadata::Image { width, height } => match mime {
-            // little_exif does not appear to parse JPEGs correctly? had 2/2 files fail
-            /* "image/jpeg" | "image/png" => {
-                // use little_exif to strip metadata except for orientation and colour profile
-                // PNGs must also be re-encoded to mitigate CVE-2023-21036
-                let metadata = revolt_little_exif::metadata::Metadata::new_from_path_with_filetype(
-                    file.path(),
-                    match mime {
-                        "image/jpeg" => revolt_little_exif::filetype::FileExtension::JPEG,
-                        "image/png" => revolt_little_exif::filetype::FileExtension::PNG {
-                            as_zTXt_chunk: true,
-                        },
-                        _ => unreachable!(),
-                    },
-                )
-                .unwrap();
-                dbg!(metadata.data());
-            } */
+            // // little_exif does not appear to parse JPEGs correctly? had 2/2 files fail
+            // "image/jpeg" | "image/png" => {
+            //     // use little_exif to strip metadata except for orientation and colour profile
+            //     // PNGs must also be re-encoded to mitigate CVE-2023-21036
+            //     let metadata = revolt_little_exif::metadata::Metadata::new_from_path_with_filetype(
+            //         file.path(),
+            //         match mime {
+            //             "image/jpeg" => revolt_little_exif::filetype::FileExtension::JPEG,
+            //             "image/png" => revolt_little_exif::filetype::FileExtension::PNG {
+            //                 as_zTXt_chunk: true,
+            //             },
+            //             _ => unreachable!(),
+            //         },
+            //     )
+            //     .unwrap();
+            //     dbg!(metadata.data());
+            //     todo!()
+            // }
             // Apply orientation manually & strip all other EXIF data
             "image/jpeg" | "image/png" | "image/avif" | "image/tiff" => {
                 // Create a reader

crates/services/january/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-january"
-version = "0.7.18"
+version = "0.7.19"
 edition = "2021"
 
 [dependencies]
@@ -31,13 +31,13 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 # Core crates
-revolt-config = { version = "0.7.18", path = "../../core/config" }
-revolt-models = { version = "0.7.18", path = "../../core/models" }
-revolt-result = { version = "0.7.18", path = "../../core/result", features = [
+revolt-config = { version = "0.7.19", path = "../../core/config" }
+revolt-models = { version = "0.7.19", path = "../../core/models" }
+revolt-result = { version = "0.7.19", path = "../../core/result", features = [
     "utoipa",
     "axum",
 ] }
-revolt-files = { version = "0.7.18", path = "../../core/files" }
+revolt-files = { version = "0.7.19", path = "../../core/files" }
 
 # Axum / web server
 axum = { version = "0.7.5" }

crates/services/january/src/api.rs

@@ -1,6 +1,7 @@
 use axum::{extract::Query, response::IntoResponse, routing::get, Json, Router};
 use reqwest::header;
-use revolt_result::Result;
+use revolt_models::v0::Embed;
+use revolt_result::{create_error, Result};
 use serde::{Deserialize, Serialize};
 use utoipa::ToSchema;
 
@@ -87,5 +88,9 @@ async fn embed(
     Query(UrlQuery { url }): Query<UrlQuery>,
     // TypedHeader(Authorization(_bearer)): TypedHeader<Authorization<Bearer>>,
 ) -> Result<impl IntoResponse> {
-    Request::generate_embed(url).await.map(Json)
+    match Request::generate_embed(url).await {
+        Ok(Embed::None) => Err(create_error!(NoEmbedData)),
+        result => result,
+    }
+    .map(Json)
 }

crates/services/january/src/requests.rs

@@ -66,7 +66,7 @@ impl Request {
             let Request { response, mime } = Request::new(url).await?;
 
             if matches!(mime.type_(), mime::IMAGE | mime::VIDEO) {
-                let bytes = response.bytes().await.map_err(|_| create_error!(LabelMe));
+                let bytes = report_internal_error!(response.bytes().await);
 
                 let result = match bytes {
                     Ok(bytes) => {
@@ -77,7 +77,7 @@ impl Request {
                                 if is_valid_image(reader, "image/gif") {
                                     Ok(("image/gif".to_owned(), bytes.to_vec()))
                                 } else {
-                                    Err(create_error!(LabelMe))
+                                    Err(create_error!(FileTypeNotAllowed))
                                 }
                             } else {
                                 Ok((
@@ -96,7 +96,7 @@ impl Request {
                             if video_size(&file).is_some() {
                                 Ok((mime.to_string(), bytes.to_vec()))
                             } else {
-                                Err(create_error!(LabelMe))
+                                Err(create_error!(FileTypeNotAllowed))
                             }
                         }
                     }
@@ -106,7 +106,7 @@ impl Request {
                 PROXY_CACHE.insert(url.to_owned(), result.clone()).await;
                 result
             } else {
-                Err(create_error!(LabelMe))
+                Err(create_error!(FileTypeNotAllowed))
             }
         }
     }
@@ -122,20 +122,21 @@ impl Request {
                 _ => Ok(None),
             }
         } else {
-            let response = if let Some(Request { response, .. }) = request {
-                response
+            let request = if let Some(request) = request {
+                request
             } else {
-                let Request { response, mime } = Request::new(url).await?;
-                if matches!(mime.type_(), mime::IMAGE) {
-                    response
+                let request = Request::new(url).await?;
+                if matches!(request.mime.type_(), mime::IMAGE) {
+                    request
                 } else {
-                    return Err(create_error!(LabelMe));
+                    return Err(create_error!(FileTypeNotAllowed));
                 }
             };
 
-            if let Some((width, height)) =
-                image_size_vec(&report_internal_error!(response.bytes().await)?)
-            {
+            if let Some((width, height)) = image_size_vec(
+                &report_internal_error!(request.response.bytes().await)?,
+                request.mime.as_ref(),
+            ) {
                 Ok(Some(Image {
                     url: url.to_owned(),
                     width,
@@ -166,7 +167,7 @@ impl Request {
                 if matches!(mime.type_(), mime::VIDEO) {
                     response
                 } else {
-                    return Err(create_error!(LabelMe));
+                    return Err(create_error!(FileTypeNotAllowed));
                 }
             };
 

crates/services/january/src/website_embed.rs

@@ -9,34 +9,38 @@ use scraper::{Html, Selector};
 
 /// Create website metadata from URL and document
 pub async fn create_website_embed(original_url: &str, document: &str) -> Option<WebsiteMetadata> {
-    let document = Html::parse_document(document);
+    let (mut meta, mut link) = {
+        let document = Html::parse_document(document);
 
-    // create selectors
-    let meta_selector = Selector::parse("meta").ok()?;
-    let link_selector = Selector::parse("link").ok()?;
+        // create selectors
+        let meta_selector = Selector::parse("meta").ok()?;
+        let link_selector = Selector::parse("link").ok()?;
 
-    // extract meta tags
-    let mut meta = HashMap::new();
-    for el in document.select(&meta_selector) {
-        let node = el.value();
+        // extract meta tags
+        let mut meta = HashMap::new();
+        for el in document.select(&meta_selector) {
+            let node = el.value();
 
-        if let (Some(property), Some(content)) = (
-            node.attr("property").or_else(|| node.attr("name")),
-            node.attr("content"),
-        ) {
-            meta.insert(property.to_string(), content.to_string());
+            if let (Some(property), Some(content)) = (
+                node.attr("property").or_else(|| node.attr("name")),
+                node.attr("content"),
+            ) {
+                meta.insert(property.to_string(), content.to_string());
+            }
         }
-    }
 
-    // extract rel links
-    let mut link = HashMap::new();
-    for el in document.select(&link_selector) {
-        let node = el.value();
+        // extract rel links
+        let mut link = HashMap::new();
+        for el in document.select(&link_selector) {
+            let node = el.value();
 
-        if let (Some(property), Some(content)) = (node.attr("rel"), node.attr("href")) {
-            link.insert(property.to_string(), content.to_string());
+            if let (Some(property), Some(content)) = (node.attr("rel"), node.attr("href")) {
+                link.insert(property.to_string(), content.to_string());
+            }
         }
-    }
+
+        (meta, link)
+    };
 
     // build metadata
     let mut metadata = WebsiteMetadata {
@@ -137,32 +141,41 @@ pub async fn create_website_embed(original_url: &str, document: &str) -> Option<
     };
 
     // populate extra metadata for popular websites
-    populate_special(original_url.to_owned(), &mut metadata);
-
-    // TODO: these do not work because compiler is tripping:
+    populate_special(original_url.to_owned(), &mut metadata).await;
 
     // fetch video size if missing
-    // if let Some(Video { width, height, url }) = &metadata.video {
-    //     if width == &0 || height == &0 {
-    //         metadata.video = match crate::requests::Request::fetch_video_metadata(url, None).await {
-    //             Ok(Some(video)) => Some(video),
-    //             _ => None,
-    //         }
-    //     }
-    // }
+    if metadata.special.is_none() {
+        if let Some(Video { width, height, url }) = &metadata.video {
+            if width == &0 || height == &0 {
+                metadata.video =
+                    match crate::requests::Request::fetch_video_metadata(url, None).await {
+                        Ok(Some(video)) => Some(video),
+                        _ => None,
+                    }
+            }
+        }
+    }
+
+    // remove image if video exists
+    if metadata.video.is_some() {
+        metadata.image.take();
+    }
 
     // fetch image size if missing
-    // if let Some(Image {
-    //     width, height, url, ..
-    // }) = &metadata.image
-    // {
-    //     if width == &0 || height == &0 {
-    //         metadata.image = match crate::requests::Request::fetch_image_metadata(url, None).await {
-    //             Ok(Some(image)) => Some(image),
-    //             _ => None,
-    //         }
-    //     }
-    // }
+    if metadata.special.is_none() {
+        if let Some(Image {
+            width, height, url, ..
+        }) = &metadata.image
+        {
+            if width == &0 || height == &0 {
+                metadata.image =
+                    match crate::requests::Request::fetch_image_metadata(url, None).await {
+                        Ok(Some(image)) => Some(image),
+                        _ => None,
+                    }
+            }
+        }
+    }
 
     // truncate data
     metadata.truncate();
@@ -175,7 +188,7 @@ pub async fn create_website_embed(original_url: &str, document: &str) -> Option<
     }
 }
 
-pub fn populate_special(original_url: String, metadata: &mut WebsiteMetadata) {
+pub async fn populate_special(original_url: String, metadata: &mut WebsiteMetadata) {
     lazy_static! {
         static ref RE_YOUTUBE: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www|m)\\.)?(?:(?:youtube\\.com|youtu.be))(?:/(?:[\\w\\-]+\\?v=|embed/|v/)?)([\\w\\-]+)(?:\\S+)?$").unwrap();
 
@@ -223,15 +236,14 @@ pub fn populate_special(original_url: String, metadata: &mut WebsiteMetadata) {
             metadata.site_name.take();
 
             // Verify the video exists
-            // TODO: breaks axum :(
-            // if !crate::requests::Request::exists(&format!(
-            //     "http://img.youtube.com/vi/{}/sddefault.jpg",
-            //     id
-            // ))
-            // .await
-            // {
-            //     return;
-            // }
+            if !crate::requests::Request::exists(&format!(
+                "http://img.youtube.com/vi/{}/sddefault.jpg",
+                id
+            ))
+            .await
+            {
+                return;
+            }
         }
 
         if let Some(timestamp_captures) = RE_TIMESTAMP.captures_iter(url).next() {

doc/src/SUMMARY.md

@@ -2,6 +2,7 @@
 
 - [Introduction](./hello.md)
 - [Project Structure]()
+- [Creating new API features](./new_features.md)
 - [Testing]()
   - [Writing a new database test]()
   - [Writing a new API test]()

doc/src/new_features.md

@@ -0,0 +1,15 @@
+# New API features
+
+New API features must be documented where appropriate, this document aims to cover everywhere you need to update for new features.
+
+Before writing new API features, generally a good idea to:
+
+- Consult with other developers in the [Revolt Developers space](https://rvlt.gg/API)
+- If it's a relatively big feature, also [write an RFC](https://github.com/revoltchat/rfcs)
+
+When your feature is ready to release, ensure to:
+
+- Update backend documentation (what you're reading now!) if applicable
+- Update the [developers documentation](https://github.com/revoltchat/wiki) if applicable
+- Update the Feature Matrix (or ask someone that can to do so)
+- Ensure it is properly listed in the backend release changelog

scripts/publish-debug-image.sh

@@ -24,6 +24,7 @@ docker build -t ghcr.io/revoltchat/base:latest -f Dockerfile.useCurrentArch .
 docker build -t ghcr.io/revoltchat/server:$TAG - < crates/delta/Dockerfile
 docker build -t ghcr.io/revoltchat/bonfire:$TAG - < crates/bonfire/Dockerfile
 docker build -t ghcr.io/revoltchat/autumn:$TAG - < crates/services/autumn/Dockerfile
+docker build -t ghcr.io/revoltchat/january:$TAG - < crates/services/january/Dockerfile
 
 if [ "$DEBUG" = "true" ]; then
   git restore Cargo.toml
@@ -32,3 +33,4 @@ fi
 docker push ghcr.io/revoltchat/server:$TAG
 docker push ghcr.io/revoltchat/bonfire:$TAG
 docker push ghcr.io/revoltchat/autumn:$TAG
+docker push ghcr.io/revoltchat/january:$TAG

scripts/start.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+cargo build \
+    --bin revolt-delta \
+    --bin revolt-bonfire \
+    --bin revolt-autumn \
+    --bin revolt-january
+
+trap 'pkill -f revolt-' SIGINT
+cargo run --bin revolt-delta &
+cargo run --bin revolt-bonfire &
+cargo run --bin revolt-autumn &
+cargo run --bin revolt-january