fix: bonfire needs uname

fixes revoltchat/self-hosted#104

.github/workflows/docker.yaml

@@ -61,7 +61,7 @@ jobs:
     if: github.event_name != 'pull_request'
     strategy:
       matrix:
-        project: [delta, bonfire, autumn, january]
+        project: [delta, bonfire, autumn, january, pushd]
     name: Build ${{ matrix.project }} image
     steps:
       # Configure build environment
@@ -106,6 +106,10 @@ jobs:
               "january": {
                 "path": "crates/services/january",
                 "tag": "${{ github.repository_owner }}/january"
+              },
+              "pushd": {
+                "path": "crates/daemons/pushd",
+                "tag": "${{ github.repository_owner }}/pushd"
               }
             }
           export_to: output

.github/workflows/rust.yaml

@@ -41,11 +41,7 @@ jobs:
 
       - name: Run services in background
         run: |
-          docker compose -f docker-compose.db.yml up -d
-
-      - name: Copy .env.example
-        run: |
-          cp .env.example .env
+          docker compose -f compose.yml up -d
 
       - name: Run cargo test
         env:

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
   "editor.formatOnSave": true,
   "rust-analyzer.checkOnSave.command": "clippy",
-  "nixEnvSelector.suggestion": false
-}
+  "nixEnvSelector.suggestion": false,
+  "nixEnvSelector.nixFile": "${workspaceFolder}/default.nix"
+}

Cargo.toml

@@ -1,16 +1,15 @@
 [workspace]
 resolver = "2"
+
 members = [
     "crates/delta",
     "crates/bonfire",
     "crates/core/*",
     "crates/services/*",
     "crates/bindings/*",
+    "crates/daemons/pushd",
 ]
 
 [patch.crates-io]
-# mobc-redis = { git = "https://github.com/insertish/mobc", rev = "8b880bb59f2ba80b4c7bc40c649c113d8857a186" }
 redis22 = { package = "redis", version = "0.22.3", git = "https://github.com/revoltchat/redis-rs", rev = "1a41faf356fd21aebba71cea7eb7eb2653e5f0ef" }
 redis23 = { package = "redis", version = "0.23.1", git = "https://github.com/revoltchat/redis-rs", rev = "f8ca28ab85da59d2ccde526b4d2fb390eff5a5f9" }
-# authifier = { package = "authifier", version = "1.0.8", path = "../authifier/crates/authifier" }
-# rocket_authifier = { package = "rocket_authifier", version = "1.0.8", path = "../authifier/crates/rocket_authifier" }

Dockerfile

@@ -29,6 +29,7 @@ COPY crates/core/presence/Cargo.toml ./crates/core/presence/
 COPY crates/core/result/Cargo.toml ./crates/core/result/
 COPY crates/services/autumn/Cargo.toml ./crates/services/autumn/
 COPY crates/services/january/Cargo.toml ./crates/services/january/
+COPY crates/daemons/pushd/Cargo.toml ./crates/daemons/pushd/
 RUN sh /tmp/build-image-layer.sh deps
 
 # Build all apps

README.md

@@ -26,6 +26,7 @@ The services and libraries that power the Revolt service.<br/>
 | `services/january` | [crates/services/january](crates/services/january) | Proxy server                             | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue)                                                                                                                                                                                                                                |
 | `services/autumn`  | [crates/services/autumn](crates/services/autumn)   | File server                              | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue)                                                                                                                                                                                                                                |
 | `bindings/node`    | [crates/bindings/node](crates/bindings/node)       | Node.js bindings for the Revolt software | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue)                                                                                                                                                                                                                                |
+| `daemons/pushd`    | [crates/daemons/pushd](crates/daemons/pushd)       | Push notification daemon server          | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue)                                                                                                                                                                                                                                |
 
 </div>
 <br/>
@@ -55,11 +56,12 @@ As a heads-up, the development environment uses the following ports:
 
 | Service                   |      Port      |
 | ------------------------- | :------------: |
-| MongoDB                   |     14017      |
-| Redis                     |     14079      |
+| MongoDB                   |     27017      |
+| Redis                     |      6379      |
 | MinIO                     |     14009      |
 | Maildev                   | 14025<br>14080 |
 | Revolt Web App            |     14701      |
+| RabbitMQ                  |  5672<br>15672 |
 | `crates/delta`            |     14702      |
 | `crates/bonfire`          |     14703      |
 | `crates/services/autumn`  |     14704      |
@@ -95,17 +97,23 @@ Then continue:
 # start other necessary services
 docker compose up -d
 
+# run everything together
+./scripts/start.sh
+# .. or individually
 # run the API server
 cargo run --bin revolt-delta
 # run the events server
 cargo run --bin revolt-bonfire
 # run the file server
 cargo run --bin revolt-autumn
-# run th proxy server
+# run the proxy server
 cargo run --bin revolt-january
+# run the push daemon (not usually needed in regular development)
+cargo run --bin revolt-pushd
 
 # hint:
 # mold -run <cargo build, cargo run, etc...>
+# mold -run ./scripts/start.sh
 ```
 
 You can start a web client by doing the following:

compose.yml

@@ -3,13 +3,13 @@ services:
   redis:
     image: eqalpha/keydb
     ports:
-      - "14079:6379"
+      - "6379:6379"
 
   # MongoDB
   database:
     image: mongo
     ports:
-      - "14017:27017"
+      - "27017:27017"
     volumes:
       - ./.data/db:/data/db
 
@@ -32,16 +32,26 @@ services:
     image: minio/mc
     depends_on:
       - minio
-    environment:
-      MINIO_ROOT_USER: minioautumn
-      MINIO_ROOT_PASSWORD: minioautumn
     entrypoint: >
-      /bin/sh -c "
-      /usr/bin/mc config host add minio http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD;
-      while ! /usr/bin/mc ready minio; do echo 'Waiting minio...' && sleep 1; done;
-      /usr/bin/mc mb minio/revolt-uploads;
-      exit 0;
-      "
+      /bin/sh -c "while ! /usr/bin/mc ready minio; do
+        /usr/bin/mc config host add minio http://minio:9000 minioautumn minioautumn;
+        echo 'Waiting minio...' && sleep 1;
+      done; /usr/bin/mc mb minio/revolt-uploads; exit 0;"
+
+  # Rabbit
+  rabbit:
+    image: rabbitmq:3-management
+    environment:
+      RABBITMQ_DEFAULT_USER: rabbituser
+      RABBITMQ_DEFAULT_PASS: rabbitpass
+    volumes:
+      - ./.data/rabbit:/var/lib/rabbitmq
+      #- ./rabbit_plugins:/opt/rabbitmq/plugins/
+      #- ./rabbit_enabled_plugins:/etc/rabbitmq/enabled_plugins
+      # uncomment this if you need to enable other plugins
+    ports:
+      - "5672:5672"
+      - "15672:15672" # management UI, for development
 
   # Mock SMTP server
   maildev:

crates/bindings/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-nodejs-bindings"
-version = "0.7.16"
+version = "0.8.0"
 description = "Node.js bindings for the Revolt software"
 authors = ["Paul Makles <me@insrt.uk>"]
 license = "MIT"
@@ -20,6 +20,6 @@ serde = { version = "1", features = ["derive"] }
 
 async-std = "1.12.0"
 
-revolt-config = { version = "0.7.16", path = "../../core/config" }
-revolt-result = { version = "0.7.16", path = "../../core/result" }
-revolt-database = { version = "0.7.16", path = "../../core/database" }
+revolt-config = { version = "0.8.0", path = "../../core/config" }
+revolt-result = { version = "0.8.0", path = "../../core/result" }
+revolt-database = { version = "0.8.0", path = "../../core/database" }

crates/bindings/node/src/lib.rs

@@ -7,25 +7,25 @@ use neon::prelude::*;
 use revolt_database::{Database, DatabaseInfo};
 
 fn js_init(mut cx: FunctionContext) -> JsResult<JsUndefined> {
-    static INIT: OnceLock<()> = OnceLock::new();
-    if INIT.get().is_none() {
-        INIT.get_or_init(|| {
-            async_std::task::block_on(async {
-                revolt_config::configure!(api);
+    // static INIT: OnceLock<()> = OnceLock::new();
+    // if INIT.get().is_none() {
+    //     INIT.get_or_init(|| {
+    //         async_std::task::block_on(async {
+    //             revolt_config::configure!(api);
 
-                match DatabaseInfo::Auto.connect().await {
-                    Ok(db) => {
-                        let authifier_db = db.clone().to_authifier().await.database;
-                        revolt_database::tasks::start_workers(db, authifier_db);
-                        Ok(())
-                    }
-                    Err(err) => Err(err),
-                }
-            })
-            .or_else(|err| cx.throw_error(err))
-            .unwrap();
-        });
-    }
+    //             match DatabaseInfo::Auto.connect().await {
+    //                 Ok(db) => {
+    //                     let authifier_db = db.clone().to_authifier().await.database;
+    //                     revolt_database::tasks::start_workers(db, authifier_db);
+    //                     Ok(())
+    //                 }
+    //                 Err(err) => Err(err),
+    //             }
+    //         })
+    //         .or_else(|err| cx.throw_error(err))
+    //         .unwrap();
+    //     });
+    // }
 
     Ok(cx.undefined())
 }

crates/bonfire/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-bonfire"
-version = "0.7.16"
+version = "0.8.0"
 license = "AGPL-3.0-or-later"
 edition = "2021"
 
@@ -36,12 +36,12 @@ async-std = { version = "1.8.0", features = [
 ] }
 
 # core
-authifier = { version = "1.0.8" }
+authifier = { version = "1.0.9" }
 revolt-result = { path = "../core/result" }
 revolt-models = { path = "../core/models" }
 revolt-config = { path = "../core/config" }
 revolt-database = { path = "../core/database" }
-revolt-permissions = { version = "0.7.16", path = "../core/permissions" }
+revolt-permissions = { version = "0.8.0", path = "../core/permissions" }
 revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] }
 
 # redis

crates/bonfire/Dockerfile

@@ -1,9 +1,11 @@
 # Build Stage
 FROM ghcr.io/revoltchat/base:latest AS builder
+FROM debian:12 AS debian
 
 # Bundle Stage
 FROM gcr.io/distroless/cc-debian12:nonroot
 COPY --from=builder /home/rust/src/target/release/revolt-bonfire ./
+COPY --from=debian /usr/bin/uname /usr/bin/uname
 
 EXPOSE 14703
 USER nonroot

crates/bonfire/src/main.rs

@@ -19,7 +19,10 @@ async fn main() {
     database::connect().await;
 
     // Clean up the current region information.
-    clear_region(None).await;
+    let no_clear_region = env::var("NO_CLEAR_PRESENCE").unwrap_or_else(|_| "0".into()) == "1";
+    if !no_clear_region {
+        clear_region(None).await;
+    }
 
     // Setup a TCP listener to accept WebSocket connections on.
     // By default, we bind to port 14703 on all interfaces.

crates/bonfire/src/websocket.rs

@@ -79,7 +79,9 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr)
     // Try to authenticate the user.
     let Some(token) = config.get_session_token().as_ref() else {
         write
-            .send(config.encode(&create_error!(InvalidSession)))
+            .send(config.encode(&EventV1::Error {
+                data: create_error!(InvalidSession),
+            }))
             .await
             .ok();
         return;
@@ -88,7 +90,10 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr)
     let (user, session_id) = match User::from_token(db, token, UserHint::Any).await {
         Ok(user) => user,
         Err(err) => {
-            write.send(config.encode(&err)).await.ok();
+            write
+                .send(config.encode(&EventV1::Error { data: err }))
+                .await
+                .ok();
             return;
         }
     };
@@ -301,25 +306,23 @@ async fn listener(
                     PayloadType::Json => message
                         .value
                         .as_str()
-                        .and_then(|s| serde_json::from_str::<EventV1>(s.as_ref()).ok()),
+                        .and_then(|s| report_internal_error!(serde_json::from_str::<EventV1>(s.as_ref())).ok()),
                     PayloadType::Msgpack => message
                         .value
                         .as_bytes()
-                        .and_then(|b| rmp_serde::from_slice::<EventV1>(b).ok()),
+                        .and_then(|b| report_internal_error!(rmp_serde::from_slice::<EventV1>(b)).ok()),
                     PayloadType::Bincode => message
                         .value
                         .as_bytes()
-                        .and_then(|b| bincode::deserialize::<EventV1>(b).ok()),
+                        .and_then(|b| report_internal_error!(bincode::deserialize::<EventV1>(b)).ok()),
                 };
 
                 let Some(mut event) = event else {
                     let err = format!(
-                        "Failed to deserialise an event for {}! Introspection: `{:?}`",
+                        "Failed to deserialise event for {}: `{:?}`",
                         message.channel,
                         message
                             .value
-                            .as_string()
-                            .map(|x| x.chars().take(32).collect::<String>())
                     );
 
                     error!("{}", err);

crates/core/config/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-config"
-version = "0.7.16"
+version = "0.8.0"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -15,7 +15,6 @@ default = ["test"]
 
 [dependencies]
 # Utility
-dotenv = "0.15.0"
 config = "0.13.3"
 cached = "0.44.0"
 once_cell = "1.18.0"
@@ -35,4 +34,4 @@ pretty_env_logger = "0.4.0"
 sentry = "0.31.5"
 
 # Core
-revolt-result = { version = "0.7.16", path = "../result", optional = true }
+revolt-result = { version = "0.8.0", path = "../result", optional = true }

crates/core/config/Revolt.test.toml

@@ -1,3 +1,9 @@
 [database]
 mongodb = "mongodb://localhost"
 redis = "redis://localhost/"
+
+[rabbit]
+host = "127.0.0.1"
+port = 5672
+username = "rabbituser"
+password = "rabbitpass"

crates/core/config/Revolt.toml

@@ -20,6 +20,12 @@ january = "http://local.revolt.chat/january"
 voso_legacy = ""
 voso_legacy_ws = ""
 
+[rabbit]
+host = "rabbit"
+port = 5672
+username = "rabbituser"
+password = "rabbitpass"
+
 [api]
 
 [api.registration]
@@ -38,34 +44,6 @@ from_address = "noreply@example.com"
 # port = 587
 # use_tls = true
 
-[api.vapid]
-# Generate your own keys:
-# 1. Run `openssl ecparam -name prime256v1 -genkey -noout -out vapid_private.pem`
-# 2. Find `private_key` using `base64 vapid_private.pem`
-# 3. Find `public_key` using `openssl ec -in vapid_private.pem -outform DER|tail -c 65|base64|tr '/+' '_-'|tr -d '\n'`
-private_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJSUWpyTWxLRnBiVWhsUHpUbERvcEliYk1yeVNrNXpKYzVYVzIxSjJDS3hvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFWnkrQkg2TGJQZ2hEa3pEempXOG0rUXVPM3pCajRXT1phdkR6ZU00c0pqbmFwd1psTFE0WAp1ZDh2TzVodU94QWhMQlU3WWRldVovWHlBdFpWZmNyQi9BPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo"
-public_key = "BGcvgR-i2z4IQ5Mw841vJvkLjt8wY-FjmWrw83jOLCY52qcGZS0OF7nfLzuYbjsQISwVO2HXrmf18gLWVX3Kwfw="
-
-[api.fcm]
-# Google Firebase Cloud Messaging Service Account Key
-# Obtained from the cloud messaging console
-key_type = ""
-project_id = ""
-private_key_id = ""
-private_key = ""
-client_email = ""
-client_id = ""
-auth_uri = ""
-token_uri = ""
-auth_provider_x509_cert_url = ""
-client_x509_cert_url = ""
-
-[api.apn]
-# Apple Push Notifications keys for sending notifications
-sandbox = false
-pkcs8 = ""
-key_id = ""
-team_id = ""
 
 [api.security]
 # Authifier Shield API key
@@ -84,6 +62,46 @@ hcaptcha_sitekey = ""
 # Maximum concurrent connections (to proxy server)
 max_concurrent_connections = 50
 
+[pushd]
+# this changes the names of the queues to not overlap 
+# prod/beta if they happen to be on the same exchange/instance.
+# Usually they have to be, so that messages sent from one or the other get sent to everyone
+production = true
+
+# none of these should need changing
+exchange = "revolt.notifications"
+message_queue = "notifications.origin.message"
+fr_accepted_queue = "notifications.ingest.fr_accepted" # friend request accepted
+fr_received_queue = "notifications.ingest.fr_received" # friend request received
+generic_queue = "notifications.ingest.generic"         # generic messages (title + body)
+ack_queue = "notifications.process.ack"                # updates badges for apple devices
+
+[pushd.vapid]
+queue = "notifications.outbound.vapid"
+private_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJSUWpyTWxLRnBiVWhsUHpUbERvcEliYk1yeVNrNXpKYzVYVzIxSjJDS3hvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFWnkrQkg2TGJQZ2hEa3pEempXOG0rUXVPM3pCajRXT1phdkR6ZU00c0pqbmFwd1psTFE0WAp1ZDh2TzVodU94QWhMQlU3WWRldVovWHlBdFpWZmNyQi9BPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo"
+public_key = "BGcvgR-i2z4IQ5Mw841vJvkLjt8wY-FjmWrw83jOLCY52qcGZS0OF7nfLzuYbjsQISwVO2HXrmf18gLWVX3Kwfw="
+
+[pushd.fcm]
+queue = "notifications.outbound.fcm"
+key_type = ""
+project_id = ""
+private_key_id = ""
+private_key = ""
+client_email = ""
+client_id = ""
+auth_uri = ""
+token_uri = ""
+auth_provider_x509_cert_url = ""
+client_x509_cert_url = ""
+
+[pushd.apn]
+sandbox = false
+queue = "notifications.outbound.apn"
+pkcs8 = ""
+key_id = ""
+team_id = ""
+
+
 [files]
 # Encryption key for stored files
 # Generate your own key using `openssl rand -base64 32`
@@ -132,6 +150,7 @@ emojis = [128, 128]
 #
 # Backblaze B2:
 # - endpoint is listed on the "Buckets" page
+# - path_style_buckets is set to true
 # - region is `eu-central-003` string from endpoint URL
 # - access_key_id is keyID generated on the "Application Keys" page
 # - secret_access_key is token generated on the "Application Keys" page
@@ -139,16 +158,20 @@ emojis = [128, 128]
 
 # S3 protocol endpoint
 endpoint = "http://minio:9000"
+# Whether to use path-style buckets
+# Generally true, except for MinIO
+path_style_buckets = false
 # S3 region name
 region = "minio"
 # S3 protocol key ID
 access_key_id = "minioautumn"
 # S3 protocol access key
 secret_access_key = "minioautumn"
-# Bucket to upload to by default
 default_bucket = "revolt-uploads"
 
+
 [features]
+# Bucket to upload to by default
 # Feature gate options
 webhooks_enabled = false
 
@@ -224,6 +247,11 @@ icons = 2_500_000
 banners = 6_000_000
 emojis = 500_000
 
+[features.advanced]
+# The max amount of messages the rabbitmq provider/db mention adder job will delay for before forcing handling of a channel.
+# default: 5
+process_message_delay_limit = 5
+
 [sentry]
 # Configuration for Sentry error reporting
 api = ""

crates/core/config/src/lib.rs

@@ -79,6 +79,14 @@ pub struct Database {
     pub redis: String,
 }
 
+#[derive(Deserialize, Debug, Clone)]
+pub struct Rabbit {
+    pub host: String,
+    pub port: u16,
+    pub username: String,
+    pub password: String,
+}
+
 #[derive(Deserialize, Debug, Clone)]
 pub struct Hosts {
     pub app: String,
@@ -107,13 +115,15 @@ pub struct ApiSmtp {
 }
 
 #[derive(Deserialize, Debug, Clone)]
-pub struct ApiVapid {
+pub struct PushVapid {
+    pub queue: String,
     pub private_key: String,
     pub public_key: String,
 }
 
 #[derive(Deserialize, Debug, Clone)]
-pub struct ApiFcm {
+pub struct PushFcm {
+    pub queue: String,
     pub key_type: String,
     pub project_id: String,
     pub private_key_id: String,
@@ -127,7 +137,8 @@ pub struct ApiFcm {
 }
 
 #[derive(Deserialize, Debug, Clone)]
-pub struct ApiApn {
+pub struct PushApn {
+    pub queue: String,
     pub sandbox: bool,
     pub pkcs8: String,
     pub key_id: String,
@@ -157,13 +168,54 @@ pub struct ApiWorkers {
 pub struct Api {
     pub registration: ApiRegistration,
     pub smtp: ApiSmtp,
-    pub vapid: ApiVapid,
-    pub fcm: ApiFcm,
-    pub apn: ApiApn,
     pub security: ApiSecurity,
     pub workers: ApiWorkers,
 }
 
+#[derive(Deserialize, Debug, Clone)]
+pub struct Pushd {
+    pub production: bool,
+    pub exchange: String,
+    pub message_queue: String,
+    pub fr_accepted_queue: String,
+    pub fr_received_queue: String,
+    pub generic_queue: String,
+    pub ack_queue: String,
+
+    pub vapid: PushVapid,
+    pub fcm: PushFcm,
+    pub apn: PushApn,
+}
+
+impl Pushd {
+    fn get_routing_key(&self, key: String) -> String {
+        match self.production {
+            true => key + "-prd",
+            false => key + "-tst",
+        }
+    }
+
+    pub fn get_ack_routing_key(&self) -> String {
+        self.get_routing_key(self.ack_queue.clone())
+    }
+
+    pub fn get_message_routing_key(&self) -> String {
+        self.get_routing_key(self.message_queue.clone())
+    }
+
+    pub fn get_fr_accepted_routing_key(&self) -> String {
+        self.get_routing_key(self.fr_accepted_queue.clone())
+    }
+
+    pub fn get_fr_received_routing_key(&self) -> String {
+        self.get_routing_key(self.fr_received_queue.clone())
+    }
+
+    pub fn get_generic_routing_key(&self) -> String {
+        self.get_routing_key(self.generic_queue.clone())
+    }
+}
+
 #[derive(Deserialize, Debug, Clone)]
 pub struct FilesLimit {
     pub min_file_size: usize,
@@ -175,6 +227,7 @@ pub struct FilesLimit {
 #[derive(Deserialize, Debug, Clone)]
 pub struct FilesS3 {
     pub endpoint: String,
+    pub path_style_buckets: bool,
     pub region: String,
     pub access_key_id: String,
     pub secret_access_key: String,
@@ -232,10 +285,26 @@ pub struct FeaturesLimitsCollection {
     pub roles: HashMap<String, FeaturesLimits>,
 }
 
+#[derive(Deserialize, Debug, Clone)]
+pub struct FeaturesAdvanced {
+    #[serde(default)]
+    pub process_message_delay_limit: u16,
+}
+
+impl Default for FeaturesAdvanced {
+    fn default() -> Self {
+        Self {
+            process_message_delay_limit: 5,
+        }
+    }
+}
+
 #[derive(Deserialize, Debug, Clone)]
 pub struct Features {
     pub limits: FeaturesLimitsCollection,
     pub webhooks_enabled: bool,
+    #[serde(default)]
+    pub advanced: FeaturesAdvanced,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -249,8 +318,10 @@ pub struct Sentry {
 #[derive(Deserialize, Debug, Clone)]
 pub struct Settings {
     pub database: Database,
+    pub rabbit: Rabbit,
     pub hosts: Hosts,
     pub api: Api,
+    pub pushd: Pushd,
     pub files: Files,
     pub features: Features,
     pub sentry: Sentry,
@@ -286,8 +357,6 @@ pub async fn config() -> Settings {
 
 /// Configure logging and common Rust variables
 pub async fn setup_logging(release: &'static str, dsn: String) -> Option<sentry::ClientInitGuard> {
-    dotenv::dotenv().ok();
-
     if std::env::var("RUST_LOG").is_err() {
         std::env::set_var("RUST_LOG", "info");
     }

crates/core/database/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-database"
-version = "0.7.16"
+version = "0.8.0"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -24,15 +24,15 @@ default = ["mongodb", "async-std-runtime", "tasks"]
 
 [dependencies]
 # Core
-revolt-config = { version = "0.7.16", path = "../config", features = [
+revolt-config = { version = "0.8.0", path = "../config", features = [
     "report-macros",
 ] }
-revolt-result = { version = "0.7.16", path = "../result" }
-revolt-models = { version = "0.7.16", path = "../models", features = [
+revolt-result = { version = "0.8.0", path = "../result" }
+revolt-models = { version = "0.8.0", path = "../models", features = [
     "validator",
 ] }
-revolt-presence = { version = "0.7.16", path = "../presence" }
-revolt-permissions = { version = "0.7.16", path = "../permissions", features = [
+revolt-presence = { version = "0.8.0", path = "../presence" }
+revolt-permissions = { version = "0.8.0", path = "../permissions", features = [
     "serde",
     "bson",
 ] }
@@ -84,11 +84,11 @@ axum = { version = "0.7.5", optional = true }
 
 # Rocket Impl
 schemars = { version = "0.8.8", optional = true }
-rocket = { version = "0.5.0-rc.2", default-features = false, features = [
+rocket = { version = "0.5.1", default-features = false, features = [
     "json",
 ], optional = true }
 revolt_okapi = { version = "0.9.1", optional = true }
-revolt_rocket_okapi = { version = "0.9.1", optional = true }
+revolt_rocket_okapi = { version = "0.10.0", optional = true }
 
 # Notifications
 fcm_v1 = "0.3.0"
@@ -96,4 +96,7 @@ web-push = "0.10.0"
 revolt_a2 = { version = "0.10", default-features = false, features = ["ring"] }
 
 # Authifier
-authifier = { version = "1.0.8" }
+authifier = { version = "1.0.9", features = ["rocket_impl"] }
+
+# RabbitMQ
+amqprs = { version = "1.7.0" }

crates/core/database/src/amqp/amqp.rs

@@ -0,0 +1,211 @@
+use std::collections::HashSet;
+
+use crate::events::rabbit::*;
+use crate::User;
+use amqprs::channel::BasicPublishArguments;
+use amqprs::{channel::Channel, connection::Connection, error::Error as AMQPError};
+use amqprs::{BasicProperties, FieldTable};
+use revolt_models::v0::PushNotification;
+use revolt_presence::filter_online;
+
+use serde_json::to_string;
+
+#[derive(Clone)]
+pub struct AMQP {
+    #[allow(unused)]
+    connection: Connection,
+    channel: Channel,
+}
+
+impl AMQP {
+    pub fn new(connection: Connection, channel: Channel) -> AMQP {
+        AMQP {
+            connection,
+            channel,
+        }
+    }
+
+    pub async fn friend_request_accepted(
+        &self,
+        accepted_request_user: &User,
+        sent_request_user: &User,
+    ) -> Result<(), AMQPError> {
+        let config = revolt_config::config().await;
+        let payload = FRAcceptedPayload {
+            accepted_user: accepted_request_user.to_owned(),
+            user: sent_request_user.id.clone(),
+        };
+        let payload = to_string(&payload).unwrap();
+
+        debug!(
+            "Sending friend request accept payload on channel {}: {}",
+            config.pushd.get_fr_accepted_routing_key(),
+            payload
+        );
+        self.channel
+            .basic_publish(
+                BasicProperties::default()
+                    .with_content_type("application/json")
+                    .with_persistence(true)
+                    .finish(),
+                payload.into(),
+                BasicPublishArguments::new(
+                    &config.pushd.exchange,
+                    &config.pushd.get_fr_accepted_routing_key(),
+                ),
+            )
+            .await
+    }
+
+    pub async fn friend_request_received(
+        &self,
+        received_request_user: &User,
+        sent_request_user: &User,
+    ) -> Result<(), AMQPError> {
+        let config = revolt_config::config().await;
+        let payload = FRReceivedPayload {
+            from_user: sent_request_user.to_owned(),
+            user: received_request_user.id.clone(),
+        };
+        let payload = to_string(&payload).unwrap();
+
+        debug!(
+            "Sending friend request received payload on channel {}: {}",
+            config.pushd.get_fr_received_routing_key(),
+            payload
+        );
+
+        self.channel
+            .basic_publish(
+                BasicProperties::default()
+                    .with_content_type("application/json")
+                    .with_persistence(true)
+                    .finish(),
+                payload.into(),
+                BasicPublishArguments::new(
+                    &config.pushd.exchange,
+                    &config.pushd.get_fr_received_routing_key(),
+                ),
+            )
+            .await
+    }
+
+    pub async fn generic_message(
+        &self,
+        user: &User,
+        title: String,
+        body: String,
+        icon: Option<String>,
+    ) -> Result<(), AMQPError> {
+        let config = revolt_config::config().await;
+        let payload = GenericPayload {
+            title,
+            body,
+            icon,
+            user: user.to_owned(),
+        };
+        let payload = to_string(&payload).unwrap();
+
+        debug!(
+            "Sending generic payload on channel {}: {}",
+            config.pushd.get_generic_routing_key(),
+            payload
+        );
+
+        self.channel
+            .basic_publish(
+                BasicProperties::default()
+                    .with_content_type("application/json")
+                    .with_persistence(true)
+                    .finish(),
+                payload.into(),
+                BasicPublishArguments::new(
+                    &config.pushd.exchange,
+                    &config.pushd.get_generic_routing_key(),
+                ),
+            )
+            .await
+    }
+
+    pub async fn message_sent(
+        &self,
+        recipients: Vec<String>,
+        payload: PushNotification,
+    ) -> Result<(), AMQPError> {
+        if recipients.is_empty() {
+            return Ok(());
+        }
+
+        let config = revolt_config::config().await;
+
+        let online_ids = filter_online(&recipients).await;
+        let recipients = (&recipients.into_iter().collect::<HashSet<String>>() - &online_ids)
+            .into_iter()
+            .collect::<Vec<String>>();
+
+        let payload = MessageSentPayload {
+            notification: payload,
+            users: recipients,
+        };
+        let payload = to_string(&payload).unwrap();
+
+        debug!(
+            "Sending message payload on channel {}: {}",
+            config.pushd.get_message_routing_key(),
+            payload
+        );
+
+        self.channel
+            .basic_publish(
+                BasicProperties::default()
+                    .with_content_type("application/json")
+                    .with_persistence(true)
+                    .finish(),
+                payload.into(),
+                BasicPublishArguments::new(
+                    &config.pushd.exchange,
+                    &config.pushd.get_message_routing_key(),
+                ),
+            )
+            .await
+    }
+
+    pub async fn ack_message(
+        &self,
+        user_id: String,
+        channel_id: String,
+        message_id: String,
+    ) -> Result<(), AMQPError> {
+        let config = revolt_config::config().await;
+
+        let payload = AckPayload {
+            user_id: user_id.clone(),
+            channel_id: channel_id.clone(),
+            message_id,
+        };
+        let payload = to_string(&payload).unwrap();
+
+        info!(
+            "Sending ack payload on channel {}: {}",
+            config.pushd.ack_queue, payload
+        );
+
+        let mut headers = FieldTable::new();
+        headers.insert(
+            "x-deduplication-header".try_into().unwrap(),
+            format!("{}-{}", &user_id, &channel_id).into(),
+        );
+
+        self.channel
+            .basic_publish(
+                BasicProperties::default()
+                    .with_content_type("application/json")
+                    .with_persistence(true)
+                    //.with_headers(headers)
+                    .finish(),
+                payload.into(),
+                BasicPublishArguments::new(&config.pushd.exchange, &config.pushd.ack_queue),
+            )
+            .await
+    }
+}

crates/core/database/src/amqp/mod.rs

@@ -0,0 +1,2 @@
+#[allow(clippy::module_inception)]
+pub mod amqp;

crates/core/database/src/events/client.rs

@@ -1,25 +1,16 @@
 use authifier::AuthifierEvent;
+use revolt_result::Error;
 use serde::{Deserialize, Serialize};
 
 use revolt_models::v0::{
-    AppendMessage, Channel, ChannelUnread, Emoji, FieldsChannel, FieldsMember, FieldsMessage, FieldsRole, FieldsServer, FieldsUser, FieldsWebhook, Member, MemberCompositeKey, Message, PartialChannel, PartialMember, PartialMessage, PartialRole, PartialServer, PartialUser, PartialWebhook, RemovalIntention, Report, Server, User, UserSettings, Webhook
+    AppendMessage, Channel, ChannelUnread, Emoji, FieldsChannel, FieldsMember, FieldsMessage,
+    FieldsRole, FieldsServer, FieldsUser, FieldsWebhook, Member, MemberCompositeKey, Message,
+    PartialChannel, PartialMember, PartialMessage, PartialRole, PartialServer, PartialUser,
+    PartialWebhook, RemovalIntention, Report, Server, User, UserSettings, Webhook,
 };
-use revolt_result::Error;
 
 use crate::Database;
 
-/// WebSocket Client Errors
-#[derive(Serialize, Deserialize, Debug, Clone)]
-#[serde(tag = "error")]
-pub enum WebSocketError {
-    LabelMe,
-    InternalError { at: String },
-    InvalidSession,
-    OnboardingNotFinished,
-    AlreadyAuthenticated,
-    MalformedData { msg: String },
-}
-
 /// Ping Packet
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(untagged)]
@@ -28,14 +19,6 @@ pub enum Ping {
     Number(usize),
 }
 
-/// Untagged Error
-#[derive(Serialize)]
-#[serde(untagged)]
-pub enum ErrorEvent {
-    Error(WebSocketError),
-    APIError(Error),
-}
-
 /// Fields provided in Ready payload
 #[derive(PartialEq)]
 pub enum ReadyPayloadFields {
@@ -55,6 +38,8 @@ pub enum ReadyPayloadFields {
 pub enum EventV1 {
     /// Multiple events
     Bulk { v: Vec<EventV1> },
+    /// Error event
+    Error { data: Error },
 
     /// Successfully authenticated
     Authenticated,
@@ -89,6 +74,7 @@ pub enum EventV1 {
         id: String,
         channel: String,
         data: PartialMessage,
+        #[serde(default)]
         clear: Vec<FieldsMessage>,
     },
 
@@ -140,6 +126,7 @@ pub enum EventV1 {
     ServerUpdate {
         id: String,
         data: PartialServer,
+        #[serde(default)]
         clear: Vec<FieldsServer>,
     },
 
@@ -150,6 +137,7 @@ pub enum EventV1 {
     ServerMemberUpdate {
         id: MemberCompositeKey,
         data: PartialMember,
+        #[serde(default)]
         clear: Vec<FieldsMember>,
     },
 
@@ -168,6 +156,7 @@ pub enum EventV1 {
         id: String,
         role_id: String,
         data: PartialRole,
+        #[serde(default)]
         clear: Vec<FieldsRole>,
     },
 
@@ -178,6 +167,7 @@ pub enum EventV1 {
     UserUpdate {
         id: String,
         data: PartialUser,
+        #[serde(default)]
         clear: Vec<FieldsUser>,
         event_id: Option<String>,
     },
@@ -212,6 +202,7 @@ pub enum EventV1 {
     ChannelUpdate {
         id: String,
         data: PartialChannel,
+        #[serde(default)]
         clear: Vec<FieldsChannel>,
     },
 

crates/core/database/src/events/mod.rs

@@ -1,2 +1,3 @@
 pub mod client;
+pub mod rabbit;
 pub mod server;

crates/core/database/src/events/rabbit.rs

@@ -0,0 +1,59 @@
+use std::collections::HashMap;
+
+use revolt_models::v0::PushNotification;
+use serde::{Deserialize, Serialize};
+
+use crate::User;
+
+#[derive(Serialize, Deserialize)]
+pub struct MessageSentPayload {
+    pub notification: PushNotification,
+    pub users: Vec<String>,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct FRAcceptedPayload {
+    pub accepted_user: User,
+    pub user: String,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct FRReceivedPayload {
+    pub from_user: User,
+    pub user: String,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct GenericPayload {
+    pub title: String,
+    pub body: String,
+    pub icon: Option<String>,
+    pub user: User,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "type", content = "data")]
+#[allow(clippy::large_enum_variant)]
+pub enum PayloadKind {
+    MessageNotification(PushNotification),
+    FRAccepted(FRAcceptedPayload),
+    FRReceived(FRReceivedPayload),
+    BadgeUpdate(usize),
+    Generic(GenericPayload),
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct PayloadToService {
+    pub notification: PayloadKind,
+    pub user_id: String,
+    pub session_id: String,
+    pub token: String,
+    pub extras: HashMap<String, String>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct AckPayload {
+    pub user_id: String,
+    pub channel_id: String,
+    pub message_id: String,
+}

crates/core/database/src/lib.rs

@@ -85,6 +85,9 @@ pub use models::*;
 pub mod events;
 pub mod tasks;
 
+mod amqp;
+pub use amqp::amqp::AMQP;
+
 /// Utility function to check if a boolean value is false
 pub fn if_false(t: &bool) -> bool {
     !t

crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs

@@ -11,6 +11,7 @@ use bson::oid::ObjectId;
 use futures::StreamExt;
 use rand::seq::SliceRandom;
 use revolt_permissions::DEFAULT_WEBHOOK_PERMISSIONS;
+use revolt_result::{Error, ErrorType};
 use serde::{Deserialize, Serialize};
 use unicode_segmentation::UnicodeSegmentation;
 
@@ -20,7 +21,7 @@ struct MigrationInfo {
     revision: i32,
 }
 
-pub const LATEST_REVISION: i32 = 30;
+pub const LATEST_REVISION: i32 = 31;
 
 pub async fn migrate_database(db: &MongoDb) {
     let migrations = db.col::<Document>("migrations");
@@ -1139,53 +1140,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
             .expect("Failed to create attachment_hashes index.");
     }
 
-    if revision <= 29 {
-        info!("Running migration [revision 29 / 29-09-2024]: Add creator_id to webhooks.");
-
-        #[derive(serde::Serialize, serde::Deserialize)]
-        struct WebhookShell {
-            _id: String,
-            channel_id: String,
-        }
-
-        let invites = db
-            .db()
-            .collection::<WebhookShell>("channel_webhooks")
-            .find(doc! {}, None)
-            .await
-            .expect("webhooks")
-            .filter_map(|s| async { s.ok() })
-            .collect::<Vec<WebhookShell>>()
-            .await;
-
-        for invite in invites {
-            let channel = db.fetch_channel(&invite.channel_id).await.expect("channel");
-            let creator_id = match channel {
-                Channel::Group { owner, .. } => owner,
-                Channel::TextChannel { server, .. } | Channel::VoiceChannel { server, .. } => {
-                    let server = db.fetch_server(&server).await.expect("server");
-                    server.owner
-                }
-                _ => unreachable!("not server or group channel!"),
-            };
-
-            db.db()
-                .collection::<Document>("channel_webhooks")
-                .update_one(
-                    doc! {
-                        "_id": invite._id,
-                    },
-                    doc! {
-                        "$set" : {
-                            "creator_id": creator_id
-                        }
-                    },
-                    None,
-                )
-                .await
-                .expect("update webhook");
-        }
-    }
+    // Revision 29 omitted due to bug.
 
     if revision <= 30 {
         info!("Running migration [revision 30 / 29-09-2024]: Add index for used_for.id to attachments.");
@@ -1209,7 +1164,68 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
             .expect("Failed to create attachments index.");
     }
 
-    // Need to migrate fields on attachments, change `user_id`, `object_id`, etc to `parent`.
+    if revision <= 31 {
+        info!("Running migration [revision 31 / 31-10-2024]: Add creator_id to webhooks and delete those whose channels don't exist.");
+
+        #[derive(serde::Serialize, serde::Deserialize)]
+        struct WebhookShell {
+            _id: String,
+            channel_id: String,
+        }
+
+        let webhooks = db
+            .db()
+            .collection::<WebhookShell>("channel_webhooks")
+            .find(doc! {}, None)
+            .await
+            .expect("webhooks")
+            .filter_map(|s| async { s.ok() })
+            .collect::<Vec<WebhookShell>>()
+            .await;
+
+        for webhook in webhooks {
+            match db.fetch_channel(&webhook.channel_id).await {
+                Ok(channel) => {
+                    let creator_id = match channel {
+                        Channel::Group { owner, .. } => owner,
+                        Channel::TextChannel { server, .. }
+                        | Channel::VoiceChannel { server, .. } => {
+                            let server = db.fetch_server(&server).await.expect("server");
+                            server.owner
+                        }
+                        _ => unreachable!("not server or group channel!"),
+                    };
+
+                    db.db()
+                        .collection::<Document>("channel_webhooks")
+                        .update_one(
+                            doc! {
+                                "_id": webhook._id,
+                            },
+                            doc! {
+                                "$set" : {
+                                    "creator_id": creator_id
+                                }
+                            },
+                            None,
+                        )
+                        .await
+                        .expect("update webhook");
+                }
+                Err(Error {
+                    error_type: ErrorType::NotFound,
+                    ..
+                }) => {
+                    db.db()
+                        .collection::<WebhookShell>("channel_webhooks")
+                        .delete_one(doc! { "_id": webhook._id }, None)
+                        .await
+                        .expect("failed to delete invalid webhook");
+                }
+                Err(err) => panic!("{err:?}"),
+            }
+        }
+    }
 
     // Reminder to update LATEST_REVISION when adding new migrations.
     LATEST_REVISION.max(revision)

crates/core/database/src/models/channel_unreads/ops.rs

@@ -26,6 +26,9 @@ pub trait AbstractChannelUnreads: Sync + Send {
         message_ids: &[String],
     ) -> Result<()>;
 
+    /// Fetch all unreads with mentions for a user.
+    async fn fetch_unread_mentions(&self, user_id: &str) -> Result<Vec<ChannelUnread>>;
+
     /// Fetch all channel unreads for a user.
     async fn fetch_unreads(&self, user_id: &str) -> Result<Vec<ChannelUnread>>;
 

crates/core/database/src/models/channel_unreads/ops/mongodb.rs

@@ -123,6 +123,18 @@ impl AbstractChannelUnreads for MongoDb {
         )
     }
 
+    async fn fetch_unread_mentions(&self, user_id: &str) -> Result<Vec<ChannelUnread>> {
+        query! {
+            self,
+            find,
+            COL,
+            doc! {
+                "_id.user": user_id,
+                "mentions": {"$ne": null}
+            }
+        }
+    }
+
     /// Fetch unread for a specific user in a channel.
     async fn fetch_unread(&self, user_id: &str, channel_id: &str) -> Result<Option<ChannelUnread>> {
         query!(
@@ -135,5 +147,4 @@ impl AbstractChannelUnreads for MongoDb {
             }
         )
     }
-
 }

crates/core/database/src/models/channel_unreads/ops/reference.rs

@@ -78,6 +78,15 @@ impl AbstractChannelUnreads for ReferenceDb {
         Ok(())
     }
 
+    async fn fetch_unread_mentions(&self, user_id: &str) -> Result<Vec<ChannelUnread>> {
+        let unreads = self.channel_unreads.lock().await;
+        Ok(unreads
+            .values()
+            .filter(|unread| unread.id.user == user_id && unread.mentions.is_some())
+            .cloned()
+            .collect())
+    }
+
     /// Fetch all channel unreads for a user.
     async fn fetch_unreads(&self, user_id: &str) -> Result<Vec<ChannelUnread>> {
         let unreads = self.channel_unreads.lock().await;
@@ -92,9 +101,11 @@ impl AbstractChannelUnreads for ReferenceDb {
     async fn fetch_unread(&self, user_id: &str, channel_id: &str) -> Result<Option<ChannelUnread>> {
         let unreads = self.channel_unreads.lock().await;
 
-        Ok(unreads.get(&ChannelCompositeKey {
-            channel: channel_id.to_string(),
-            user: user_id.to_string()
-        }).cloned())
+        Ok(unreads
+            .get(&ChannelCompositeKey {
+                channel: channel_id.to_string(),
+                user: user_id.to_string(),
+            })
+            .cloned())
     }
 }

crates/core/database/src/models/channels/model.rs

@@ -9,7 +9,7 @@ use ulid::Ulid;
 
 use crate::{
     events::client::EventV1, tasks::ack::AckEvent, Database, File, IntoDocumentPath, PartialServer,
-    Server, SystemMessage, User,
+    Server, SystemMessage, User, AMQP,
 };
 
 auto_derived!(
@@ -337,6 +337,7 @@ impl Channel {
     pub async fn add_user_to_group(
         &mut self,
         db: &Database,
+        amqp: &AMQP,
         user: &User,
         by_id: &str,
     ) -> Result<()> {
@@ -373,6 +374,7 @@ impl Channel {
                 .into_message(id.to_string())
                 .send(
                     db,
+                    Some(amqp),
                     MessageAuthor::System {
                         username: &user.username,
                         avatar: user.avatar.as_ref().map(|file| file.id.as_ref()),
@@ -639,7 +641,7 @@ impl Channel {
         .private(user.to_string())
         .await;
 
-        crate::tasks::ack::queue(
+        crate::tasks::ack::queue_ack(
             self.id().to_string(),
             user.to_string(),
             AckEvent::AckMessage {
@@ -655,6 +657,7 @@ impl Channel {
     pub async fn remove_user_from_group(
         &self,
         db: &Database,
+        amqp: &AMQP,
         user: &User,
         by_id: Option<&str>,
         silent: bool,
@@ -686,6 +689,7 @@ impl Channel {
                         .into_message(id.to_string())
                         .send(
                             db,
+                            Some(amqp),
                             MessageAuthor::System {
                                 username: name,
                                 avatar: None,
@@ -725,6 +729,7 @@ impl Channel {
                     .into_message(id.to_string())
                     .send(
                         db,
+                        Some(amqp),
                         MessageAuthor::System {
                             username: &user.username,
                             avatar: user.avatar.as_ref().map(|file| file.id.as_ref()),

crates/core/database/src/models/messages/model.rs

@@ -1,4 +1,4 @@
-use std::collections::HashSet;
+use std::{collections::HashSet, hash::RandomState};
 
 use indexmap::{IndexMap, IndexSet};
 use iso8601_timestamp::Timestamp;
@@ -15,8 +15,8 @@ use validator::Validate;
 use crate::{
     events::client::EventV1,
     tasks::{self, ack::AckEvent},
-    util::idempotency::IdempotencyKey,
-    Channel, Database, Emoji, File, User,
+    util::{bulk_permissions::BulkDatabasePermissionQuery, idempotency::IdempotencyKey},
+    Channel, Database, Emoji, File, User, AMQP,
 };
 
 auto_derived_partial!(
@@ -230,6 +230,7 @@ impl Message {
     #[allow(clippy::too_many_arguments)]
     pub async fn create_from_api(
         db: &Database,
+        amqp: Option<&AMQP>,
         channel: Channel,
         data: DataMessageSend,
         author: MessageAuthor<'_>,
@@ -337,6 +338,50 @@ impl Message {
             }
         }
 
+        // Validate the mentions go to users in the channel/server
+        if !mentions.is_empty() {
+            match channel {
+                Channel::DirectMessage { ref recipients, .. }
+                | Channel::Group { ref recipients, .. } => {
+                    let recipients_hash: HashSet<&String, RandomState> =
+                        HashSet::from_iter(recipients);
+                    mentions.retain(|m| recipients_hash.contains(m));
+                }
+                Channel::TextChannel { ref server, .. }
+                | Channel::VoiceChannel { ref server, .. } => {
+                    let mentions_vec = Vec::from_iter(mentions.iter().cloned());
+
+                    let valid_members = db.fetch_members(server.as_str(), &mentions_vec[..]).await;
+                    if let Ok(valid_members) = valid_members {
+                        let valid_mentions: HashSet<&String, RandomState> =
+                            HashSet::from_iter(valid_members.iter().map(|m| &m.id.user));
+
+                        mentions.retain(|m| valid_mentions.contains(m)); // quick pass, validate mentions are in the server
+
+                        if !mentions.is_empty() {
+                            // if there are still mentions, drill down to a channel-level
+                            let member_channel_view_perms =
+                                BulkDatabasePermissionQuery::from_server_id(db, server)
+                                    .await
+                                    .channel(&channel)
+                                    .members(&valid_members)
+                                    .members_can_see_channel()
+                                    .await;
+
+                            mentions
+                                .retain(|m| *member_channel_view_perms.get(m).unwrap_or(&false));
+                        }
+                    } else {
+                        revolt_config::capture_error(&valid_members.unwrap_err());
+                        return Err(create_error!(InternalError));
+                    }
+                }
+                Channel::SavedMessages { .. } => {
+                    mentions.clear();
+                }
+            }
+        }
+
         if !mentions.is_empty() {
             message.mentions.replace(mentions.into_iter().collect());
         }
@@ -391,7 +436,7 @@ impl Message {
 
         // Send the message
         message
-            .send(db, author, user, member, &channel, generate_embeds)
+            .send(db, amqp, author, user, member, &channel, generate_embeds)
             .await?;
 
         Ok(message)
@@ -405,6 +450,9 @@ impl Message {
         member: Option<v0::Member>,
         is_dm: bool,
         generate_embeds: bool,
+        // This determines if this function should queue the mentions task or if somewhere else will.
+        // If this is true, you MUST call tasks::ack::queue yourself.
+        mentions_elsewhere: bool,
     ) -> Result<()> {
         db.insert_message(self).await?;
 
@@ -417,13 +465,12 @@ impl Message {
         tasks::last_message_id::queue(self.channel.to_string(), self.id.to_string(), is_dm).await;
 
         // Add mentions for affected users
-        if let Some(mentions) = &self.mentions {
-            for user in mentions {
-                tasks::ack::queue(
+        if !mentions_elsewhere {
+            if let Some(mentions) = &self.mentions {
+                tasks::ack::queue_message(
                     self.channel.to_string(),
-                    user.to_string(),
-                    AckEvent::AddMention {
-                        ids: vec![self.id.to_string()],
+                    AckEvent::ProcessMessage {
+                        messages: vec![(None, self.clone(), mentions.clone(), true)],
                     },
                 )
                 .await;
@@ -446,9 +493,11 @@ impl Message {
     }
 
     /// Send a message
+    #[allow(clippy::too_many_arguments)]
     pub async fn send(
         &mut self,
         db: &Database,
+        amqp: Option<&AMQP>, // this is optional mostly for tests.
         author: MessageAuthor<'_>,
         user: Option<v0::User>,
         member: Option<v0::Member>,
@@ -461,26 +510,36 @@ impl Message {
             member.clone(),
             matches!(channel, Channel::DirectMessage { .. }),
             generate_embeds,
+            true,
         )
         .await?;
 
         if !self.has_suppressed_notifications() {
-            // Push out Web Push notifications
-            crate::tasks::web_push::queue(
-                {
-                    match channel {
-                        Channel::DirectMessage { recipients, .. }
-                        | Channel::Group { recipients, .. } => recipients.clone(),
-                        Channel::TextChannel { .. } => self.mentions.clone().unwrap_or_default(),
-                        _ => vec![],
-                    }
+            // send Push notifications
+            tasks::ack::queue_message(
+                self.channel.to_string(),
+                AckEvent::ProcessMessage {
+                    messages: vec![(
+                        Some(
+                            PushNotification::from(
+                                self.clone().into_model(user, member),
+                                Some(author),
+                                channel.to_owned().into(),
+                            )
+                            .await,
+                        ),
+                        self.clone(),
+                        match channel {
+                            Channel::DirectMessage { recipients, .. }
+                            | Channel::Group { recipients, .. } => recipients.clone(),
+                            Channel::TextChannel { .. } => {
+                                self.mentions.clone().unwrap_or_default()
+                            }
+                            _ => vec![],
+                        },
+                        self.has_suppressed_notifications(),
+                    )],
                 },
-                PushNotification::from(
-                    self.clone().into_model(user, member),
-                    Some(author),
-                    channel.id(),
-                )
-                .await,
             )
             .await;
         }

crates/core/database/src/models/server_members/model.rs

@@ -150,7 +150,7 @@ impl Member {
                 id: user.id.clone(),
             }
             .into_message(id.to_string())
-            .send_without_notifications(db, None, None, false, false)
+            .send_without_notifications(db, None, None, false, false, false)
             .await
             .ok();
         }
@@ -251,7 +251,7 @@ impl Member {
                 }
                 .into_message(id.to_string())
                 // TODO: support notifications here in the future?
-                .send_without_notifications(db, None, None, false, false)
+                .send_without_notifications(db, None, None, false, false, false)
                 .await
                 .ok();
             }

crates/core/database/src/models/server_members/ops/reference.rs

@@ -53,17 +53,17 @@ impl AbstractServerMembers for ReferenceDb {
     /// Fetch multiple members by their ids
     async fn fetch_members<'a>(&self, server_id: &str, ids: &'a [String]) -> Result<Vec<Member>> {
         let server_members = self.server_members.lock().await;
-        ids.iter()
-            .map(|id| {
+        Ok(ids
+            .iter()
+            .filter_map(|id| {
                 server_members
                     .get(&MemberCompositeKey {
                         server: server_id.to_string(),
                         user: id.to_string(),
                     })
                     .cloned()
-                    .ok_or_else(|| create_error!(NotFound))
             })
-            .collect()
+            .collect())
     }
 
     /// Fetch member count of a server

crates/core/database/src/models/users/model.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashSet, str::FromStr, time::Duration};
 
-use crate::{events::client::EventV1, Database, File, RatelimitEvent};
+use crate::{events::client::EventV1, Database, File, RatelimitEvent, AMQP};
 
 use authifier::config::{EmailVerificationConfig, Template};
 use iso8601_timestamp::Timestamp;
@@ -497,7 +497,12 @@ impl User {
     }
 
     /// Add another user as a friend
-    pub async fn add_friend(&mut self, db: &Database, target: &mut User) -> Result<()> {
+    pub async fn add_friend(
+        &mut self,
+        db: &Database,
+        amqp: &AMQP,
+        target: &mut User,
+    ) -> Result<()> {
         match self.relationship_with(&target.id) {
             RelationshipStatus::User => Err(create_error!(NoEffect)),
             RelationshipStatus::Friend => Err(create_error!(AlreadyFriends)),
@@ -506,6 +511,8 @@ impl User {
             RelationshipStatus::BlockedOther => Err(create_error!(BlockedByOther)),
             RelationshipStatus::Incoming => {
                 // Accept incoming friend request
+                _ = amqp.friend_request_accepted(self, target).await;
+
                 self.apply_relationship(
                     db,
                     target,
@@ -534,6 +541,8 @@ impl User {
                     }));
                 }
 
+                _ = amqp.friend_request_received(target, self).await;
+
                 // Send the friend request
                 self.apply_relationship(
                     db,

crates/core/database/src/models/users/rocket.rs

@@ -38,7 +38,7 @@ impl<'r> FromRequest<'r> for User {
         if let Some(user) = user {
             Outcome::Success(user.clone())
         } else {
-            Outcome::Failure((Status::Unauthorized, authifier::Error::InvalidSession))
+            Outcome::Error((Status::Unauthorized, authifier::Error::InvalidSession))
         }
     }
 }

crates/core/database/src/tasks/ack.rs

@@ -1,21 +1,27 @@
 // Queue Type: Debounced
-use crate::Database;
+use crate::{Database, Message, AMQP};
 
 use deadqueue::limited::Queue;
 use once_cell::sync::Lazy;
-use std::{collections::HashMap, time::Duration};
+use revolt_models::v0::PushNotification;
+use rocket::form::validate::Contains;
+use std::{
+    collections::{HashMap, HashSet},
+    time::Duration,
+};
+use validator::HasLen;
 
 use revolt_result::Result;
 
-use super::{apple_notifications::{self, ApnJob}, DelayedTask};
+use super::DelayedTask;
 
 /// Enumeration of possible events
 #[derive(Debug, Eq, PartialEq)]
 pub enum AckEvent {
-    /// Add mentions for a user in a channel
-    AddMention {
-        /// Message IDs
-        ids: Vec<String>,
+    /// Add mentions for a channel
+    ProcessMessage {
+        /// push notification, message, recipients, push silenced
+        messages: Vec<(Option<PushNotification>, Message, Vec<String>, bool)>,
     },
 
     /// Acknowledge message in a channel for a user
@@ -30,7 +36,7 @@ struct Data {
     /// Channel to ack
     channel: String,
     /// User to ack for
-    user: String,
+    user: Option<String>,
     /// Event
     event: AckEvent,
 }
@@ -43,21 +49,49 @@ struct Task {
 static Q: Lazy<Queue<Data>> = Lazy::new(|| Queue::new(10_000));
 
 /// Queue a new task for a worker
-pub async fn queue(channel: String, user: String, event: AckEvent) {
+pub async fn queue_ack(channel: String, user: String, event: AckEvent) {
     Q.try_push(Data {
         channel,
-        user,
+        user: Some(user),
         event,
     })
     .ok();
 
-    info!("Queue is using {} slots from {}.", Q.len(), Q.capacity());
+    info!(
+        "Queue is using {} slots from {}. Queued type: ACK",
+        Q.len(),
+        Q.capacity()
+    );
 }
 
-pub async fn handle_ack_event(event: &AckEvent, db: &Database, authifier_db: &authifier::Database, user: &str, channel: &str) -> Result<()> {
+pub async fn queue_message(channel: String, event: AckEvent) {
+    Q.try_push(Data {
+        channel,
+        user: None,
+        event,
+    })
+    .ok();
+
+    info!(
+        "Queue is using {} slots from {}. Queued type: MENTION",
+        Q.len(),
+        Q.capacity()
+    );
+}
+
+pub async fn handle_ack_event(
+    event: &AckEvent,
+    db: &Database,
+    amqp: &AMQP,
+    user: &Option<String>,
+    channel: &str,
+) -> Result<()> {
     match &event {
         #[allow(clippy::disallowed_methods)] // event is sent by higher level function
         AckEvent::AckMessage { id } => {
+            let user = user.as_ref().unwrap();
+            let user: &str = user.as_str();
+
             let unread = db.fetch_unread(user, channel).await?;
             let updated = db.acknowledge_message(channel, user, id).await?;
 
@@ -68,21 +102,67 @@ pub async fn handle_ack_event(event: &AckEvent, db: &Database, authifier_db: &au
                 let mentions_acked = before_mentions - after_mentions;
 
                 if mentions_acked > 0 {
-                    if let Ok(sessions) = authifier_db.find_sessions(user).await {
-                        for session in sessions {
-                            if let Some(sub) = session.subscription {
-                                if sub.endpoint == "apn" {
-                                    apple_notifications::queue(ApnJob::from_ack(session.id, user.to_string(), sub.auth)).await;
-                                }
-                            }
-                        }
+                    if let Err(err) = amqp
+                        .ack_message(user.to_string(), channel.to_string(), id.to_owned())
+                        .await
+                    {
+                        revolt_config::capture_error(&err);
                     }
                 };
-
             }
-        },
-        AckEvent::AddMention { ids } => {
-            db.add_mention_to_unread(channel, user, ids).await?;
+        }
+        AckEvent::ProcessMessage { messages } => {
+            let mut users: HashSet<&String> = HashSet::new();
+            debug!(
+                "Processing {} messages from channel {}",
+                messages.len(),
+                messages[0].1.channel
+            );
+
+            // find all the users we'll be notifying
+            messages
+                .iter()
+                .for_each(|(_, _, recipents, _)| users.extend(recipents.iter()));
+
+            debug!("Found {} users to notify.", users.len());
+
+            for user in users {
+                let message_ids: Vec<String> = messages
+                    .iter()
+                    .filter(|(_, _, recipients, _)| recipients.contains(user))
+                    .map(|(_, message, _, _)| message.id.clone())
+                    .collect();
+
+                if !message_ids.is_empty() {
+                    db.add_mention_to_unread(channel, user, &message_ids)
+                        .await?;
+                }
+                debug!("Added {} mentions for user {}", message_ids.len(), &user);
+            }
+
+            for (push, _, recipients, silenced) in messages {
+                if *silenced || recipients.is_empty() || push.is_none() {
+                    debug!(
+                        "Rejecting push: silenced: {}, recipient count: {}, push exists: {:?}",
+                        *silenced,
+                        recipients.length(),
+                        push
+                    );
+                    continue;
+                }
+
+                debug!(
+                    "Sending push event to AMQP; message {} for {} users",
+                    push.as_ref().unwrap().message.id,
+                    recipients.len()
+                );
+                if let Err(err) = amqp
+                    .message_sent(recipients.clone(), push.clone().unwrap())
+                    .await
+                {
+                    revolt_config::capture_error(&err);
+                }
+            }
         }
     };
 
@@ -90,9 +170,9 @@ pub async fn handle_ack_event(event: &AckEvent, db: &Database, authifier_db: &au
 }
 
 /// Start a new worker
-pub async fn worker(db: Database, authifier_db: authifier::Database) {
-    let mut tasks = HashMap::<(String, String), DelayedTask<Task>>::new();
-    let mut keys = vec![];
+pub async fn worker(db: Database, amqp: AMQP) {
+    let mut tasks = HashMap::<(Option<String>, String, u8), DelayedTask<Task>>::new();
+    let mut keys: Vec<(Option<String>, String, u8)> = vec![];
 
     loop {
         // Find due tasks.
@@ -106,12 +186,13 @@ pub async fn worker(db: Database, authifier_db: authifier::Database) {
         for key in &keys {
             if let Some(task) = tasks.remove(key) {
                 let Task { event } = task.data;
-                let (user, channel) = key;
+                let (user, channel, _) = key;
 
-                if let Err(err) = handle_ack_event(&event, &db, &authifier_db, user, channel).await {
-                    error!("{err:?} for {event:?}. ({user}, {channel})");
+                if let Err(err) = handle_ack_event(&event, &db, &amqp, user, channel).await {
+                    revolt_config::capture_error(&err);
+                    error!("{err:?} for {event:?}. ({user:?}, {channel})");
                 } else {
-                    info!("User {user} ack in {channel} with {event:?}");
+                    info!("User {user:?} ack in {channel} with {event:?}");
                 }
             }
         }
@@ -126,20 +207,41 @@ pub async fn worker(db: Database, authifier_db: authifier::Database) {
             mut event,
         }) = Q.try_pop()
         {
-            let key = (user, channel);
+            let key: (Option<String>, String, u8) = (
+                user,
+                channel,
+                match &event {
+                    AckEvent::AckMessage { .. } => 0,
+                    AckEvent::ProcessMessage { .. } => 1,
+                },
+            );
             if let Some(task) = tasks.get_mut(&key) {
-                task.delay();
-
                 match &mut event {
-                    AckEvent::AddMention { ids } => {
-                        if let AckEvent::AddMention { ids: existing } = &mut task.data.event {
-                            existing.append(ids);
+                    AckEvent::ProcessMessage { messages: new_data } => {
+                        if let AckEvent::ProcessMessage { messages: existing } =
+                            &mut task.data.event
+                        {
+                            // add the new message to the list of messages to be processed.
+                            existing.append(new_data);
+
+                            // put a cap on the amount of messages that can be queued, for particularly active channels
+                            if (existing.length() as u16)
+                                < revolt_config::config()
+                                    .await
+                                    .features
+                                    .advanced
+                                    .process_message_delay_limit
+                            {
+                                task.delay();
+                            }
                         } else {
-                            task.data.event = event;
+                            panic!("Somehow got an ack message in the add mention arm");
                         }
                     }
                     AckEvent::AckMessage { .. } => {
+                        // replace the last acked message with the new acked message
                         task.data.event = event;
+                        task.delay();
                     }
                 }
             } else {

crates/core/database/src/tasks/apple_notifications.rs

@@ -1,314 +0,0 @@
-use std::io::Cursor;
-
-use base64::{
-    engine::{self},
-    Engine as _,
-};
-use deadqueue::limited::Queue;
-use once_cell::sync::Lazy;
-use revolt_a2::{
-    request::{
-        notification::{DefaultAlert, NotificationOptions},
-        payload::{APSAlert, APSSound, PayloadLike, APS},
-    },
-    Client, ClientConfig, Endpoint, Error, ErrorBody, ErrorReason, Priority, PushType, Response,
-};
-use revolt_config::config;
-use revolt_models::v0::{Message, PushNotification};
-
-use crate::Database;
-
-/// Payload information, before assembly
-#[derive(Debug)]
-#[allow(non_snake_case)]
-pub struct ApnPayload {
-    message: Message,
-    url: String,
-    authorAvatar: String,
-    authorDisplayName: String,
-    channelName: String,
-}
-
-#[derive(Serialize, Debug)]
-#[allow(non_snake_case)]
-struct Payload<'a> {
-    aps: APS<'a>,
-    #[serde(skip_serializing)]
-    options: NotificationOptions<'a>,
-    #[serde(skip_serializing)]
-    device_token: &'a str,
-
-    message: &'a Message,
-    url: &'a str,
-    authorAvatar: &'a str,
-    authorDisplayName: &'a str,
-    channelName: &'a str,
-}
-
-impl<'a> PayloadLike for Payload<'a> {
-    fn get_device_token(&self) -> &'a str {
-        self.device_token
-    }
-    fn get_options(&self) -> &NotificationOptions {
-        &self.options
-    }
-}
-
-/// Task information
-#[derive(Debug)]
-pub struct AlertJob {
-    /// Session Id
-    session_id: String,
-
-    /// Device token
-    device_token: String,
-
-    /// User Id
-    user_id: String,
-
-    /// Title
-    title: String,
-
-    /// Body
-    body: String,
-
-    /// Thread Id
-    thread_id: String,
-
-    /// Category (informs the client what kind of notification is being sent.)
-    category: String,
-
-    /// Payload used by the iOS client to modify the notification
-    custom_payload: ApnPayload,
-}
-
-impl AlertJob {
-    fn format_title(notification: &PushNotification) -> String {
-        // ideally this changes depending on context
-        // in a server, it would look like "Sendername, #channelname in servername"
-        // in a group, it would look like "Sendername in groupname"
-        // in a dm it should just be "Sendername".
-        // not sure how feasible all those are given the PushNotification object as it currently stands.
-        format!(
-            "{} in {}",
-            notification.author, notification.message.channel
-        ) // TODO: this absolutely needs a channel name
-    }
-}
-
-#[derive(Debug)]
-pub struct BadgeJob {
-    /// Session Id
-    session_id: String,
-
-    /// Device token
-    device_token: String,
-
-    /// User Id
-    user_id: String,
-}
-
-#[derive(Debug)]
-pub enum JobType {
-    Alert(AlertJob),
-    Badge(BadgeJob),
-}
-
-#[derive(Debug)]
-pub struct ApnJob {
-    job_type: JobType,
-}
-
-impl ApnJob {
-    pub fn from_notification(
-        session_id: String,
-        user_id: String,
-        device_token: String,
-        notification: &PushNotification,
-    ) -> ApnJob {
-        ApnJob {
-            job_type: JobType::Alert(AlertJob {
-                session_id,
-                device_token,
-                user_id,
-                title: AlertJob::format_title(notification),
-                body: notification.body.to_string(),
-                thread_id: notification.tag.to_string(),
-                category: "ALERT_MESSAGE".to_string(),
-                custom_payload: ApnPayload {
-                    message: notification.message.clone(),
-                    url: notification.url.clone(),
-                    authorAvatar: notification.icon.clone(),
-                    authorDisplayName: notification.author.clone(),
-                    channelName: "#fetchchannelnamehere".to_string(), // TODO: get actual channel name
-                },
-            }),
-        }
-    }
-
-    pub fn from_ack(session_id: String, user_id: String, device_token: String) -> ApnJob {
-        ApnJob {
-            job_type: JobType::Badge(BadgeJob {
-                session_id,
-                device_token,
-                user_id,
-            }),
-        }
-    }
-}
-
-enum AssembledPayload<'a> {
-    Alert(Payload<'a>),
-    Default(revolt_a2::request::payload::Payload<'a>),
-}
-
-static Q: Lazy<Queue<ApnJob>> = Lazy::new(|| Queue::new(10_000));
-
-/// Queue a new task for a worker
-pub async fn queue(task: ApnJob) {
-    Q.try_push(task).ok();
-    info!("Queue is using {} slots from {}.", Q.len(), Q.capacity());
-}
-
-async fn get_badge_count(db: &Database, user: &str) -> Option<u32> {
-    if let Ok(unreads) = db.fetch_unreads(user).await {
-        let mut mention_count = 0;
-        for channel in unreads {
-            if let Some(mentions) = channel.mentions {
-                mention_count += mentions.len() as u32
-            }
-        }
-
-        return Some(mention_count);
-    }
-    None
-}
-
-/// Start a new worker
-pub async fn worker(db: Database) {
-    let config = config().await;
-    if config.api.apn.pkcs8.is_empty()
-        || config.api.apn.key_id.is_empty()
-        || config.api.apn.team_id.is_empty()
-    {
-        eprintln!("Missing APN keys.");
-        return;
-    }
-
-    let endpoint = if config.api.apn.sandbox {
-        Endpoint::Sandbox
-    } else {
-        Endpoint::Production
-    };
-
-    let pkcs8 = engine::general_purpose::STANDARD
-        .decode(config.api.apn.pkcs8)
-        .expect("valid `pcks8`");
-
-    let client_config = ClientConfig::new(endpoint);
-
-    let client = Client::token(
-        &mut Cursor::new(pkcs8),
-        config.api.apn.key_id,
-        config.api.apn.team_id,
-        client_config,
-    )
-    .expect("could not create APN client");
-
-    let payload_options = NotificationOptions {
-        apns_id: None,
-        apns_push_type: Some(PushType::Alert),
-        apns_expiration: None,
-        apns_priority: Some(Priority::High),
-        apns_topic: Some("chat.revolt.app"),
-        apns_collapse_id: None,
-    };
-
-    loop {
-        let task = Q.pop().await;
-        let payload: AssembledPayload;
-
-        match task.job_type {
-            JobType::Alert(ref alert) => {
-                payload = AssembledPayload::Alert(Payload {
-                    aps: APS {
-                        alert: Some(APSAlert::Default(DefaultAlert {
-                            title: Some(&alert.title),
-                            subtitle: None,
-                            body: Some(&alert.body),
-                            title_loc_key: None,
-                            title_loc_args: None,
-                            action_loc_key: None,
-                            loc_key: None,
-                            loc_args: None,
-                            launch_image: None,
-                        })),
-                        badge: get_badge_count(&db, &alert.user_id).await,
-                        sound: Some(APSSound::Sound("default")),
-                        thread_id: Some(&alert.thread_id),
-                        content_available: None,
-                        category: Some(&alert.category),
-                        mutable_content: Some(1),
-                        url_args: None,
-                    },
-                    device_token: &alert.device_token,
-                    options: payload_options.clone(),
-                    message: &alert.custom_payload.message,
-                    url: &alert.custom_payload.url,
-                    authorAvatar: &alert.custom_payload.authorAvatar,
-                    authorDisplayName: &alert.custom_payload.authorDisplayName,
-                    channelName: &alert.custom_payload.channelName,
-                });
-            }
-            JobType::Badge(ref alert) => {
-                payload = AssembledPayload::Default(revolt_a2::request::payload::Payload {
-                    aps: APS {
-                        alert: None,
-                        badge: get_badge_count(&db, &alert.user_id).await,
-                        sound: None,
-                        thread_id: None,
-                        content_available: None,
-                        category: None,
-                        mutable_content: None,
-                        url_args: None,
-                    },
-                    device_token: &alert.device_token,
-                    options: payload_options.clone(),
-                    data: std::collections::BTreeMap::new(),
-                })
-            }
-        }
-
-        let resp = match payload {
-            AssembledPayload::Alert(p) => client.send(p).await,
-            AssembledPayload::Default(p) => client.send(p).await,
-        };
-        //println!("response from APNS: {:?}", resp);
-
-        if let Err(err) = resp {
-            match err {
-                Error::ResponseError(Response {
-                    error:
-                        Some(ErrorBody {
-                            reason: ErrorReason::BadDeviceToken | ErrorReason::Unregistered,
-                            ..
-                        }),
-                    ..
-                }) => {
-                    if let Err(err) = db
-                        .remove_push_subscription_by_session_id(match task.job_type {
-                            JobType::Alert(ref a) => &a.session_id.as_str(),
-                            JobType::Badge(ref a) => &a.session_id.as_str(),
-                        })
-                        .await
-                    {
-                        revolt_config::capture_error(&err);
-                    }
-                }
-                err => {
-                    revolt_config::capture_error(&err);
-                }
-            }
-        }
-    }
-}

crates/core/database/src/tasks/mod.rs

@@ -1,6 +1,6 @@
 //! Semi-important background task management
 
-use crate::Database;
+use crate::{Database, AMQP};
 
 use async_std::task;
 use std::time::Instant;
@@ -8,22 +8,18 @@ use std::time::Instant;
 const WORKER_COUNT: usize = 5;
 
 pub mod ack;
-pub mod apple_notifications;
 pub mod authifier_relay;
 pub mod last_message_id;
 pub mod process_embeds;
-pub mod web_push;
 
 /// Spawn background workers
-pub fn start_workers(db: Database, authifier_db: authifier::Database) {
+pub fn start_workers(db: Database, amqp: AMQP) {
     task::spawn(authifier_relay::worker());
-    task::spawn(apple_notifications::worker(db.clone()));
 
     for _ in 0..WORKER_COUNT {
-        task::spawn(ack::worker(db.clone(), authifier_db.clone()));
+        task::spawn(ack::worker(db.clone(), amqp.clone()));
         task::spawn(last_message_id::worker(db.clone()));
         task::spawn(process_embeds::worker(db.clone()));
-        task::spawn(web_push::worker(db.clone(), authifier_db.clone()));
     }
 }
 

crates/core/database/src/tasks/web_push.rs

@@ -1,198 +0,0 @@
-use std::{
-    collections::{HashMap, HashSet},
-    time::Duration,
-};
-
-use authifier::Database as AuthifierDatabase;
-use base64::{
-    engine::{self},
-    Engine as _,
-};
-use deadqueue::limited::Queue;
-use fcm_v1::auth::{Authenticator, ServiceAccountKey};
-use once_cell::sync::Lazy;
-use revolt_config::{config, report_internal_error};
-use revolt_models::v0::PushNotification;
-use revolt_presence::filter_online;
-use serde_json::json;
-use web_push::{
-    ContentEncoding, IsahcWebPushClient, SubscriptionInfo, SubscriptionKeys, VapidSignatureBuilder,
-    WebPushClient, WebPushMessageBuilder,
-};
-
-use crate::Database;
-
-use super::apple_notifications;
-
-/// Task information
-#[derive(Debug)]
-struct PushTask {
-    /// User IDs of the targets that are to receive this notification
-    recipients: Vec<String>,
-    /// Push Notification
-    payload: PushNotification,
-}
-
-static Q: Lazy<Queue<PushTask>> = Lazy::new(|| Queue::new(10_000));
-
-/// Queue a new task for a worker
-pub async fn queue(recipients: Vec<String>, payload: PushNotification) {
-    if recipients.is_empty() {
-        return;
-    }
-
-    let online_ids = filter_online(&recipients).await;
-    let recipients = (&recipients.into_iter().collect::<HashSet<String>>() - &online_ids)
-        .into_iter()
-        .collect::<Vec<String>>();
-
-    Q.try_push(PushTask {
-        recipients,
-        payload,
-    })
-    .ok();
-
-    info!("Queue is using {} slots from {}.", Q.len(), Q.capacity());
-}
-
-/// Start a new worker
-pub async fn worker(db: Database, authifier_db: AuthifierDatabase) {
-    let config = config().await;
-
-    let web_push_client = IsahcWebPushClient::new().unwrap();
-    let fcm_client = if config.api.fcm.key_type.is_empty() {
-        None
-    } else {
-        Some(fcm_v1::Client::new(
-            Authenticator::service_account::<&str>(ServiceAccountKey {
-                key_type: Some(config.api.fcm.key_type),
-                project_id: Some(config.api.fcm.project_id.clone()),
-                private_key_id: Some(config.api.fcm.private_key_id),
-                private_key: config.api.fcm.private_key,
-                client_email: config.api.fcm.client_email,
-                client_id: Some(config.api.fcm.client_id),
-                auth_uri: Some(config.api.fcm.auth_uri),
-                token_uri: config.api.fcm.token_uri,
-                auth_provider_x509_cert_url: Some(config.api.fcm.auth_provider_x509_cert_url),
-                client_x509_cert_url: Some(config.api.fcm.client_x509_cert_url),
-            })
-            .await
-            .unwrap(),
-            config.api.fcm.project_id,
-            false,
-            Duration::from_secs(5),
-        ))
-    };
-
-    let web_push_private_key = engine::general_purpose::URL_SAFE_NO_PAD
-        .decode(config.api.vapid.private_key)
-        .expect("valid `VAPID_PRIVATE_KEY`");
-
-    loop {
-        let task = Q.pop().await;
-
-        if let Ok(sessions) = authifier_db
-            .find_sessions_with_subscription(&task.recipients)
-            .await
-        {
-            for session in sessions {
-                if let Some(sub) = session.subscription {
-                    if sub.endpoint == "fcm" {
-                        // Use Firebase Cloud Messaging
-                        if let Some(client) = &fcm_client {
-                            let message = fcm_v1::message::Message {
-                                token: Some(sub.auth),
-                                data: Some(HashMap::from([(
-                                    "payload".to_owned(),
-                                    serde_json::Value::String(json!(&task.payload).to_string()),
-                                )])),
-                                ..Default::default()
-                            };
-
-                            if let Err(err) = client.send(&message).await {
-                                error!("Failed to send FCM notification! {:?}", err);
-
-                                if let fcm_v1::Error::FCM(fcm_error) = err {
-                                    if fcm_error.contains("404 (Not Found)") {
-                                        println!("Unregistering {:?}", session.id);
-
-                                        report_internal_error!(
-                                            db.remove_push_subscription_by_session_id(&session.id)
-                                                .await
-                                        )
-                                        .ok();
-                                    }
-                                }
-                            } else {
-                                info!("Sent FCM notification to {:?}.", session.id);
-                            }
-                        } else {
-                            info!("No FCM token was specified!");
-                        }
-                    } else if sub.endpoint == "apn" {
-                        apple_notifications::queue(apple_notifications::ApnJob::from_notification(
-                            session.id,
-                            session.user_id,
-                            sub.auth,
-                            &task.payload,
-                        ))
-                        .await;
-                    } else {
-                        // Use Web Push Standard
-                        let subscription = SubscriptionInfo {
-                            endpoint: sub.endpoint,
-                            keys: SubscriptionKeys {
-                                auth: sub.auth,
-                                p256dh: sub.p256dh,
-                            },
-                        };
-
-                        match VapidSignatureBuilder::from_pem(
-                            std::io::Cursor::new(&web_push_private_key),
-                            &subscription,
-                        ) {
-                            Ok(sig_builder) => match sig_builder.build() {
-                                Ok(signature) => {
-                                    let mut builder = WebPushMessageBuilder::new(&subscription);
-                                    builder.set_vapid_signature(signature);
-
-                                    let payload = json!(task.payload).to_string();
-                                    builder
-                                        .set_payload(ContentEncoding::AesGcm, payload.as_bytes());
-
-                                    match builder.build() {
-                                        Ok(msg) => match web_push_client.send(msg).await {
-                                            Ok(_) => {
-                                                info!(
-                                                    "Sent Web Push notification to {:?}.",
-                                                    session.id
-                                                )
-                                            }
-                                            Err(err) => {
-                                                error!("Hit error sending Web Push! {:?}", err)
-                                            }
-                                        },
-                                        Err(err) => {
-                                            error!(
-                                                "Failed to build message for {}! {:?}",
-                                                session.user_id, err
-                                            )
-                                        }
-                                    }
-                                }
-                                Err(err) => error!(
-                                    "Failed to build signature for {}! {:?}",
-                                    session.user_id, err
-                                ),
-                            },
-                            Err(err) => error!(
-                                "Failed to create signature builder for {}! {:?}",
-                                session.user_id, err
-                            ),
-                        }
-                    }
-                }
-            }
-        }
-    }
-}

crates/core/database/src/util/bulk_permissions.rs

@@ -0,0 +1,337 @@
+use std::{collections::HashMap, hash::RandomState};
+
+use revolt_permissions::{
+    ChannelPermission, ChannelType, Override, OverrideField, PermissionValue, ALLOW_IN_TIMEOUT,
+    DEFAULT_PERMISSION_DIRECT_MESSAGE,
+};
+
+use crate::{Channel, Database, Member, Server, User};
+
+#[derive(Clone)]
+pub struct BulkDatabasePermissionQuery<'a> {
+    #[allow(dead_code)]
+    database: &'a Database,
+
+    server: Server,
+    channel: Option<Channel>,
+    users: Option<Vec<User>>,
+    members: Option<Vec<Member>>,
+
+    // In case the users or members are fetched as part of the permissions checking operation
+    pub(crate) cached_users: Option<Vec<User>>,
+    pub(crate) cached_members: Option<Vec<Member>>,
+
+    cached_member_perms: Option<HashMap<String, PermissionValue>>,
+}
+
+impl<'z, 'x> BulkDatabasePermissionQuery<'x> {
+    pub async fn members_can_see_channel(&'z mut self) -> HashMap<String, bool>
+    where
+        'z: 'x,
+    {
+        let member_perms = if self.cached_member_perms.is_some() {
+            // This isn't done as an if let to prevent borrow checker errors with the mut self call when the perms aren't cached.
+            let perms = self.cached_member_perms.as_ref().unwrap();
+            perms
+                .iter()
+                .map(|(m, p)| {
+                    (
+                        m.clone(),
+                        p.has_channel_permission(ChannelPermission::ViewChannel),
+                    )
+                })
+                .collect()
+        } else {
+            calculate_members_permissions(self)
+                .await
+                .iter()
+                .map(|(m, p)| {
+                    (
+                        m.clone(),
+                        p.has_channel_permission(ChannelPermission::ViewChannel),
+                    )
+                })
+                .collect()
+        };
+        member_perms
+    }
+}
+
+impl<'z> BulkDatabasePermissionQuery<'z> {
+    pub fn new(database: &Database, server: Server) -> BulkDatabasePermissionQuery<'_> {
+        BulkDatabasePermissionQuery {
+            database,
+            server,
+            channel: None,
+            users: None,
+            members: None,
+            cached_members: None,
+            cached_users: None,
+            cached_member_perms: None,
+        }
+    }
+
+    pub async fn from_server_id<'a>(
+        db: &'a Database,
+        server: &str,
+    ) -> BulkDatabasePermissionQuery<'a> {
+        BulkDatabasePermissionQuery {
+            database: db,
+            server: db.fetch_server(server).await.unwrap(),
+            channel: None,
+            users: None,
+            members: None,
+            cached_members: None,
+            cached_users: None,
+            cached_member_perms: None,
+        }
+    }
+
+    pub fn channel(self, channel: &'z Channel) -> BulkDatabasePermissionQuery {
+        BulkDatabasePermissionQuery {
+            channel: Some(channel.clone()),
+            ..self
+        }
+    }
+
+    pub fn members(self, members: &'z [Member]) -> BulkDatabasePermissionQuery {
+        BulkDatabasePermissionQuery {
+            members: Some(members.to_owned()),
+            ..self
+        }
+    }
+
+    pub fn users(self, users: &'z [User]) -> BulkDatabasePermissionQuery {
+        BulkDatabasePermissionQuery {
+            users: Some(users.to_owned()),
+            ..self
+        }
+    }
+
+    /// Get the default channel permissions
+    /// Group channel defaults should be mapped to an allow-only override
+    #[allow(dead_code)]
+    async fn get_default_channel_permissions(&mut self) -> Override {
+        if let Some(channel) = &self.channel {
+            match channel {
+                Channel::Group { permissions, .. } => Override {
+                    allow: permissions.unwrap_or(*DEFAULT_PERMISSION_DIRECT_MESSAGE as i64) as u64,
+                    deny: 0,
+                },
+                Channel::TextChannel {
+                    default_permissions,
+                    ..
+                }
+                | Channel::VoiceChannel {
+                    default_permissions,
+                    ..
+                } => default_permissions.unwrap_or_default().into(),
+                _ => Default::default(),
+            }
+        } else {
+            Default::default()
+        }
+    }
+
+    #[allow(dead_code)]
+    fn get_channel_type(&mut self) -> ChannelType {
+        if let Some(channel) = &self.channel {
+            match channel {
+                Channel::DirectMessage { .. } => ChannelType::DirectMessage,
+                Channel::Group { .. } => ChannelType::Group,
+                Channel::SavedMessages { .. } => ChannelType::SavedMessages,
+                Channel::TextChannel { .. } | Channel::VoiceChannel { .. } => {
+                    ChannelType::ServerChannel
+                }
+            }
+        } else {
+            ChannelType::Unknown
+        }
+    }
+
+    /// Get the ordered role overrides (from lowest to highest) for this member in this channel
+    #[allow(dead_code)]
+    async fn get_channel_role_overrides(&mut self) -> &HashMap<String, OverrideField> {
+        if let Some(channel) = &self.channel {
+            match channel {
+                Channel::TextChannel {
+                    role_permissions, ..
+                }
+                | Channel::VoiceChannel {
+                    role_permissions, ..
+                } => role_permissions,
+                _ => panic!("Not supported for non-server channels"),
+            }
+        } else {
+            panic!("No channel added to query")
+        }
+    }
+}
+
+/// Calculate members permissions in a server channel.
+async fn calculate_members_permissions<'a>(
+    query: &'a mut BulkDatabasePermissionQuery<'a>,
+) -> HashMap<String, PermissionValue> {
+    let mut resp = HashMap::new();
+
+    let (_, channel_role_permissions, channel_default_permissions) = match query
+        .channel
+        .as_ref()
+        .expect("A channel must be assigned to calculate channel permissions")
+        .clone()
+    {
+        Channel::TextChannel {
+            id,
+            role_permissions,
+            default_permissions,
+            ..
+        }
+        | Channel::VoiceChannel {
+            id,
+            role_permissions,
+            default_permissions,
+            ..
+        } => (id, role_permissions, default_permissions),
+        _ => panic!("Calculation of member permissions must be done on a server channel"),
+    };
+
+    if query.users.is_none() {
+        let ids: Vec<String> = query
+            .members
+            .as_ref()
+            .expect("No users or members added to the query")
+            .iter()
+            .map(|m| m.id.user.clone())
+            .collect();
+
+        query.cached_users = Some(
+            query
+                .database
+                .fetch_users(&ids[..])
+                .await
+                .expect("Failed to get data from the db"),
+        );
+
+        query.users = Some(query.cached_users.as_ref().unwrap().to_vec())
+    }
+
+    let users = query.users.as_ref().unwrap();
+
+    if query.members.is_none() {
+        let ids: Vec<String> = query
+            .users
+            .as_ref()
+            .expect("No users or members added to the query")
+            .iter()
+            .map(|m| m.id.clone())
+            .collect();
+
+        query.cached_members = Some(
+            query
+                .database
+                .fetch_members(&query.server.id, &ids[..])
+                .await
+                .expect("Failed to get data from the db"),
+        );
+        query.members = Some(query.cached_members.as_ref().unwrap().to_vec())
+    }
+
+    let members: HashMap<&String, &Member, RandomState> = HashMap::from_iter(
+        query
+            .members
+            .as_ref()
+            .unwrap()
+            .iter()
+            .map(|m| (&m.id.user, m)),
+    );
+
+    for user in users {
+        let member = members.get(&user.id);
+
+        // User isn't a part of the server
+        if member.is_none() {
+            resp.insert(user.id.clone(), 0_u64.into());
+            continue;
+        }
+
+        let member = *member.unwrap();
+
+        if user.privileged {
+            resp.insert(
+                user.id.clone(),
+                PermissionValue::from(ChannelPermission::GrantAllSafe),
+            );
+            continue;
+        }
+
+        if user.id == query.server.owner {
+            resp.insert(
+                user.id.clone(),
+                PermissionValue::from(ChannelPermission::GrantAllSafe),
+            );
+            continue;
+        }
+
+        // Get the user's server permissions
+        let mut permission = calculate_server_permissions(&query.server, user, member);
+
+        if let Some(defaults) = channel_default_permissions {
+            permission.apply(defaults.into());
+        }
+
+        // Get the applicable role overrides
+        let mut roles = channel_role_permissions
+            .iter()
+            .filter(|(id, _)| member.roles.contains(id))
+            .filter_map(|(id, permission)| {
+                query.server.roles.get(id).map(|role| {
+                    let v: Override = (*permission).into();
+                    (role.rank, v)
+                })
+            })
+            .collect::<Vec<(i64, Override)>>();
+
+        roles.sort_by(|a, b| b.0.cmp(&a.0));
+        let overrides = roles.into_iter().map(|(_, v)| v);
+
+        for role_override in overrides {
+            permission.apply(role_override)
+        }
+
+        resp.insert(user.id.clone(), permission);
+    }
+
+    resp
+}
+
+/// Calculates a member's server permissions
+fn calculate_server_permissions(server: &Server, user: &User, member: &Member) -> PermissionValue {
+    if user.privileged || server.owner == user.id {
+        return ChannelPermission::GrantAllSafe.into();
+    }
+
+    let mut permissions: PermissionValue = server.default_permissions.into();
+
+    let mut roles = server
+        .roles
+        .iter()
+        .filter(|(id, _)| member.roles.contains(id))
+        .map(|(_, role)| {
+            let v: Override = role.permissions.into();
+            (role.rank, v)
+        })
+        .collect::<Vec<(i64, Override)>>();
+
+    roles.sort_by(|a, b| b.0.cmp(&a.0));
+    let role_overrides: Vec<Override> = roles.into_iter().map(|(_, v)| v).collect();
+
+    for role in role_overrides {
+        permissions.apply(role);
+    }
+
+    if member.in_timeout() {
+        permissions.restrict(*ALLOW_IN_TIMEOUT);
+    }
+
+    permissions
+}

crates/core/database/src/util/idempotency.rs

@@ -102,7 +102,7 @@ impl<'r> FromRequest<'r> for IdempotencyKey {
             .map(|k| k.to_string())
         {
             if key.len() > 64 {
-                return Outcome::Failure((
+                return Outcome::Error((
                     Status::BadRequest,
                     create_error!(FailedValidation {
                         error: "idempotency key too long".to_string(),
@@ -113,7 +113,7 @@ impl<'r> FromRequest<'r> for IdempotencyKey {
             let idempotency = IdempotencyKey { key };
             let mut cache = TOKEN_CACHE.lock().await;
             if cache.get(&idempotency.key).is_some() {
-                return Outcome::Failure((Status::Conflict, create_error!(DuplicateNonce)));
+                return Outcome::Error((Status::Conflict, create_error!(DuplicateNonce)));
             }
 
             cache.put(idempotency.key.clone(), ());

crates/core/database/src/util/mod.rs

@@ -1,4 +1,5 @@
 pub mod bridge;
+pub mod bulk_permissions;
 pub mod idempotency;
 pub mod permissions;
 pub mod reference;

crates/core/files/Cargo.toml

@@ -1,12 +1,18 @@
 [package]
 name = "revolt-files"
-version = "0.7.16"
+version = "0.8.0"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <me@insrt.uk>"]
 description = "Revolt Backend: S3 and encryption subroutines"
 
 [dependencies]
+tracing = "0.1"
+
+ffprobe = "0.4.0"
+imagesize = "0.13.0"
+tempfile = "3.12.0"
+
 base64 = "0.22.1"
 aes-gcm = "0.10.3"
 typenum = "1.17.0"
@@ -14,7 +20,19 @@ typenum = "1.17.0"
 aws-config = "1.5.5"
 aws-sdk-s3 = { version = "1.46.0", features = ["behavior-version-latest"] }
 
-revolt-config = { version = "0.7.16", path = "../config", features = [
+revolt-config = { version = "0.8.0", path = "../config", features = [
     "report-macros",
 ] }
-revolt-result = { version = "0.7.16", path = "../result" }
+revolt-result = { version = "0.8.0", path = "../result" }
+
+# image processing
+jxl-oxide = "0.8.1"
+image = { version = "0.25.2" }
+
+# svg rendering
+usvg = "0.44.0"
+resvg = "0.44.0"
+tiny-skia = "0.11.4"
+
+# encoding
+webp = "0.3.0"

crates/core/files/src/lib.rs

@@ -1,9 +1,10 @@
-use std::io::Write;
+use std::io::{BufRead, Read, Seek, Write};
 
 use aes_gcm::{
     aead::{AeadCore, AeadMutInPlace, OsRng},
     Aes256Gcm, Key, KeyInit, Nonce,
 };
+use image::{DynamicImage, ImageBuffer};
 use revolt_config::{config, report_internal_error, FilesS3};
 use revolt_result::{create_error, Result};
 
@@ -13,6 +14,8 @@ use aws_sdk_s3::{
 };
 
 use base64::prelude::*;
+use tempfile::NamedTempFile;
+use tiny_skia::Pixmap;
 
 /// Size of the authentication tag in the buffer
 pub const AUTHENTICATION_TAG_SIZE_BYTES: usize = 16;
@@ -31,6 +34,7 @@ pub fn create_client(s3_config: FilesS3) -> Client {
     let config = Config::builder()
         .region(Region::new(s3_config.region))
         .endpoint_url(s3_config.endpoint)
+        .force_path_style(s3_config.path_style_buckets)
         .credentials_provider(creds)
         .build();
 
@@ -108,3 +112,162 @@ pub async fn upload_to_s3(bucket_id: &str, path: &str, buf: &[u8]) -> Result<Str
 
     Ok(BASE64_STANDARD.encode(nonce))
 }
+
+/// Determine size of image at temp file
+pub fn image_size(f: &NamedTempFile) -> Option<(usize, usize)> {
+    if let Ok(size) = imagesize::size(f.path())
+        .inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}"))
+    {
+        Some((size.width, size.height))
+    } else {
+        None
+    }
+}
+
+/// Determine size of image with buffer
+pub fn image_size_vec(v: &[u8], mime: &str) -> Option<(usize, usize)> {
+    match mime {
+        "image/svg+xml" => {
+            let tree =
+                report_internal_error!(usvg::Tree::from_data(v, &Default::default())).ok()?;
+
+            let size = tree.size();
+            Some((size.width() as usize, size.height() as usize))
+        }
+        _ => {
+            if let Ok(size) = imagesize::blob_size(v)
+                .inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}"))
+            {
+                Some((size.width, size.height))
+            } else {
+                None
+            }
+        }
+    }
+}
+
+/// Determine size of video at temp file
+pub fn video_size(f: &NamedTempFile) -> Option<(i64, i64)> {
+    if let Ok(data) = ffprobe::ffprobe(f.path())
+        .inspect_err(|err| tracing::error!("Failed to ffprobe file! {err:?}"))
+    {
+        // Use first valid stream
+        for stream in data.streams {
+            if let (Some(w), Some(h)) = (stream.width, stream.height) {
+                return Some((w, h));
+            }
+        }
+
+        None
+    } else {
+        None
+    }
+}
+
+/// Decode image from reader
+pub fn decode_image<R: Read + BufRead + Seek>(reader: &mut R, mime: &str) -> Result<DynamicImage> {
+    match mime {
+        // Read image using jxl-oxide crate
+        "image/jxl" => {
+            let jxl_image = report_internal_error!(jxl_oxide::JxlImage::builder().read(reader))?;
+            if let Ok(frame) = jxl_image.render_frame(0) {
+                match frame.color_channels().len() {
+                    3 => Ok(DynamicImage::ImageRgb8(
+                        DynamicImage::ImageRgb32F(
+                            ImageBuffer::from_vec(
+                                jxl_image.width(),
+                                jxl_image.height(),
+                                frame.image().buf().to_vec(),
+                            )
+                            .ok_or_else(|| create_error!(ImageProcessingFailed))?,
+                        )
+                        .to_rgb8(),
+                    )),
+                    4 => Ok(DynamicImage::ImageRgba8(
+                        DynamicImage::ImageRgba32F(
+                            ImageBuffer::from_vec(
+                                jxl_image.width(),
+                                jxl_image.height(),
+                                frame.image().buf().to_vec(),
+                            )
+                            .ok_or_else(|| create_error!(ImageProcessingFailed))?,
+                        )
+                        .to_rgba8(),
+                    )),
+                    _ => Err(create_error!(ImageProcessingFailed)),
+                }
+            } else {
+                Err(create_error!(ImageProcessingFailed))
+            }
+        }
+        // Read image using resvg
+        "image/svg+xml" => {
+            // usvg doesn't support Read trait so copy to buffer
+            let mut buf = Vec::new();
+            report_internal_error!(reader.read_to_end(&mut buf))?;
+
+            let tree = report_internal_error!(usvg::Tree::from_data(&buf, &Default::default()))?;
+            let size = tree.size();
+            let mut pixmap = Pixmap::new(size.width() as u32, size.height() as u32)
+                .ok_or_else(|| create_error!(ImageProcessingFailed))?;
+
+            let mut pixmap_mut = pixmap.as_mut();
+            resvg::render(&tree, Default::default(), &mut pixmap_mut);
+
+            Ok(DynamicImage::ImageRgba8(
+                ImageBuffer::from_vec(
+                    size.width() as u32,
+                    size.height() as u32,
+                    pixmap.data().to_vec(),
+                )
+                .ok_or_else(|| create_error!(ImageProcessingFailed))?,
+            ))
+        }
+        // Check if we can read using image-rs crate
+        _ => report_internal_error!(report_internal_error!(
+            image::ImageReader::new(reader).with_guessed_format()
+        )?
+        .decode()),
+    }
+}
+
+/// Check whether given reader has a valid image
+pub fn is_valid_image<R: Read + BufRead + Seek>(reader: &mut R, mime: &str) -> bool {
+    match mime {
+        // Check if we can read using jxl-oxide crate
+        "image/jxl" => jxl_oxide::JxlImage::builder()
+            .read(reader)
+            .inspect_err(|err| tracing::error!("Failed to read JXL! {err:?}"))
+            .is_ok(),
+        // Check if we can read using image-rs crate
+        _ => !matches!(
+            image::ImageReader::new(reader)
+                .with_guessed_format()
+                .inspect_err(|err| tracing::error!("Failed to read image! {err:?}"))
+                .map(|f| f.decode()),
+            Err(_) | Ok(Err(_))
+        ),
+    }
+}
+
+/// Create thumbnail from given image
+pub async fn create_thumbnail(image: DynamicImage, tag: &str) -> Vec<u8> {
+    // Load configuration
+    let config = config().await;
+    let [w, h] = config.files.preview.get(tag).unwrap();
+
+    // Create thumbnail
+    //.resize(width as u32, height as u32, image::imageops::FilterType::Gaussian)
+    // resize is about 2.5x slower,
+    // thumbnail doesn't have terrible quality
+    // so we use thumbnail
+    let image = image.thumbnail(image.width().min(*w as u32), image.height().min(*h as u32));
+
+    // Encode it into WEBP
+    let encoder = webp::Encoder::from_image(&image).expect("Could not create encoder.");
+    if config.files.webp_quality != 100.0 {
+        encoder.encode(config.files.webp_quality).to_vec()
+    } else {
+        encoder.encode_lossless().to_vec()
+    }
+}

crates/core/models/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-models"
-version = "0.7.16"
+version = "0.8.0"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -20,8 +20,8 @@ default = ["serde", "partials", "rocket"]
 
 [dependencies]
 # Core
-revolt-config = { version = "0.7.16", path = "../config" }
-revolt-permissions = { version = "0.7.16", path = "../permissions" }
+revolt-config = { version = "0.8.0", path = "../config" }
+revolt-permissions = { version = "0.8.0", path = "../permissions" }
 
 # Utility
 regex = "1"

crates/core/models/src/v0/channels.rs

@@ -306,4 +306,18 @@ impl Channel {
             | Channel::VoiceChannel { id, .. } => id,
         }
     }
+
+    /// This returns a Result because the recipient name can't be determined here without a db call,
+    /// which can't be done since this is models, which can't reference the database crate.
+    ///
+    /// If it returns Err, you need to fetch the name from the db.
+    pub fn name(&self) -> Result<&str, ()> {
+        match self {
+            Channel::DirectMessage { .. } => Err(()),
+            Channel::SavedMessages { .. } => Ok("Saved Messages"),
+            Channel::TextChannel { name, .. }
+            | Channel::Group { name, .. }
+            | Channel::VoiceChannel { name, .. } => Ok(name),
+        }
+    }
 }

crates/core/models/src/v0/embeds.rs

@@ -14,9 +14,9 @@ auto_derived!(
         /// URL to the original image
         pub url: String,
         /// Width of the image
-        pub width: isize,
+        pub width: usize,
         /// Height of the image
-        pub height: isize,
+        pub height: usize,
         /// Positioning and size
         pub size: ImageSize,
     }
@@ -26,9 +26,9 @@ auto_derived!(
         /// URL to the original video
         pub url: String,
         /// Width of the video
-        pub width: isize,
+        pub width: usize,
         /// Height of the video
-        pub height: isize,
+        pub height: usize,
     }
 
     /// Type of remote Twitch content
@@ -86,7 +86,7 @@ auto_derived!(
         },
         AppleMusic {
             album_id: String,
-            
+
             #[serde(skip_serializing_if = "Option::is_none")]
             track_id: Option<String>,
         },
@@ -119,8 +119,6 @@ auto_derived!(
         #[serde(skip_serializing_if = "Option::is_none")]
         pub video: Option<Video>,
 
-        // #[serde(skip_serializing_if = "Option::is_none")]
-        // opengraph_type: Option<String>,
         /// Site name
         #[serde(skip_serializing_if = "Option::is_none")]
         pub site_name: Option<String>,
@@ -156,11 +154,56 @@ auto_derived!(
 
     /// Embed
     #[serde(tag = "type")]
+    #[derive(Default)]
     pub enum Embed {
         Website(WebsiteMetadata),
         Image(Image),
         Video(Video),
         Text(Text),
+        #[default]
         None,
     }
 );
+
+impl WebsiteMetadata {
+    /// Truncate strings in metadata
+    pub fn truncate(&mut self) {
+        if let Some(s) = self.url.as_mut() {
+            s.truncate(256);
+        }
+
+        if let Some(s) = self.original_url.as_mut() {
+            s.truncate(256);
+        }
+
+        if let Some(s) = self.title.as_mut() {
+            s.truncate(100);
+        }
+
+        if let Some(s) = self.description.as_mut() {
+            s.truncate(1000);
+        }
+
+        if let Some(s) = self.site_name.as_mut() {
+            s.truncate(32);
+        }
+
+        if let Some(s) = self.icon_url.as_mut() {
+            s.truncate(256);
+        }
+
+        if let Some(s) = self.colour.as_mut() {
+            s.truncate(32);
+        }
+    }
+
+    /// Check if this is considered "empty"
+    pub fn is_empty(&self) -> bool {
+        (self.title.is_none() || self.title.as_ref().is_some_and(|f| f.is_empty()))
+            && (self.description.is_none()
+                || self.description.as_ref().is_some_and(|f| f.is_empty()))
+            && self.special.is_none()
+            && self.video.is_none()
+            && self.image.is_none()
+    }
+}

crates/core/models/src/v0/messages.rs

@@ -13,7 +13,7 @@ use rocket::{FromForm, FromFormField};
 
 use iso8601_timestamp::Timestamp;
 
-use super::{Embed, File, Member, MessageWebhook, User, Webhook, RE_COLOUR};
+use super::{Channel, Embed, File, Member, MessageWebhook, User, Webhook, RE_COLOUR};
 
 pub static RE_MENTION: Lazy<Regex> =
     Lazy::new(|| Regex::new(r"<@([0-9A-HJKMNP-TV-Z]{26})>").unwrap());
@@ -209,6 +209,8 @@ auto_derived!(
         pub url: String,
         /// The message object itself, to send to clients for processing
         pub message: Message,
+        /// The channel object itself, for clients to process
+        pub channel: Channel,
     }
 
     /// Representation of a text embed before it is sent.
@@ -369,7 +371,7 @@ auto_derived!(
 
     /// Optional fields on message
     pub enum FieldsMessage {
-        Pinned
+        Pinned,
     }
 );
 
@@ -442,7 +444,7 @@ impl From<SystemMessage> for String {
 
 impl PushNotification {
     /// Create a new notification from a given message, author and channel ID
-    pub async fn from(msg: Message, author: Option<MessageAuthor<'_>>, channel_id: &str) -> Self {
+    pub async fn from(msg: Message, author: Option<MessageAuthor<'_>>, channel: Channel) -> Self {
         let config = config().await;
 
         let icon = if let Some(author) = &author {
@@ -496,10 +498,11 @@ impl PushNotification {
             icon,
             image,
             body,
-            tag: channel_id.to_string(),
+            tag: channel.id().to_string(),
             timestamp,
-            url: format!("{}/channel/{}/{}", config.hosts.app, channel_id, msg.id),
+            url: format!("{}/channel/{}/{}", config.hosts.app, channel.id(), msg.id),
             message: msg,
+            channel,
         }
     }
 }

crates/core/permissions/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-permissions"
-version = "0.7.16"
+version = "0.8.0"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] }
 
 [dependencies]
 # Core
-revolt-result = { version = "0.7.16", path = "../result" }
+revolt-result = { version = "0.8.0", path = "../result" }
 
 # Utility
 auto_ops = "0.3.0"

crates/core/presence/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-presence"
-version = "0.7.16"
+version = "0.8.0"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <me@insrt.uk>"]

crates/core/result/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-result"
-version = "0.7.16"
+version = "0.8.0"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -29,7 +29,7 @@ utoipa = { version = "4.2.3", optional = true }
 
 # Rocket
 rocket = { optional = true, version = "0.5.0-rc.2", default-features = false }
-revolt_rocket_okapi = { version = "0.9.1", optional = true }
+revolt_rocket_okapi = { version = "0.10.0", optional = true }
 revolt_okapi = { version = "0.9.1", optional = true }
 
 # Axum

crates/core/result/src/axum.rs

@@ -1,4 +1,5 @@
 use axum::{http::StatusCode, response::IntoResponse, Json};
+use rocket::http::Status;
 
 use crate::{Error, ErrorType};
 
@@ -78,6 +79,8 @@ impl IntoResponse for Error {
             ErrorType::FileTooSmall => StatusCode::UNPROCESSABLE_ENTITY,
             ErrorType::FileTooLarge { .. } => StatusCode::UNPROCESSABLE_ENTITY,
             ErrorType::FileTypeNotAllowed => StatusCode::BAD_REQUEST,
+            ErrorType::ImageProcessingFailed => StatusCode::INTERNAL_SERVER_ERROR,
+            ErrorType::NoEmbedData => StatusCode::BAD_REQUEST,
         };
 
         (status, Json(&self)).into_response()

crates/core/result/src/lib.rs

@@ -162,6 +162,8 @@ pub enum ErrorType {
         max: usize,
     },
     FileTypeNotAllowed,
+    ImageProcessingFailed,
+    NoEmbedData,
 
     // ? Legacy errors
     VosoUnavailable,

crates/core/result/src/rocket.rs

@@ -84,6 +84,8 @@ impl<'r> Responder<'r, 'static> for Error {
             ErrorType::FileTooSmall => Status::UnprocessableEntity,
             ErrorType::FileTooLarge { .. } => Status::UnprocessableEntity,
             ErrorType::FileTypeNotAllowed => Status::BadRequest,
+            ErrorType::ImageProcessingFailed => Status::InternalServerError,
+            ErrorType::NoEmbedData => Status::BadRequest,
         };
 
         // Serialize the error data structure into JSON.

crates/daemons/pushd/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "revolt-pushd"
+version = "0.8.0"
+edition = "2021"
+
+[dependencies]
+revolt-config = { version = "0.8.0", path = "../../core/config" }
+revolt-database = { version = "0.8.0", path = "../../core/database" }
+revolt-models = { version = "0.8.0", path = "../../core/models", features = [
+    "validator",
+] }
+
+amqprs = { version = "1.7.0" }
+fcm_v1 = "0.3.0"
+web-push = "0.10.0"
+isahc = { optional = true, version = "1.7", features = ["json"] }
+revolt_a2 = { version = "0.10", default-features = false, features = ["ring"] }
+tokio = "1.39.2"
+async-trait = "0.1.81"
+ulid = "1.0.0"
+
+authifier = "1.0.8"
+
+log = "0.4.11"
+
+#serialization
+serde_json = "1"
+revolt_optional_struct = "0.2.0"
+serde = { version = "1", features = ["derive"] }
+iso8601-timestamp = { version = "0.2.10", features = ["serde", "bson"] }
+base64 = "0.22.1"

crates/daemons/pushd/Dockerfile

@@ -0,0 +1,9 @@
+# Build Stage
+FROM ghcr.io/revoltchat/base:latest AS builder
+
+# Bundle Stage
+FROM gcr.io/distroless/cc-debian12:nonroot
+COPY --from=builder /home/rust/src/target/release/revolt-pushd ./
+
+USER nonroot
+CMD ["./revolt-pushd"]

crates/daemons/pushd/src/consumers/inbound/ack.rs

@@ -0,0 +1,138 @@
+use crate::consumers::inbound::internal::*;
+use amqprs::{
+    channel::{BasicPublishArguments, Channel},
+    connection::Connection,
+    consumer::AsyncConsumer,
+    BasicProperties, Deliver,
+};
+use async_trait::async_trait;
+use revolt_database::{events::rabbit::*, Database};
+
+pub struct AckConsumer {
+    #[allow(dead_code)]
+    db: Database,
+    authifier_db: authifier::Database,
+    conn: Option<Connection>,
+    channel: Option<Channel>,
+}
+
+impl Channeled for AckConsumer {
+    fn get_connection(&self) -> Option<&Connection> {
+        if self.conn.is_none() {
+            None
+        } else {
+            Some(self.conn.as_ref().unwrap())
+        }
+    }
+
+    fn get_channel(&self) -> Option<&Channel> {
+        if self.channel.is_none() {
+            None
+        } else {
+            Some(self.channel.as_ref().unwrap())
+        }
+    }
+
+    fn set_connection(&mut self, conn: Connection) {
+        self.conn = Some(conn);
+    }
+
+    fn set_channel(&mut self, channel: Channel) {
+        self.channel = Some(channel)
+    }
+}
+
+impl AckConsumer {
+    pub fn new(db: Database, authifier_db: authifier::Database) -> AckConsumer {
+        AckConsumer {
+            db,
+            authifier_db,
+            conn: None,
+            channel: None,
+        }
+    }
+}
+
+#[allow(unused_variables)]
+#[async_trait]
+impl AsyncConsumer for AckConsumer {
+    /// This consumer processes all acks the platform receives, and sends relevant badge updates to apple platforms.
+    async fn consume(
+        &mut self,
+        channel: &Channel,
+        deliver: Deliver,
+        basic_properties: BasicProperties,
+        content: Vec<u8>,
+    ) {
+        let content = String::from_utf8(content).unwrap();
+        let payload: AckPayload = serde_json::from_str(content.as_str()).unwrap();
+
+        // Step 1: fetch unreads and don't continue if there's no unreads
+        #[allow(clippy::disallowed_methods)]
+        let unreads = self.db.fetch_unread_mentions(&payload.user_id).await;
+
+        if let Ok(u) = &unreads {
+            if u.is_empty() {
+                return;
+            }
+        } else {
+            return;
+        }
+
+        if let Ok(sessions) = self.authifier_db.find_sessions(&payload.user_id).await {
+            let config = revolt_config::config().await;
+            // Step 2: find any apple sessions, since we don't need to calculate this for anything else.
+            // If there's no apple sessions, we can return early
+            let apple_sessions: Vec<&authifier::models::Session> = sessions
+                .iter()
+                .filter(|session| {
+                    if let Some(sub) = &session.subscription {
+                        sub.endpoint == "apn"
+                    } else {
+                        false
+                    }
+                })
+                .collect();
+
+            if apple_sessions.is_empty() {
+                return;
+            }
+
+            // Step 3: calculate the actual mention count, since we have to send it out
+            let mut mention_count = 0;
+            for u in &unreads.unwrap() {
+                mention_count += u.mentions.as_ref().unwrap().len()
+            }
+
+            // Step 4: loop through each apple session and send the badge update
+            for session in apple_sessions {
+                let service_payload = PayloadToService {
+                    notification: PayloadKind::BadgeUpdate(mention_count),
+                    user_id: payload.user_id.clone(),
+                    session_id: session.id.clone(),
+                    token: session.subscription.as_ref().unwrap().auth.clone(),
+                    extras: Default::default(),
+                };
+                let raw_service_payload = serde_json::to_string(&service_payload);
+
+                if let Ok(p) = raw_service_payload {
+                    let args = BasicPublishArguments::new(
+                        config.pushd.exchange.as_str(),
+                        config.pushd.apn.queue.as_str(),
+                    )
+                    .finish();
+
+                    log::debug!(
+                        "Publishing ack to apn session {}",
+                        session.subscription.as_ref().unwrap().auth
+                    );
+
+                    publish_message(self, p.into(), args).await;
+                } else {
+                    log::warn!("Failed to serialize ack badge update payload!");
+                    revolt_config::capture_error(&raw_service_payload.unwrap_err());
+                }
+            }
+        }
+    }
+}

crates/daemons/pushd/src/consumers/inbound/fr_accepted.rs

@@ -0,0 +1,121 @@
+use std::collections::HashMap;
+
+use crate::consumers::inbound::internal::*;
+use amqprs::{
+    channel::{BasicPublishArguments, Channel},
+    connection::Connection,
+    consumer::AsyncConsumer,
+    BasicProperties, Deliver,
+};
+use async_trait::async_trait;
+use log::debug;
+use revolt_database::{events::rabbit::*, Database};
+
+pub struct FRAcceptedConsumer {
+    #[allow(dead_code)]
+    db: Database,
+    authifier_db: authifier::Database,
+    conn: Option<Connection>,
+    channel: Option<Channel>,
+}
+
+impl Channeled for FRAcceptedConsumer {
+    fn get_connection(&self) -> Option<&Connection> {
+        if self.conn.is_none() {
+            None
+        } else {
+            Some(self.conn.as_ref().unwrap())
+        }
+    }
+
+    fn get_channel(&self) -> Option<&Channel> {
+        if self.channel.is_none() {
+            None
+        } else {
+            Some(self.channel.as_ref().unwrap())
+        }
+    }
+
+    fn set_connection(&mut self, conn: Connection) {
+        self.conn = Some(conn);
+    }
+
+    fn set_channel(&mut self, channel: Channel) {
+        self.channel = Some(channel)
+    }
+}
+
+impl FRAcceptedConsumer {
+    pub fn new(db: Database, authifier_db: authifier::Database) -> FRAcceptedConsumer {
+        FRAcceptedConsumer {
+            db,
+            authifier_db,
+            conn: None,
+            channel: None,
+        }
+    }
+}
+
+#[allow(unused_variables)]
+#[async_trait]
+impl AsyncConsumer for FRAcceptedConsumer {
+    /// This consumer handles delegating messages into their respective platform queues.
+    async fn consume(
+        &mut self,
+        channel: &Channel,
+        deliver: Deliver,
+        basic_properties: BasicProperties,
+        content: Vec<u8>,
+    ) {
+        let content = String::from_utf8(content).unwrap();
+        let payload: FRAcceptedPayload = serde_json::from_str(content.as_str()).unwrap();
+
+        debug!("Received FR accept event");
+
+        if let Ok(sessions) = self.authifier_db.find_sessions(&payload.user).await {
+            let config = revolt_config::config().await;
+            for session in sessions {
+                if let Some(sub) = session.subscription {
+                    let mut sendable = PayloadToService {
+                        notification: PayloadKind::FRAccepted(payload.clone()),
+                        token: sub.auth,
+                        user_id: session.user_id,
+                        session_id: session.id,
+                        extras: HashMap::new(),
+                    };
+
+                    let args: BasicPublishArguments;
+
+                    if sub.endpoint == "apn" {
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.apn.queue.as_str(),
+                        )
+                        .finish();
+                    } else if sub.endpoint == "fcm" {
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.fcm.queue.as_str(),
+                        )
+                        .finish();
+                    } else {
+                        // web push (vapid)
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.vapid.queue.as_str(),
+                        )
+                        .finish();
+                        sendable.extras.insert("p265dh".to_string(), sub.p256dh);
+                        sendable
+                            .extras
+                            .insert("endpoint".to_string(), sub.endpoint.clone());
+                    }
+
+                    let payload = serde_json::to_string(&sendable).unwrap();
+
+                    publish_message(self, payload.into(), args).await;
+                }
+            }
+        }
+    }
+}

crates/daemons/pushd/src/consumers/inbound/fr_received.rs

@@ -0,0 +1,121 @@
+use std::collections::HashMap;
+
+use crate::consumers::inbound::internal::*;
+use amqprs::{
+    channel::{BasicPublishArguments, Channel},
+    connection::Connection,
+    consumer::AsyncConsumer,
+    BasicProperties, Deliver,
+};
+use async_trait::async_trait;
+use log::debug;
+use revolt_database::{events::rabbit::*, Database};
+
+pub struct FRReceivedConsumer {
+    #[allow(dead_code)]
+    db: Database,
+    authifier_db: authifier::Database,
+    conn: Option<Connection>,
+    channel: Option<Channel>,
+}
+
+impl Channeled for FRReceivedConsumer {
+    fn get_connection(&self) -> Option<&Connection> {
+        if self.conn.is_none() {
+            None
+        } else {
+            Some(self.conn.as_ref().unwrap())
+        }
+    }
+
+    fn get_channel(&self) -> Option<&Channel> {
+        if self.channel.is_none() {
+            None
+        } else {
+            Some(self.channel.as_ref().unwrap())
+        }
+    }
+
+    fn set_connection(&mut self, conn: Connection) {
+        self.conn = Some(conn);
+    }
+
+    fn set_channel(&mut self, channel: Channel) {
+        self.channel = Some(channel)
+    }
+}
+
+impl FRReceivedConsumer {
+    pub fn new(db: Database, authifier_db: authifier::Database) -> FRReceivedConsumer {
+        FRReceivedConsumer {
+            db,
+            authifier_db,
+            conn: None,
+            channel: None,
+        }
+    }
+}
+
+#[allow(unused_variables)]
+#[async_trait]
+impl AsyncConsumer for FRReceivedConsumer {
+    /// This consumer handles delegating messages into their respective platform queues.
+    async fn consume(
+        &mut self,
+        channel: &Channel,
+        deliver: Deliver,
+        basic_properties: BasicProperties,
+        content: Vec<u8>,
+    ) {
+        let content = String::from_utf8(content).unwrap();
+        let payload: FRReceivedPayload = serde_json::from_str(content.as_str()).unwrap();
+
+        debug!("Received FR received event");
+
+        if let Ok(sessions) = self.authifier_db.find_sessions(&payload.user).await {
+            let config = revolt_config::config().await;
+            for session in sessions {
+                if let Some(sub) = session.subscription {
+                    let mut sendable = PayloadToService {
+                        notification: PayloadKind::FRReceived(payload.clone()),
+                        token: sub.auth,
+                        user_id: session.user_id,
+                        session_id: session.id,
+                        extras: HashMap::new(),
+                    };
+
+                    let args: BasicPublishArguments;
+
+                    if sub.endpoint == "apn" {
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.apn.queue.as_str(),
+                        )
+                        .finish();
+                    } else if sub.endpoint == "fcm" {
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.fcm.queue.as_str(),
+                        )
+                        .finish();
+                    } else {
+                        // web push (vapid)
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.vapid.queue.as_str(),
+                        )
+                        .finish();
+                        sendable.extras.insert("p265dh".to_string(), sub.p256dh);
+                        sendable
+                            .extras
+                            .insert("endpoint".to_string(), sub.endpoint.clone());
+                    }
+
+                    let payload = serde_json::to_string(&sendable).unwrap();
+
+                    publish_message(self, payload.into(), args).await;
+                }
+            }
+        }
+    }
+}

crates/daemons/pushd/src/consumers/inbound/generic.rs

@@ -0,0 +1,127 @@
+use std::collections::HashMap;
+
+use crate::consumers::inbound::internal::*;
+use amqprs::{
+    channel::{BasicPublishArguments, Channel},
+    connection::Connection,
+    consumer::AsyncConsumer,
+    BasicProperties, Deliver,
+};
+use async_trait::async_trait;
+use log::debug;
+use revolt_database::{events::rabbit::*, Database};
+
+pub struct GenericConsumer {
+    #[allow(dead_code)]
+    db: Database,
+    authifier_db: authifier::Database,
+    conn: Option<Connection>,
+    channel: Option<Channel>,
+}
+
+impl Channeled for GenericConsumer {
+    fn get_connection(&self) -> Option<&Connection> {
+        if self.conn.is_none() {
+            None
+        } else {
+            Some(self.conn.as_ref().unwrap())
+        }
+    }
+
+    fn get_channel(&self) -> Option<&Channel> {
+        if self.channel.is_none() {
+            None
+        } else {
+            Some(self.channel.as_ref().unwrap())
+        }
+    }
+
+    fn set_connection(&mut self, conn: Connection) {
+        self.conn = Some(conn);
+    }
+
+    fn set_channel(&mut self, channel: Channel) {
+        self.channel = Some(channel)
+    }
+}
+
+impl GenericConsumer {
+    pub fn new(db: Database, authifier_db: authifier::Database) -> GenericConsumer {
+        GenericConsumer {
+            db,
+            authifier_db,
+            conn: None,
+            channel: None,
+        }
+    }
+}
+
+#[allow(unused_variables)]
+#[async_trait]
+impl AsyncConsumer for GenericConsumer {
+    /// This consumer handles delegating messages into their respective platform queues.
+    async fn consume(
+        &mut self,
+        channel: &Channel,
+        deliver: Deliver,
+        basic_properties: BasicProperties,
+        content: Vec<u8>,
+    ) {
+        let content = String::from_utf8(content).unwrap();
+        let payload: MessageSentPayload = serde_json::from_str(content.as_str()).unwrap();
+
+        debug!("Received message event on origin");
+
+        if let Ok(sessions) = self
+            .authifier_db
+            .find_sessions_with_subscription(&payload.users)
+            .await
+        {
+            let config = revolt_config::config().await;
+            for session in sessions {
+                if let Some(sub) = session.subscription {
+                    let mut sendable = PayloadToService {
+                        notification: PayloadKind::MessageNotification(
+                            payload.notification.clone(),
+                        ),
+                        token: sub.auth,
+                        user_id: session.user_id,
+                        session_id: session.id,
+                        extras: HashMap::new(),
+                    };
+
+                    let args: BasicPublishArguments;
+
+                    if sub.endpoint == "apn" {
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.apn.queue.as_str(),
+                        )
+                        .finish();
+                    } else if sub.endpoint == "fcm" {
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.fcm.queue.as_str(),
+                        )
+                        .finish();
+                    } else {
+                        // web push (vapid)
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.vapid.queue.as_str(),
+                        )
+                        .finish();
+                        sendable.extras.insert("p265dh".to_string(), sub.p256dh);
+                        sendable
+                            .extras
+                            .insert("endpoint".to_string(), sub.endpoint.clone());
+                    }
+
+                    let payload = serde_json::to_string(&sendable).unwrap();
+
+                    publish_message(self, payload.into(), args).await;
+                }
+            }
+        }
+    }
+}

crates/daemons/pushd/src/consumers/inbound/internal.rs

@@ -0,0 +1,53 @@
+use amqprs::{
+    channel::{BasicPublishArguments, Channel},
+    connection::{Connection, OpenConnectionArguments},
+    BasicProperties,
+};
+use log::{debug, warn};
+
+pub(crate) trait Channeled {
+    #[allow(unused)]
+    fn get_connection(&self) -> Option<&Connection>;
+    fn get_channel(&self) -> Option<&Channel>;
+    fn set_connection(&mut self, conn: Connection);
+    fn set_channel(&mut self, channel: Channel);
+}
+
+pub(crate) async fn make_channel<T: Channeled>(consumer: &mut T) {
+    let config = revolt_config::config().await;
+
+    let args = OpenConnectionArguments::new(
+        &config.rabbit.host,
+        config.rabbit.port,
+        &config.rabbit.username,
+        &config.rabbit.password,
+    );
+    let conn = amqprs::connection::Connection::open(&args).await.unwrap();
+
+    let channel = conn.open_channel(None).await.unwrap();
+
+    consumer.set_connection(conn);
+    consumer.set_channel(channel);
+}
+
+pub(crate) async fn publish_message<T: Channeled>(
+    consumer: &mut T,
+    payload: Vec<u8>,
+    args: BasicPublishArguments,
+) {
+    let routing_key = &args.routing_key.clone();
+    let mut channel = consumer.get_channel();
+    if channel.is_none() {
+        make_channel(consumer).await;
+        channel = consumer.get_channel();
+    }
+
+    if let Some(chnl) = channel {
+        chnl.basic_publish(BasicProperties::default(), payload.clone(), args.clone())
+            .await
+            .unwrap();
+        debug!("Sent message to queue for target {}", routing_key);
+    } else {
+        warn!("Failed to unwrap channel (including attempt to make a channel)!")
+    }
+}

crates/daemons/pushd/src/consumers/inbound/message.rs

@@ -0,0 +1,127 @@
+use std::collections::HashMap;
+
+use crate::consumers::inbound::internal::*;
+use amqprs::{
+    channel::{BasicPublishArguments, Channel},
+    connection::Connection,
+    consumer::AsyncConsumer,
+    BasicProperties, Deliver,
+};
+use async_trait::async_trait;
+use log::debug;
+use revolt_database::{events::rabbit::*, Database};
+
+pub struct MessageConsumer {
+    #[allow(dead_code)]
+    db: Database,
+    authifier_db: authifier::Database,
+    conn: Option<Connection>,
+    channel: Option<Channel>,
+}
+
+impl Channeled for MessageConsumer {
+    fn get_connection(&self) -> Option<&Connection> {
+        if self.conn.is_none() {
+            None
+        } else {
+            Some(self.conn.as_ref().unwrap())
+        }
+    }
+
+    fn get_channel(&self) -> Option<&Channel> {
+        if self.channel.is_none() {
+            None
+        } else {
+            Some(self.channel.as_ref().unwrap())
+        }
+    }
+
+    fn set_connection(&mut self, conn: Connection) {
+        self.conn = Some(conn);
+    }
+
+    fn set_channel(&mut self, channel: Channel) {
+        self.channel = Some(channel)
+    }
+}
+
+impl MessageConsumer {
+    pub fn new(db: Database, authifier_db: authifier::Database) -> MessageConsumer {
+        MessageConsumer {
+            db,
+            authifier_db,
+            conn: None,
+            channel: None,
+        }
+    }
+}
+
+#[allow(unused_variables)]
+#[async_trait]
+impl AsyncConsumer for MessageConsumer {
+    /// This consumer handles delegating messages into their respective platform queues.
+    async fn consume(
+        &mut self,
+        channel: &Channel,
+        deliver: Deliver,
+        basic_properties: BasicProperties,
+        content: Vec<u8>,
+    ) {
+        let content = String::from_utf8(content).unwrap();
+        let payload: MessageSentPayload = serde_json::from_str(content.as_str()).unwrap();
+
+        debug!("Received message event on origin");
+
+        if let Ok(sessions) = self
+            .authifier_db
+            .find_sessions_with_subscription(&payload.users)
+            .await
+        {
+            let config = revolt_config::config().await;
+            for session in sessions {
+                if let Some(sub) = session.subscription {
+                    let mut sendable = PayloadToService {
+                        notification: PayloadKind::MessageNotification(
+                            payload.notification.clone(),
+                        ),
+                        token: sub.auth,
+                        user_id: session.user_id,
+                        session_id: session.id,
+                        extras: HashMap::new(),
+                    };
+
+                    let args: BasicPublishArguments;
+
+                    if sub.endpoint == "apn" {
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.apn.queue.as_str(),
+                        )
+                        .finish();
+                    } else if sub.endpoint == "fcm" {
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.fcm.queue.as_str(),
+                        )
+                        .finish();
+                    } else {
+                        // web push (vapid)
+                        args = BasicPublishArguments::new(
+                            config.pushd.exchange.as_str(),
+                            config.pushd.vapid.queue.as_str(),
+                        )
+                        .finish();
+                        sendable.extras.insert("p265dh".to_string(), sub.p256dh);
+                        sendable
+                            .extras
+                            .insert("endpoint".to_string(), sub.endpoint.clone());
+                    }
+
+                    let payload = serde_json::to_string(&sendable).unwrap();
+
+                    publish_message(self, payload.into(), args).await;
+                }
+            }
+        }
+    }
+}

crates/daemons/pushd/src/consumers/inbound/mod.rs

@@ -0,0 +1,6 @@
+pub mod ack;
+pub mod fr_accepted;
+pub mod fr_received;
+pub mod generic;
+mod internal;
+pub mod message;

crates/daemons/pushd/src/consumers/mod.rs

@@ -0,0 +1,2 @@
+pub mod inbound;
+pub mod outbound;

crates/daemons/pushd/src/consumers/outbound/apn.rs

@@ -0,0 +1,338 @@
+use std::{borrow::Cow, collections::BTreeMap, io::Cursor};
+
+use amqprs::{channel::Channel as AmqpChannel, consumer::AsyncConsumer, BasicProperties, Deliver};
+use async_trait::async_trait;
+use base64::{
+    engine::{self},
+    Engine as _,
+};
+use revolt_a2::{
+    request::{
+        notification::{DefaultAlert, NotificationOptions},
+        payload::{APSAlert, APSSound, Payload, PayloadLike, APS},
+    },
+    Client, ClientConfig, Endpoint, Error, ErrorBody, ErrorReason, Priority, PushType, Response,
+};
+use revolt_database::{events::rabbit::*, Database};
+use revolt_models::v0::{Channel, Message, PushNotification};
+use serde::Serialize;
+
+// region: payload
+
+#[derive(Serialize, Debug)]
+struct MessagePayload<'a> {
+    aps: APS<'a>,
+    #[serde(skip_serializing)]
+    options: NotificationOptions<'a>,
+    #[serde(skip_serializing)]
+    device_token: &'a str,
+
+    message: &'a Message,
+    url: &'a str,
+    #[serde(rename = "camelCase")]
+    author_avatar: &'a str,
+    #[serde(rename = "camelCase")]
+    author_display_name: &'a str,
+    #[serde(rename = "camelCase")]
+    channel_name: &'a str,
+}
+
+impl<'a> PayloadLike for MessagePayload<'a> {
+    fn get_device_token(&self) -> &'a str {
+        self.device_token
+    }
+    fn get_options(&self) -> &NotificationOptions {
+        &self.options
+    }
+}
+
+// region: consumer
+
+pub struct ApnsOutboundConsumer {
+    #[allow(dead_code)]
+    db: Database,
+    client: Client,
+}
+
+impl ApnsOutboundConsumer {
+    fn format_title(&self, notification: &PushNotification) -> String {
+        // ideally this changes depending on context
+        // in a server, it would look like "Sendername, #channelname in servername"
+        // in a group, it would look like "Sendername in groupname"
+        // in a dm it should just be "Sendername".
+        // not sure how feasible all those are given the PushNotification object as it currently stands.
+
+        match &notification.channel {
+            Channel::DirectMessage { .. } => notification.author.clone(),
+            Channel::Group { name, .. } => format!("{}, #{}", notification.author, name),
+            Channel::TextChannel { name, .. } | Channel::VoiceChannel { name, .. } => {
+                format!("{} in #{}", notification.author, name)
+            }
+            _ => "Unknown".to_string(),
+        }
+    }
+
+    async fn get_badge_count(&self, user: &str) -> Option<u32> {
+        if let Ok(unreads) = self.db.fetch_unread_mentions(user).await {
+            let mut mention_count = 0;
+            for channel in unreads {
+                if let Some(mentions) = channel.mentions {
+                    mention_count += mentions.len() as u32
+                }
+            }
+
+            println!("Got badge count for APN: {}", mention_count);
+
+            return Some(mention_count);
+        }
+        None
+    }
+}
+
+impl ApnsOutboundConsumer {
+    pub async fn new(db: Database) -> Result<ApnsOutboundConsumer, &'static str> {
+        let config = revolt_config::config().await;
+
+        if config.pushd.apn.pkcs8.is_empty()
+            || config.pushd.apn.key_id.is_empty()
+            || config.pushd.apn.team_id.is_empty()
+        {
+            return Err("Missing APN keys.");
+        }
+
+        let endpoint = if config.pushd.apn.sandbox {
+            Endpoint::Sandbox
+        } else {
+            Endpoint::Production
+        };
+
+        let pkcs8 = engine::general_purpose::STANDARD
+            .decode(config.pushd.apn.pkcs8.clone())
+            .expect("valid `pcks8`");
+
+        let client_config = ClientConfig::new(endpoint);
+
+        let client = Client::token(
+            &mut Cursor::new(pkcs8),
+            config.pushd.apn.key_id.clone(),
+            config.pushd.apn.team_id.clone(),
+            client_config,
+        )
+        .expect("could not create APN client");
+
+        Ok(ApnsOutboundConsumer { db, client })
+    }
+}
+
+#[allow(unused_variables)]
+#[async_trait]
+impl AsyncConsumer for ApnsOutboundConsumer {
+    async fn consume(
+        &mut self,
+        channel: &AmqpChannel,
+        deliver: Deliver,
+        basic_properties: BasicProperties,
+        content: Vec<u8>,
+    ) {
+        let content = String::from_utf8(content).unwrap();
+        let payload: PayloadToService = serde_json::from_str(content.as_str()).unwrap();
+
+        let payload_options = NotificationOptions {
+            apns_id: None,
+            apns_push_type: Some(PushType::Alert),
+            apns_expiration: None,
+            apns_priority: Some(Priority::High),
+            apns_topic: Some("chat.revolt.app"),
+            apns_collapse_id: None,
+        };
+
+        let resp: Result<Response, Error>;
+
+        match payload.notification {
+            PayloadKind::FRReceived(alert) => {
+                let loc_args = vec![Cow::from(
+                    alert
+                        .from_user
+                        .display_name
+                        .or(Some(format!(
+                            "{}#{}",
+                            alert.from_user.username, alert.from_user.discriminator
+                        )))
+                        .clone()
+                        .unwrap(),
+                )];
+
+                let apn_payload = Payload {
+                    aps: APS {
+                        alert: Some(APSAlert::Default(DefaultAlert {
+                            title: None,
+                            subtitle: None,
+                            body: None,
+                            title_loc_key: None,
+                            title_loc_args: None,
+                            action_loc_key: None,
+                            loc_key: Some("push.fr.received"),
+                            loc_args: Some(loc_args),
+                            launch_image: None,
+                        })),
+                        badge: self.get_badge_count(&payload.user_id).await,
+                        sound: Some(APSSound::Sound("default")),
+                        thread_id: None,
+                        content_available: None,
+                        category: None,
+                        mutable_content: Some(1),
+                        url_args: None,
+                    },
+                    device_token: &payload.token,
+                    options: payload_options.clone(),
+                    data: BTreeMap::new(),
+                };
+
+                resp = self.client.send(apn_payload).await;
+            }
+
+            PayloadKind::FRAccepted(alert) => {
+                let loc_args = vec![Cow::from(
+                    alert
+                        .accepted_user
+                        .display_name
+                        .or(Some(format!(
+                            "{}#{}",
+                            alert.accepted_user.username, alert.accepted_user.discriminator
+                        )))
+                        .clone()
+                        .unwrap(),
+                )];
+
+                let apn_payload = Payload {
+                    aps: APS {
+                        alert: Some(APSAlert::Default(DefaultAlert {
+                            title: None,
+                            subtitle: None,
+                            body: None,
+                            title_loc_key: None,
+                            title_loc_args: None,
+                            action_loc_key: None,
+                            loc_key: Some("push.fr.accepted"),
+                            loc_args: Some(loc_args),
+                            launch_image: None,
+                        })),
+                        badge: self.get_badge_count(&payload.user_id).await,
+                        sound: Some(APSSound::Sound("default")),
+                        thread_id: None,
+                        content_available: None,
+                        category: None,
+                        mutable_content: Some(1),
+                        url_args: None,
+                    },
+                    device_token: &payload.token,
+                    options: payload_options.clone(),
+                    data: BTreeMap::new(),
+                };
+
+                resp = self.client.send(apn_payload).await;
+            }
+            PayloadKind::Generic(alert) => {
+                let apn_payload = Payload {
+                    aps: APS {
+                        alert: Some(APSAlert::Default(DefaultAlert {
+                            title: Some(&alert.title),
+                            subtitle: None,
+                            body: Some(&alert.body),
+                            title_loc_key: None,
+                            title_loc_args: None,
+                            action_loc_key: None,
+                            loc_key: None,
+                            loc_args: None,
+                            launch_image: None,
+                        })),
+                        badge: self.get_badge_count(&payload.user_id).await,
+                        sound: Some(APSSound::Sound("default")),
+                        thread_id: None,
+                        content_available: None,
+                        category: None,
+                        mutable_content: Some(1),
+                        url_args: None,
+                    },
+                    device_token: &payload.token,
+                    options: payload_options.clone(),
+                    data: BTreeMap::new(),
+                };
+
+                resp = self.client.send(apn_payload).await;
+            }
+
+            PayloadKind::MessageNotification(alert) => {
+                let title = self.format_title(&alert);
+                let apn_payload = MessagePayload {
+                    aps: APS {
+                        alert: Some(APSAlert::Default(DefaultAlert {
+                            title: Some(&title),
+                            subtitle: None,
+                            body: Some(&alert.body),
+                            title_loc_key: None,
+                            title_loc_args: None,
+                            action_loc_key: None,
+                            loc_key: None,
+                            loc_args: None,
+                            launch_image: None,
+                        })),
+                        badge: self.get_badge_count(&payload.user_id).await,
+                        sound: Some(APSSound::Sound("default")),
+                        thread_id: Some(alert.channel.id()),
+                        content_available: None,
+                        category: None,
+                        mutable_content: Some(1),
+                        url_args: None,
+                    },
+                    device_token: &payload.token,
+                    options: payload_options.clone(),
+                    message: &alert.message,
+                    url: &alert.url,
+                    author_avatar: &alert.icon,
+                    author_display_name: &alert.author,
+                    channel_name: alert.channel.name().unwrap_or(&title),
+                };
+
+                resp = self.client.send(apn_payload).await;
+            }
+            PayloadKind::BadgeUpdate(badge) => {
+                let apn_payload = Payload {
+                    aps: APS {
+                        badge: Some(badge as u32),
+                        ..Default::default()
+                    },
+                    device_token: &payload.token,
+                    options: payload_options.clone(),
+                    data: BTreeMap::new(),
+                };
+
+                resp = self.client.send(apn_payload).await;
+            }
+        }
+
+        if let Err(err) = resp {
+            match err {
+                Error::ResponseError(Response {
+                    error:
+                        Some(ErrorBody {
+                            reason: ErrorReason::BadDeviceToken | ErrorReason::Unregistered,
+                            ..
+                        }),
+                    ..
+                }) => {
+                    if let Err(err) = self
+                        .db
+                        .remove_push_subscription_by_session_id(&payload.session_id)
+                        .await
+                    {
+                        revolt_config::capture_error(&err);
+                    }
+                }
+                err => {
+                    revolt_config::capture_error(&err);
+                }
+            }
+        }
+    }
+}

crates/daemons/pushd/src/consumers/outbound/fcm.rs

@@ -0,0 +1,199 @@
+use std::{collections::HashMap, time::Duration};
+
+use amqprs::{channel::Channel as AmqpChannel, consumer::AsyncConsumer, BasicProperties, Deliver};
+
+use async_trait::async_trait;
+use fcm_v1::{
+    android::AndroidConfig,
+    auth::{Authenticator, ServiceAccountKey},
+    message::{Message, Notification},
+    Client, Error as FcmError,
+};
+use revolt_database::{events::rabbit::*, Database};
+use revolt_models::v0::{Channel, PushNotification};
+use serde_json::Value;
+
+pub struct FcmOutboundConsumer {
+    db: Database,
+    client: Client,
+}
+
+impl FcmOutboundConsumer {
+    fn format_title(&self, notification: &PushNotification) -> String {
+        // ideally this changes depending on context
+        // in a server, it would look like "Sendername, #channelname in servername"
+        // in a group, it would look like "Sendername in groupname"
+        // in a dm it should just be "Sendername".
+        // not sure how feasible all those are given the PushNotification object as it currently stands.
+
+        match &notification.channel {
+            Channel::DirectMessage { .. } => notification.author.clone(),
+            Channel::Group { name, .. } => format!("{}, #{}", notification.author, name),
+            Channel::TextChannel { name, .. } | Channel::VoiceChannel { name, .. } => {
+                format!("{} in #{}", notification.author, name)
+            }
+            _ => "Unknown".to_string(),
+        }
+    }
+}
+
+impl FcmOutboundConsumer {
+    pub async fn new(db: Database) -> Result<FcmOutboundConsumer, &'static str> {
+        let config = revolt_config::config().await;
+
+        Ok(FcmOutboundConsumer {
+            db,
+            client: Client::new(
+                Authenticator::service_account::<&str>(ServiceAccountKey {
+                    key_type: Some(config.pushd.fcm.key_type),
+                    project_id: Some(config.pushd.fcm.project_id.clone()),
+                    private_key_id: Some(config.pushd.fcm.private_key_id),
+                    private_key: config.pushd.fcm.private_key,
+                    client_email: config.pushd.fcm.client_email,
+                    client_id: Some(config.pushd.fcm.client_id),
+                    auth_uri: Some(config.pushd.fcm.auth_uri),
+                    token_uri: config.pushd.fcm.token_uri,
+                    auth_provider_x509_cert_url: Some(config.pushd.fcm.auth_provider_x509_cert_url),
+                    client_x509_cert_url: Some(config.pushd.fcm.client_x509_cert_url),
+                })
+                .await
+                .unwrap(),
+                config.pushd.fcm.project_id,
+                false,
+                Duration::from_secs(5),
+            ),
+        })
+    }
+}
+
+#[allow(unused_variables)]
+#[async_trait]
+impl AsyncConsumer for FcmOutboundConsumer {
+    async fn consume(
+        &mut self,
+        channel: &AmqpChannel,
+        deliver: Deliver,
+        basic_properties: BasicProperties,
+        content: Vec<u8>,
+    ) {
+        let content = String::from_utf8(content).unwrap();
+        let payload: PayloadToService = serde_json::from_str(content.as_str()).unwrap();
+
+        let config = revolt_config::config().await;
+
+        #[allow(clippy::needless_late_init)]
+        let resp: Result<Message, FcmError>;
+
+        match payload.notification {
+            PayloadKind::FRReceived(alert) => {
+                let name = alert
+                    .from_user
+                    .display_name
+                    .or(Some(format!(
+                        "{}#{}",
+                        alert.from_user.username, alert.from_user.discriminator
+                    )))
+                    .clone()
+                    .unwrap();
+
+                let mut data = HashMap::new();
+                data.insert(
+                    "type".to_string(),
+                    Value::String("push.fr.receive".to_string()),
+                );
+                data.insert("id".to_string(), Value::String(alert.from_user.id));
+                data.insert("username".to_string(), Value::String(name));
+
+                let msg = Message {
+                    token: Some(payload.token),
+                    data: Some(data),
+                    ..Default::default()
+                };
+
+                resp = self.client.send(&msg).await;
+            }
+
+            PayloadKind::FRAccepted(alert) => {
+                let name = alert
+                    .accepted_user
+                    .display_name
+                    .or(Some(format!(
+                        "{}#{}",
+                        alert.accepted_user.username, alert.accepted_user.discriminator
+                    )))
+                    .clone()
+                    .unwrap();
+
+                let mut data: HashMap<String, Value> = HashMap::new();
+                data.insert(
+                    "type".to_string(),
+                    Value::String("push.fr.accept".to_string()),
+                );
+                data.insert("id".to_string(), Value::String(alert.accepted_user.id));
+                data.insert("username".to_string(), Value::String(name));
+
+                let msg = Message {
+                    token: Some(payload.token),
+                    data: Some(data),
+                    ..Default::default()
+                };
+
+                resp = self.client.send(&msg).await;
+            }
+            PayloadKind::Generic(alert) => {
+                let msg = Message {
+                    token: Some(payload.token),
+                    notification: Some(Notification {
+                        title: Some(alert.title),
+                        body: Some(alert.body),
+                        image: alert.icon,
+                    }),
+                    ..Default::default()
+                };
+
+                resp = self.client.send(&msg).await;
+            }
+
+            PayloadKind::MessageNotification(alert) => {
+                let title = self.format_title(&alert);
+
+                let msg = Message {
+                    token: Some(payload.token),
+                    notification: Some(Notification {
+                        title: Some(title),
+                        body: Some(alert.body),
+                        image: Some(alert.icon),
+                    }),
+                    android: Some(AndroidConfig {
+                        collapse_key: Some(alert.tag),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                };
+
+                resp = self.client.send(&msg).await;
+            }
+
+            PayloadKind::BadgeUpdate(_) => {
+                panic!("FCM cannot handle badge updates, and they should not be sent here.")
+            }
+        }
+
+        if let Err(err) = resp {
+            match err {
+                FcmError::Auth => {
+                    if let Err(err) = self
+                        .db
+                        .remove_push_subscription_by_session_id(&payload.session_id)
+                        .await
+                    {
+                        revolt_config::capture_error(&err);
+                    }
+                }
+                err => {
+                    revolt_config::capture_error(&err);
+                }
+            }
+        }
+    }
+}

crates/daemons/pushd/src/consumers/outbound/mod.rs

@@ -0,0 +1,3 @@
+pub mod apn;
+pub mod fcm;
+pub mod vapid;

crates/daemons/pushd/src/consumers/outbound/vapid.rs

@@ -0,0 +1,149 @@
+use std::collections::HashMap;
+
+use amqprs::{channel::Channel as AmqpChannel, consumer::AsyncConsumer, BasicProperties, Deliver};
+
+use async_trait::async_trait;
+use base64::{
+    engine::{self},
+    Engine as _,
+};
+use revolt_database::{events::rabbit::*, Database};
+// use revolt_models::v0::{Channel, PushNotification};
+use web_push::{
+    ContentEncoding, IsahcWebPushClient, SubscriptionInfo, SubscriptionKeys, VapidSignatureBuilder,
+    WebPushClient, WebPushError, WebPushMessageBuilder,
+};
+
+pub struct VapidOutboundConsumer {
+    db: Database,
+    client: IsahcWebPushClient,
+    pkey: Vec<u8>,
+}
+
+impl VapidOutboundConsumer {
+    pub async fn new(db: Database) -> Result<VapidOutboundConsumer, &'static str> {
+        let config = revolt_config::config().await;
+
+        if config.pushd.vapid.private_key.is_empty() | config.pushd.vapid.public_key.is_empty() {
+            return Err("No Vapid keys present");
+        }
+
+        let web_push_private_key = engine::general_purpose::URL_SAFE_NO_PAD
+            .decode(config.pushd.vapid.private_key)
+            .expect("valid `VAPID_PRIVATE_KEY`");
+
+        Ok(VapidOutboundConsumer {
+            db,
+            client: IsahcWebPushClient::new().unwrap(),
+            pkey: web_push_private_key,
+        })
+    }
+}
+
+#[allow(unused_variables)]
+#[async_trait]
+impl AsyncConsumer for VapidOutboundConsumer {
+    async fn consume(
+        &mut self,
+        channel: &AmqpChannel,
+        deliver: Deliver,
+        basic_properties: BasicProperties,
+        content: Vec<u8>,
+    ) {
+        let content = String::from_utf8(content).unwrap();
+        let payload: PayloadToService = serde_json::from_str(content.as_str()).unwrap();
+
+        let config = revolt_config::config().await;
+
+        let subscription = SubscriptionInfo {
+            endpoint: payload.extras.get("endpoint").unwrap().clone(),
+            keys: SubscriptionKeys {
+                auth: payload.token,
+                p256dh: payload.extras.get("p256dh").unwrap().clone(),
+            },
+        };
+
+        #[allow(clippy::needless_late_init)]
+        let payload_body: String;
+
+        match payload.notification {
+            PayloadKind::FRReceived(alert) => {
+                let name = alert
+                    .from_user
+                    .display_name
+                    .or(Some(format!(
+                        "{}#{}",
+                        alert.from_user.username, alert.from_user.discriminator
+                    )))
+                    .clone()
+                    .unwrap();
+
+                let mut body = HashMap::new();
+                body.insert("body", format!("{} sent you a friend request", name));
+
+                payload_body = serde_json::to_string(&body).unwrap();
+            }
+            PayloadKind::FRAccepted(alert) => {
+                let name = alert
+                    .accepted_user
+                    .display_name
+                    .or(Some(format!(
+                        "{}#{}",
+                        alert.accepted_user.username, alert.accepted_user.discriminator
+                    )))
+                    .clone()
+                    .unwrap();
+
+                let mut body = HashMap::new();
+                body.insert("body", format!("{} accepted your friend request", name));
+
+                payload_body = serde_json::to_string(&body).unwrap();
+            }
+            PayloadKind::Generic(alert) => {
+                payload_body = serde_json::to_string(&alert).unwrap();
+            }
+            PayloadKind::MessageNotification(alert) => {
+                payload_body = serde_json::to_string(&alert).unwrap();
+            }
+            PayloadKind::BadgeUpdate(_) => {
+                panic!("Vapid cannot handle badge updates, and they should not be sent here.")
+            }
+        }
+
+        match VapidSignatureBuilder::from_pem(std::io::Cursor::new(&self.pkey), &subscription) {
+            Ok(sig_builder) => match sig_builder.build() {
+                Ok(signature) => {
+                    let mut builder = WebPushMessageBuilder::new(&subscription);
+                    builder.set_vapid_signature(signature);
+
+                    builder.set_payload(ContentEncoding::AesGcm, payload_body.as_bytes());
+
+                    match builder.build() {
+                        Ok(msg) => {
+                            if let Err(err) = self.client.send(msg).await {
+                                if err == WebPushError::Unauthorized {
+                                    if let Err(err) = self
+                                        .db
+                                        .remove_push_subscription_by_session_id(&payload.session_id)
+                                        .await
+                                    {
+                                        revolt_config::capture_error(&err);
+                                    }
+                                }
+                            }
+                        }
+                        Err(err) => {
+                            revolt_config::capture_error(&err);
+                        }
+                    }
+                }
+                Err(err) => {
+                    revolt_config::capture_error(&err);
+                }
+            },
+            Err(err) => {
+                revolt_config::capture_error(&err);
+            }
+        }
+    }
+}

crates/daemons/pushd/src/main.rs

@@ -0,0 +1,233 @@
+use amqprs::{
+    channel::{
+        BasicConsumeArguments, Channel, ExchangeDeclareArguments, QueueBindArguments,
+        QueueDeclareArguments,
+    },
+    connection::{Connection, OpenConnectionArguments},
+    consumer::AsyncConsumer,
+    FieldTable,
+};
+use revolt_config::{config, Settings};
+use tokio::sync::Notify;
+
+mod consumers;
+use consumers::{
+    inbound::{
+        ack::AckConsumer, fr_accepted::FRAcceptedConsumer, fr_received::FRReceivedConsumer,
+        generic::GenericConsumer, message::MessageConsumer,
+    },
+    outbound::{apn::ApnsOutboundConsumer, fcm::FcmOutboundConsumer, vapid::VapidOutboundConsumer},
+};
+
+#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
+async fn main() {
+    let config = config().await;
+
+    // Setup database
+    let db = revolt_database::DatabaseInfo::Auto.connect().await.unwrap();
+    let authifier: authifier::Database;
+
+    if let Some(client) = match &db {
+        revolt_database::Database::Reference(_) => None,
+        revolt_database::Database::MongoDb(mongo) => Some(mongo),
+    } {
+        authifier =
+            authifier::Database::MongoDb(authifier::database::MongoDb(client.database("revolt")));
+    } else {
+        panic!("Mongo is not in use, can't connect via authifier!")
+    }
+
+    let mut connections: Vec<(Channel, Connection)> = Vec::new();
+
+    // An explainer of how this works:
+    // The inbound connections are on separate routing keys, such that they only receive the proper payload
+    // from their respective api (prod or test).
+    // However, the outbound queues that go to the services are routed to receive from both, so that messages
+    // sent from beta are still notified on prod, and vice versa.
+
+    // This'll require some interesting shimming if we need to add more events once this is in prod (different payloads between prod and test),
+    // but that sounds like a problem for future us.
+
+    // inbound: generic
+    connections.push(
+        make_queue_and_consume(
+            &config,
+            &config.pushd.generic_queue,
+            config.pushd.get_generic_routing_key().as_str(),
+            None,
+            GenericConsumer::new(db.clone(), authifier.clone()),
+        )
+        .await,
+    );
+
+    // inbound: messages
+    connections.push(
+        make_queue_and_consume(
+            &config,
+            &config.pushd.message_queue,
+            config.pushd.get_message_routing_key().as_str(),
+            None,
+            MessageConsumer::new(db.clone(), authifier.clone()),
+        )
+        .await,
+    );
+
+    // inbound: FR received
+    connections.push(
+        make_queue_and_consume(
+            &config,
+            &config.pushd.fr_received_queue,
+            config.pushd.get_fr_received_routing_key().as_str(),
+            None,
+            FRReceivedConsumer::new(db.clone(), authifier.clone()),
+        )
+        .await,
+    );
+
+    // inbound: FR accepted
+    connections.push(
+        make_queue_and_consume(
+            &config,
+            &config.pushd.fr_accepted_queue,
+            config.pushd.get_fr_accepted_routing_key().as_str(),
+            None,
+            FRAcceptedConsumer::new(db.clone(), authifier.clone()),
+        )
+        .await,
+    );
+
+    if !config.pushd.apn.pkcs8.is_empty() {
+        connections.push(
+            make_queue_and_consume(
+                &config,
+                &config.pushd.apn.queue,
+                &config.pushd.apn.queue,
+                None,
+                ApnsOutboundConsumer::new(db.clone()).await.unwrap(),
+            )
+            .await,
+        );
+
+        let mut table = FieldTable::new();
+        table.insert("x-message-deduplication".try_into().unwrap(), "true".into());
+
+        connections.push(
+            make_queue_and_consume(
+                &config,
+                &config.pushd.ack_queue,
+                &config.pushd.ack_queue,
+                Some(table),
+                AckConsumer::new(db.clone(), authifier.clone()),
+            )
+            .await,
+        );
+    }
+
+    if !config.pushd.fcm.auth_uri.is_empty() {
+        connections.push(
+            make_queue_and_consume(
+                &config,
+                &config.pushd.fcm.queue,
+                &config.pushd.fcm.queue,
+                None,
+                FcmOutboundConsumer::new(db.clone()).await.unwrap(),
+            )
+            .await,
+        )
+    }
+
+    if !config.pushd.vapid.public_key.is_empty() {
+        connections.push(
+            make_queue_and_consume(
+                &config,
+                &config.pushd.vapid.queue,
+                &config.pushd.vapid.queue,
+                None,
+                VapidOutboundConsumer::new(db.clone()).await.unwrap(),
+            )
+            .await,
+        )
+    }
+
+    let guard = Notify::new();
+    guard.notified().await;
+
+    for (channel, conn) in connections {
+        channel.close().await.expect("Unable to close channel");
+        conn.close().await.expect("Unable to close connection");
+    }
+}
+
+async fn make_queue_and_consume<F>(
+    config: &Settings,
+    queue_name: &str,
+    routing_key: &str,
+    queue_args: Option<FieldTable>,
+    consumer: F,
+) -> (Channel, Connection)
+where
+    F: AsyncConsumer + Send + 'static,
+{
+    let connection = Connection::open(&OpenConnectionArguments::new(
+        &config.rabbit.host,
+        config.rabbit.port,
+        &config.rabbit.username,
+        &config.rabbit.password,
+    ))
+    .await
+    .unwrap();
+
+    let channel = connection.open_channel(None).await.unwrap();
+
+    channel
+        .exchange_declare(
+            ExchangeDeclareArguments::new(&config.pushd.exchange, "direct")
+                .durable(true)
+                .finish(),
+        )
+        .await
+        .expect("Failed to declare pushd exchange");
+
+    let mut queue_name = queue_name.to_string();
+
+    if config.pushd.production {
+        queue_name += "-prd";
+    } else {
+        queue_name += "-tst";
+    }
+
+    let queue_name = queue_name.as_str();
+
+    let mut args = QueueDeclareArguments::new(queue_name);
+    args.durable(true);
+
+    if let Some(arg) = queue_args {
+        args.arguments(arg);
+    }
+
+    let args = args.finish();
+    _ = channel.queue_declare(args).await.unwrap().unwrap();
+
+    channel
+        .queue_bind(QueueBindArguments::new(
+            queue_name,
+            &config.pushd.exchange,
+            routing_key,
+        ))
+        .await
+        .expect(
+            "This probably means the revolt.notifications exchange does not exist in rabbitmq!",
+        );
+
+    let args = BasicConsumeArguments::new(queue_name, "")
+        .manual_ack(false)
+        .finish();
+
+    channel.basic_consume(consumer, args).await.unwrap();
+    log::info!(
+        "Consuming routing key {} as queue {}",
+        routing_key,
+        queue_name
+    );
+    (channel, connection)
+}

crates/delta/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-delta"
-version = "0.7.16"
+version = "0.8.0"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <paulmakles@gmail.com>"]
 edition = "2018"
@@ -16,7 +16,6 @@ redis-kiss = "0.1.4"
 lru = "0.7.0"
 url = "2.2.2"
 log = "0.4.11"
-dotenv = "0.15.0"
 dashmap = "5.2.0"
 linkify = "0.6.0"
 once_cell = "1.17.1"
@@ -53,20 +52,21 @@ async-std = { version = "1.8.0", features = [
 lettre = "0.10.0-alpha.4"
 
 # web
-rocket = { version = "0.5.0-rc.2", default-features = false, features = [
-    "json",
-] }
-rocket_cors = { git = "https://github.com/lawliet89/rocket_cors", rev = "c17e8145baa4790319fdb6a473e465b960f55e7c" }
+rocket = { version = "0.5.1", default-features = false, features = ["json"] }
+rocket_cors = { git = "https://github.com/lawliet89/rocket_cors", rev = "072d90359b23e9b291df6b672c07c93de9c46011" }
 rocket_empty = { version = "0.1.1", features = ["schema"] }
-rocket_authifier = { version = "1.0.8" }
+rocket_authifier = { version = "1.0.9" }
 rocket_prometheus = "0.10.0-rc.3"
 
 # spec generation
 schemars = "0.8.8"
-revolt_rocket_okapi = { version = "0.9.1", features = ["swagger"] }
+revolt_rocket_okapi = { version = "0.10.0", features = ["swagger"] }
+
+# rabbit
+amqprs = { version = "1.7.0" }
 
 # core
-authifier = "1.0.8"
+authifier = "1.0.9"
 revolt-config = { path = "../core/config" }
 revolt-database = { path = "../core/database", features = [
     "rocket-impl",

crates/delta/src/main.rs

@@ -10,12 +10,17 @@ pub mod util;
 
 use revolt_config::config;
 use revolt_database::events::client::EventV1;
+use revolt_database::AMQP;
 use rocket::{Build, Rocket};
 use rocket_cors::{AllowedOrigins, CorsOptions};
 use rocket_prometheus::PrometheusMetrics;
 use std::net::Ipv4Addr;
 use std::str::FromStr;
 
+use amqprs::{
+    channel::ExchangeDeclareArguments,
+    connection::{Connection, OpenConnectionArguments},
+};
 use async_std::channel::unbounded;
 use authifier::AuthifierEvent;
 use rocket::data::ToByteUnit;
@@ -32,7 +37,7 @@ pub async fn web() -> Rocket<Build> {
     db.migrate_database().await.unwrap();
 
     // Setup Authifier event channel
-    let (sender, receiver) = unbounded();
+    let (_, receiver) = unbounded();
 
     // Setup Authifier
     let authifier = db.clone().to_authifier().await;
@@ -53,9 +58,6 @@ pub async fn web() -> Rocket<Build> {
         }
     });
 
-    // Launch background task workers
-    revolt_database::tasks::start_workers(db.clone(), authifier.database.clone());
-
     // Configure CORS
     let cors = CorsOptions {
         allowed_origins: AllowedOrigins::All,
@@ -79,6 +81,31 @@ pub async fn web() -> Rocket<Build> {
     )
     .into();
 
+    // Configure Rabbit
+    let connection = Connection::open(&OpenConnectionArguments::new(
+        &config.rabbit.host,
+        config.rabbit.port,
+        &config.rabbit.username,
+        &config.rabbit.password,
+    ))
+    .await
+    .unwrap();
+    let channel = connection.open_channel(None).await.unwrap();
+
+    channel
+        .exchange_declare(
+            ExchangeDeclareArguments::new(&config.pushd.exchange, "direct")
+                .durable(true)
+                .finish(),
+        )
+        .await
+        .expect("Failed to declare exchange");
+
+    let amqp = AMQP::new(connection, channel);
+
+    // Launch background task workers
+    revolt_database::tasks::start_workers(db.clone(), amqp.clone());
+
     // Configure Rocket
     let rocket = rocket::build();
     let prometheus = PrometheusMetrics::new();
@@ -91,6 +118,7 @@ pub async fn web() -> Rocket<Build> {
         .mount("/swagger/", swagger)
         .manage(authifier)
         .manage(db)
+        .manage(amqp)
         .manage(cors.clone())
         .attach(util::ratelimiter::RatelimitFairing)
         .attach(cors)

crates/delta/src/routes/bots/invite.rs

@@ -1,6 +1,6 @@
 use revolt_database::util::permissions::DatabasePermissionQuery;
-use revolt_database::Member;
 use revolt_database::{util::reference::Reference, Database, User};
+use revolt_database::{Member, AMQP};
 use revolt_models::v0;
 use revolt_permissions::{
     calculate_channel_permissions, calculate_server_permissions, ChannelPermission,
@@ -18,6 +18,7 @@ use rocket_empty::EmptyResponse;
 #[post("/<target>/invite", data = "<dest>")]
 pub async fn invite_bot(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     user: User,
     target: Reference,
     dest: Json<v0::InviteBotDestination>,
@@ -55,7 +56,7 @@ pub async fn invite_bot(
                 .throw_if_lacking_channel_permission(ChannelPermission::InviteOthers)?;
 
             channel
-                .add_user_to_group(db, &bot_user, &user.id)
+                .add_user_to_group(db, amqp, &bot_user, &user.id)
                 .await
                 .map(|_| EmptyResponse)
         }
@@ -93,9 +94,12 @@ mod test {
             .client
             .post(format!("/bots/{}/invite", bot.id))
             .header(ContentType::JSON)
-            .body(json!(v0::InviteBotDestination::Group {
-                group: group.id().to_string()
-            }).to_string())
+            .body(
+                json!(v0::InviteBotDestination::Group {
+                    group: group.id().to_string()
+                })
+                .to_string(),
+            )
             .header(Header::new("x-session-token", session.token.to_string()))
             .dispatch()
             .await;

crates/delta/src/routes/channels/channel_delete.rs

@@ -1,6 +1,6 @@
 use revolt_database::{
     util::{permissions::DatabasePermissionQuery, reference::Reference},
-    Channel, Database, PartialChannel, User,
+    Channel, Database, PartialChannel, User, AMQP,
 };
 use revolt_models::v0;
 use revolt_permissions::{calculate_channel_permissions, ChannelPermission};
@@ -15,6 +15,7 @@ use rocket_empty::EmptyResponse;
 #[delete("/<target>?<options..>")]
 pub async fn delete(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     user: User,
     target: Reference,
     options: v0::OptionsChannelDelete,
@@ -39,7 +40,13 @@ pub async fn delete(
             .await
             .map(|_| EmptyResponse),
         Channel::Group { .. } => channel
-            .remove_user_from_group(db, &user, None, options.leave_silently.unwrap_or_default())
+            .remove_user_from_group(
+                db,
+                amqp,
+                &user,
+                None,
+                options.leave_silently.unwrap_or_default(),
+            )
             .await
             .map(|_| EmptyResponse),
         Channel::TextChannel { .. } | Channel::VoiceChannel { .. } => {

crates/delta/src/routes/channels/channel_edit.rs

@@ -1,6 +1,6 @@
 use revolt_database::{
     util::{permissions::DatabasePermissionQuery, reference::Reference},
-    Channel, Database, File, PartialChannel, SystemMessage, User,
+    Channel, Database, File, PartialChannel, SystemMessage, User, AMQP,
 };
 use revolt_models::v0;
 use revolt_permissions::{calculate_channel_permissions, ChannelPermission};
@@ -15,6 +15,7 @@ use validator::Validate;
 #[patch("/<target>", data = "<data>")]
 pub async fn edit(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     user: User,
     target: Reference,
     data: Json<v0::DataEditChannel>,
@@ -73,7 +74,15 @@ pub async fn edit(
             return Err(create_error!(InvalidOperation));
         }
         .into_message(channel.id().to_string())
-        .send(db, user.as_author_for_system(), None, None, &channel, false)
+        .send(
+            db,
+            Some(amqp),
+            user.as_author_for_system(),
+            None,
+            None,
+            &channel,
+            false,
+        )
         .await
         .ok();
     }
@@ -151,7 +160,15 @@ pub async fn edit(
                         by: user.id.clone(),
                     }
                     .into_message(channel.id().to_string())
-                    .send(db, user.as_author_for_system(), None, None, &channel, false)
+                    .send(
+                        db,
+                        Some(amqp),
+                        user.as_author_for_system(),
+                        None,
+                        None,
+                        &channel,
+                        false,
+                    )
                     .await
                     .ok();
                 }
@@ -161,7 +178,15 @@ pub async fn edit(
                         by: user.id.clone(),
                     }
                     .into_message(channel.id().to_string())
-                    .send(db, user.as_author_for_system(), None, None, &channel, false)
+                    .send(
+                        db,
+                        Some(amqp),
+                        user.as_author_for_system(),
+                        None,
+                        None,
+                        &channel,
+                        false,
+                    )
                     .await
                     .ok();
                 }
@@ -171,7 +196,15 @@ pub async fn edit(
                         by: user.id.clone(),
                     }
                     .into_message(channel.id().to_string())
-                    .send(db, user.as_author_for_system(), None, None, &channel, false)
+                    .send(
+                        db,
+                        Some(amqp),
+                        user.as_author_for_system(),
+                        None,
+                        None,
+                        &channel,
+                        false,
+                    )
                     .await
                     .ok();
                 }

crates/delta/src/routes/channels/group_add_member.rs

@@ -1,6 +1,6 @@
 use revolt_database::{
     util::{permissions::DatabasePermissionQuery, reference::Reference},
-    Channel, Database, User,
+    Channel, Database, User, AMQP,
 };
 use revolt_permissions::{calculate_channel_permissions, ChannelPermission};
 use revolt_result::{create_error, Result};
@@ -15,6 +15,7 @@ use rocket_empty::EmptyResponse;
 #[put("/<group_id>/recipients/<member_id>")]
 pub async fn add_member(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     user: User,
     group_id: Reference,
     member_id: Reference,
@@ -38,7 +39,7 @@ pub async fn add_member(
             }
 
             channel
-                .add_user_to_group(db, &member, &user.id)
+                .add_user_to_group(db, amqp, &member, &user.id)
                 .await
                 .map(|_| EmptyResponse)
         }

crates/delta/src/routes/channels/group_remove_member.rs

@@ -1,4 +1,4 @@
-use revolt_database::{util::reference::Reference, Channel, Database, User};
+use revolt_database::{util::reference::Reference, Channel, Database, User, AMQP};
 use revolt_permissions::ChannelPermission;
 use revolt_result::{create_error, Result};
 
@@ -12,6 +12,7 @@ use rocket_empty::EmptyResponse;
 #[delete("/<target>/recipients/<member>")]
 pub async fn remove_member(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     user: User,
     target: Reference,
     member: Reference,
@@ -42,7 +43,7 @@ pub async fn remove_member(
             }
 
             channel
-                .remove_user_from_group(db, &member, Some(&user.id), false)
+                .remove_user_from_group(db, amqp, &member, Some(&user.id), false)
                 .await
                 .map(|_| EmptyResponse)
         }

crates/delta/src/routes/channels/message_pin.rs

@@ -1,5 +1,6 @@
 use revolt_database::{
-    util::{permissions::DatabasePermissionQuery, reference::Reference}, Database, PartialMessage, SystemMessage, User
+    util::{permissions::DatabasePermissionQuery, reference::Reference},
+    Database, PartialMessage, SystemMessage, User, AMQP,
 };
 use revolt_models::v0::MessageAuthor;
 use revolt_permissions::{calculate_channel_permissions, ChannelPermission};
@@ -14,6 +15,7 @@ use rocket_empty::EmptyResponse;
 #[post("/<target>/messages/<msg>/pin")]
 pub async fn message_pin(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     user: User,
     target: Reference,
     msg: Reference,
@@ -28,30 +30,38 @@ pub async fn message_pin(
     let mut message = msg.as_message_in_channel(db, channel.id()).await?;
 
     if message.pinned.unwrap_or_default() {
-        return Err(create_error!(AlreadyPinned))
+        return Err(create_error!(AlreadyPinned));
     }
 
-    message.update(db, PartialMessage {
-        pinned: Some(true),
-        ..Default::default()
-    }, vec![]).await?;
+    message
+        .update(
+            db,
+            PartialMessage {
+                pinned: Some(true),
+                ..Default::default()
+            },
+            vec![],
+        )
+        .await?;
 
     SystemMessage::MessagePinned {
         id: message.id.clone(),
-        by: user.id.clone()
+        by: user.id.clone(),
     }
     .into_message(channel.id().to_string())
     .send(
         db,
+        Some(amqp),
         MessageAuthor::System {
             username: &user.username,
-            avatar: user.avatar.as_ref().map(|file| file.id.as_ref())
+            avatar: user.avatar.as_ref().map(|file| file.id.as_ref()),
         },
         None,
         None,
         &channel,
-        false
-    ).await?;
+        false,
+    )
+    .await?;
 
     Ok(EmptyResponse)
 }
@@ -59,7 +69,11 @@ pub async fn message_pin(
 #[cfg(test)]
 mod test {
     use crate::{rocket, util::test::TestHarness};
-    use revolt_database::{events::client::EventV1, util::{idempotency::IdempotencyKey, reference::Reference}, Member, Message, Server};
+    use revolt_database::{
+        events::client::EventV1,
+        util::{idempotency::IdempotencyKey, reference::Reference},
+        Member, Message, Server,
+    };
     use revolt_models::v0::{self, SystemMessage};
     use rocket::http::{Header, Status};
 
@@ -75,24 +89,29 @@ mod test {
                 ..Default::default()
             },
             &user,
-            true
-        ).await.expect("Failed to create test server");
+            true,
+        )
+        .await
+        .expect("Failed to create test server");
 
-        let (member, channels) = Member::create(&harness.db, &server, &user, Some(channels)).await.expect("Failed to create member");
+        let (member, channels) = Member::create(&harness.db, &server, &user, Some(channels))
+            .await
+            .expect("Failed to create member");
         let channel = &channels[0];
 
         let message = Message::create_from_api(
             &harness.db,
+            None,
             channel.clone(),
             v0::DataMessageSend {
-                content:Some("Test message".to_string()),
+                content: Some("Test message".to_string()),
                 nonce: None,
                 attachments: None,
                 replies: None,
                 embeds: None,
                 masquerade: None,
                 interactions: None,
-                flags: None
+                flags: None,
             },
             v0::MessageAuthor::User(&user.clone().into(&harness.db, Some(&user)).await),
             Some(user.clone().into(&harness.db, Some(&user)).await),
@@ -100,14 +119,18 @@ mod test {
             user.limits().await,
             IdempotencyKey::unchecked_from_string("0".to_string()),
             false,
-            false
+            false,
         )
         .await
         .expect("Failed to create message");
 
         let response = harness
             .client
-            .post(format!("/channels/{}/messages/{}/pin", channel.id(), &message.id))
+            .post(format!(
+                "/channels/{}/messages/{}/pin",
+                channel.id(),
+                &message.id
+            ))
             .header(Header::new("x-session-token", session.token.to_string()))
             .dispatch()
             .await;
@@ -115,34 +138,37 @@ mod test {
         assert_eq!(response.status(), Status::NoContent);
         drop(response);
 
-        harness.wait_for_event(channel.id(), |event| {
-            match event {
-                EventV1::Message(message) => {
-                    match &message.system {
-                        Some(SystemMessage::MessagePinned { by, .. }) => {
-                            assert_eq!(by, &user.id);
+        harness
+            .wait_for_event(channel.id(), |event| match event {
+                EventV1::Message(message) => match &message.system {
+                    Some(SystemMessage::MessagePinned { by, .. }) => {
+                        assert_eq!(by, &user.id);
 
-                            true
-                        },
-                        _ => false
+                        true
                     }
+                    _ => false,
                 },
-                _ => false
-            }
-        }).await;
+                _ => false,
+            })
+            .await;
 
-        harness.wait_for_event(channel.id(), |event| {
-            match event {
-                EventV1::MessageUpdate { id, channel: channel_id, data, .. } => {
+        harness
+            .wait_for_event(channel.id(), |event| match event {
+                EventV1::MessageUpdate {
+                    id,
+                    channel: channel_id,
+                    data,
+                    ..
+                } => {
                     assert_eq!(id, &message.id);
                     assert_eq!(channel_id, channel.id());
                     assert_eq!(data.pinned, Some(true));
 
                     true
-                },
-                _ => false
-            }
-        }).await;
+                }
+                _ => false,
+            })
+            .await;
 
         let updated_message = Reference::from_unchecked(message.id)
             .as_message(&harness.db)

crates/delta/src/routes/channels/message_send.rs

@@ -3,7 +3,7 @@ use revolt_database::util::permissions::DatabasePermissionQuery;
 use revolt_database::{
     util::idempotency::IdempotencyKey, util::reference::Reference, Database, User,
 };
-use revolt_database::{Interactions, Message};
+use revolt_database::{Interactions, Message, AMQP};
 use revolt_models::v0;
 use revolt_permissions::PermissionQuery;
 use revolt_permissions::{calculate_channel_permissions, ChannelPermission};
@@ -19,6 +19,7 @@ use validator::Validate;
 #[post("/<target>/messages", data = "<data>")]
 pub async fn message_send(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     user: User,
     target: Reference,
     data: Json<v0::DataMessageSend>,
@@ -93,6 +94,7 @@ pub async fn message_send(
     Ok(Json(
         Message::create_from_api(
             db,
+            Some(amqp),
             channel,
             data,
             v0::MessageAuthor::User(&author),
@@ -107,3 +109,218 @@ pub async fn message_send(
         .into_model(Some(model_user), model_member),
     ))
 }
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashMap;
+
+    use crate::{rocket, util::test::TestHarness};
+    use revolt_database::{
+        util::{idempotency::IdempotencyKey, reference::Reference},
+        Channel, Member, Message, PartialChannel, PartialMember, Role, Server,
+    };
+    use revolt_models::v0::{self, DataCreateServerChannel};
+    use revolt_permissions::{ChannelPermission, OverrideField};
+
+    #[rocket::async_test]
+    async fn message_mention_constraints() {
+        let harness = TestHarness::new().await;
+        let (_, _, user) = harness.new_user().await;
+        let (_, _, second_user) = harness.new_user().await;
+
+        let (server, channels) = Server::create(
+            &harness.db,
+            v0::DataCreateServer {
+                name: "Test Server".to_string(),
+                ..Default::default()
+            },
+            &user,
+            true,
+        )
+        .await
+        .expect("Failed to create test server");
+
+        let server_mut: &mut Server = &mut server.clone();
+        let mut locked_channel = Channel::create_server_channel(
+            &harness.db,
+            server_mut,
+            DataCreateServerChannel {
+                channel_type: v0::LegacyServerChannelType::Text,
+                name: "Hidden Channel".to_string(),
+                description: None,
+                nsfw: Some(false),
+            },
+            true,
+        )
+        .await
+        .expect("Failed to make new channel");
+
+        let role = Role {
+            name: "Show Hidden Channel".to_string(),
+            permissions: OverrideField { a: 0, d: 0 },
+            colour: None,
+            hoist: false,
+            rank: 5,
+        };
+
+        let role_id = role
+            .create(&harness.db, &server.id)
+            .await
+            .expect("Failed to create the role");
+
+        let mut overrides = HashMap::new();
+        overrides.insert(
+            role_id.clone(),
+            OverrideField {
+                a: (ChannelPermission::ViewChannel) as i64,
+                d: 0,
+            },
+        );
+
+        let partial = PartialChannel {
+            name: None,
+            owner: None,
+            description: None,
+            icon: None,
+            nsfw: None,
+            active: None,
+            permissions: None,
+            role_permissions: Some(overrides),
+            default_permissions: Some(OverrideField {
+                a: 0,
+                d: ChannelPermission::ViewChannel as i64,
+            }),
+            last_message_id: None,
+        };
+        locked_channel
+            .update(&harness.db, partial, vec![])
+            .await
+            .expect("Failed to update the channel permissions for special role");
+
+        Member::create(&harness.db, &server, &user, Some(channels.clone()))
+            .await
+            .expect("Failed to create member");
+        let member = Reference::from_unchecked(user.id.clone())
+            .as_member(&harness.db, &server.id)
+            .await
+            .expect("Failed to get member");
+
+        // Second user is not part of the server
+        let message = Message::create_from_api(
+            &harness.db,
+            Some(&harness.amqp),
+            locked_channel.clone(),
+            v0::DataMessageSend {
+                content: Some(format!("<@{}>", second_user.id)),
+                nonce: None,
+                attachments: None,
+                replies: None,
+                embeds: None,
+                masquerade: None,
+                interactions: None,
+                flags: None,
+            },
+            v0::MessageAuthor::User(&user.clone().into(&harness.db, Some(&user)).await),
+            Some(user.clone().into(&harness.db, Some(&user)).await),
+            Some(member.clone().into()),
+            user.limits().await,
+            IdempotencyKey::unchecked_from_string("0".to_string()),
+            false,
+            true,
+        )
+        .await
+        .expect("Failed to create message");
+
+        // The mention should not go through here
+        assert!(
+            message.mentions.is_none() || message.mentions.unwrap().is_empty(),
+            "Mention failed to be scrubbed when the user is not part of the server"
+        );
+
+        Member::create(&harness.db, &server, &second_user, Some(channels.clone()))
+            .await
+            .expect("Failed to create second member");
+        let mut second_member = Reference::from_unchecked(second_user.id.clone())
+            .as_member(&harness.db, &server.id)
+            .await
+            .expect("Failed to get second member");
+
+        // Second user cannot see the channel
+        let message = Message::create_from_api(
+            &harness.db,
+            Some(&harness.amqp),
+            locked_channel.clone(),
+            v0::DataMessageSend {
+                content: Some(format!("<@{}>", second_user.id)),
+                nonce: None,
+                attachments: None,
+                replies: None,
+                embeds: None,
+                masquerade: None,
+                interactions: None,
+                flags: None,
+            },
+            v0::MessageAuthor::User(&user.clone().into(&harness.db, Some(&user)).await),
+            Some(user.clone().into(&harness.db, Some(&user)).await),
+            Some(member.clone().into()),
+            user.limits().await,
+            IdempotencyKey::unchecked_from_string("1".to_string()),
+            false,
+            true,
+        )
+        .await
+        .expect("Failed to create message");
+
+        // The mention should not go through here
+        assert!(
+            message.mentions.is_none() || message.mentions.unwrap().is_empty(),
+            "Mention failed to be scrubbed when the user cannot see the channel"
+        );
+
+        let second_member_roles = vec![role_id.clone()];
+        let partial = PartialMember {
+            id: None,
+            joined_at: None,
+            nickname: None,
+            avatar: None,
+            timeout: None,
+            roles: Some(second_member_roles),
+        };
+        second_member
+            .update(&harness.db, partial, vec![])
+            .await
+            .expect("Failed to update the second user's roles");
+
+        // This time the mention SHOULD go through
+        let message = Message::create_from_api(
+            &harness.db,
+            Some(&harness.amqp),
+            locked_channel.clone(),
+            v0::DataMessageSend {
+                content: Some(format!("<@{}>", second_user.id)),
+                nonce: None,
+                attachments: None,
+                replies: None,
+                embeds: None,
+                masquerade: None,
+                interactions: None,
+                flags: None,
+            },
+            v0::MessageAuthor::User(&user.clone().into(&harness.db, Some(&user)).await),
+            Some(user.clone().into(&harness.db, Some(&user)).await),
+            Some(member.clone().into()),
+            user.limits().await,
+            IdempotencyKey::unchecked_from_string("2".to_string()),
+            false,
+            true,
+        )
+        .await
+        .expect("Failed to create message");
+
+        // The mention SHOULD go through here
+        assert!(
+            message.mentions.is_some() && !message.mentions.unwrap().is_empty(),
+            "Mention was scrubbed when the user can see the channel"
+        );
+    }
+}

crates/delta/src/routes/channels/message_unpin.rs

@@ -1,5 +1,6 @@
 use revolt_database::{
-    util::{permissions::DatabasePermissionQuery, reference::Reference}, Database, FieldsMessage, PartialMessage, SystemMessage, User
+    util::{permissions::DatabasePermissionQuery, reference::Reference},
+    Database, FieldsMessage, PartialMessage, SystemMessage, User, AMQP,
 };
 use revolt_models::v0::MessageAuthor;
 use revolt_permissions::{calculate_channel_permissions, ChannelPermission};
@@ -14,6 +15,7 @@ use rocket_empty::EmptyResponse;
 #[delete("/<target>/messages/<msg>/pin")]
 pub async fn message_unpin(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     user: User,
     target: Reference,
     msg: Reference,
@@ -28,27 +30,31 @@ pub async fn message_unpin(
     let mut message = msg.as_message_in_channel(db, channel.id()).await?;
 
     if !message.pinned.unwrap_or_default() {
-        return Err(create_error!(NotPinned))
+        return Err(create_error!(NotPinned));
     }
 
-    message.update(db, PartialMessage::default(), vec![FieldsMessage::Pinned]).await?;
+    message
+        .update(db, PartialMessage::default(), vec![FieldsMessage::Pinned])
+        .await?;
 
     SystemMessage::MessageUnpinned {
         id: message.id.clone(),
-        by: user.id.clone()
+        by: user.id.clone(),
     }
     .into_message(channel.id().to_string())
     .send(
         db,
+        Some(amqp),
         MessageAuthor::System {
             username: &user.username,
-            avatar: user.avatar.as_ref().map(|file| file.id.as_ref())
+            avatar: user.avatar.as_ref().map(|file| file.id.as_ref()),
         },
         None,
         None,
         &channel,
-        false
-    ).await?;
+        false,
+    )
+    .await?;
 
     Ok(EmptyResponse)
 }
@@ -56,7 +62,11 @@ pub async fn message_unpin(
 #[cfg(test)]
 mod test {
     use crate::{rocket, util::test::TestHarness};
-    use revolt_database::{events::client::EventV1, util::{idempotency::IdempotencyKey, reference::Reference}, Member, Message, PartialMessage, Server};
+    use revolt_database::{
+        events::client::EventV1,
+        util::{idempotency::IdempotencyKey, reference::Reference},
+        Member, Message, PartialMessage, Server,
+    };
     use revolt_models::v0::{self, FieldsMessage, SystemMessage};
     use rocket::http::{Header, Status};
 
@@ -72,26 +82,34 @@ mod test {
                 ..Default::default()
             },
             &user,
-            true
-        ).await.expect("Failed to create test server");
+            true,
+        )
+        .await
+        .expect("Failed to create test server");
 
         let channel = &channels[0];
 
-        Member::create(&harness.db, &server, &user, Some(channels.clone())).await.expect("Failed to create member");
-        let member = Reference::from_unchecked(user.id.clone()).as_member(&harness.db, &server.id).await.expect("Failed to get member");
+        Member::create(&harness.db, &server, &user, Some(channels.clone()))
+            .await
+            .expect("Failed to create member");
+        let member = Reference::from_unchecked(user.id.clone())
+            .as_member(&harness.db, &server.id)
+            .await
+            .expect("Failed to get member");
 
         let message = Message::create_from_api(
             &harness.db,
+            None,
             channel.clone(),
             v0::DataMessageSend {
-                content:Some("Test message".to_string()),
+                content: Some("Test message".to_string()),
                 nonce: None,
                 attachments: None,
                 replies: None,
                 embeds: None,
                 masquerade: None,
                 interactions: None,
-                flags: None
+                flags: None,
             },
             v0::MessageAuthor::User(&user.clone().into(&harness.db, Some(&user)).await),
             Some(user.clone().into(&harness.db, Some(&user)).await),
@@ -99,23 +117,31 @@ mod test {
             user.limits().await,
             IdempotencyKey::unchecked_from_string("0".to_string()),
             false,
-            false
+            false,
         )
         .await
         .expect("Failed to create message");
 
-        harness.db.update_message(
-            &message.id,
-            &PartialMessage {
-                pinned: Some(true),
-                ..Default::default()
-            },
-            vec![]
-        ).await.expect("Failed to update message");
+        harness
+            .db
+            .update_message(
+                &message.id,
+                &PartialMessage {
+                    pinned: Some(true),
+                    ..Default::default()
+                },
+                vec![],
+            )
+            .await
+            .expect("Failed to update message");
 
         let response = harness
             .client
-            .delete(format!("/channels/{}/messages/{}/pin", channel.id(), &message.id))
+            .delete(format!(
+                "/channels/{}/messages/{}/pin",
+                channel.id(),
+                &message.id
+            ))
             .header(Header::new("x-session-token", session.token.to_string()))
             .dispatch()
             .await;
@@ -123,33 +149,31 @@ mod test {
         assert_eq!(response.status(), Status::NoContent);
         drop(response);
 
-        harness.wait_for_event(channel.id(), |event| {
-            match event {
-                EventV1::Message(message) => {
-                    match &message.system {
-                        Some(SystemMessage::MessageUnpinned { by, .. }) => {
-                            assert_eq!(by, &user.id);
+        harness
+            .wait_for_event(channel.id(), |event| match event {
+                EventV1::Message(message) => match &message.system {
+                    Some(SystemMessage::MessageUnpinned { by, .. }) => {
+                        assert_eq!(by, &user.id);
 
-                            true
-                        },
-                        _ => false
+                        true
                     }
+                    _ => false,
                 },
-                _ => false
-            }
-        }).await;
+                _ => false,
+            })
+            .await;
 
-        harness.wait_for_event(channel.id(), |event| {
-            match event {
+        harness
+            .wait_for_event(channel.id(), |event| match event {
                 EventV1::MessageUpdate { id, clear, .. } => {
                     assert_eq!(&message.id, id);
                     assert_eq!(clear, &[FieldsMessage::Pinned]);
 
                     true
-                },
-                _ => false
-            }
-        }).await;
+                }
+                _ => false,
+            })
+            .await;
 
         let updated_message = Reference::from_unchecked(message.id)
             .as_message(&harness.db)

crates/delta/src/routes/invites/invite_join.rs

@@ -1,4 +1,4 @@
-use revolt_database::{util::reference::Reference, Channel, Database, Invite, Member, User};
+use revolt_database::{util::reference::Reference, Channel, Database, Invite, Member, User, AMQP};
 use revolt_models::v0::{self, InviteJoinResponse};
 use revolt_result::{create_error, Result};
 use rocket::{serde::json::Json, State};
@@ -10,6 +10,7 @@ use rocket::{serde::json::Json, State};
 #[post("/<target>")]
 pub async fn join(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     user: User,
     target: Reference,
 ) -> Result<Json<v0::InviteJoinResponse>> {
@@ -34,7 +35,7 @@ pub async fn join(
             channel, creator, ..
         } => {
             let mut channel = db.fetch_channel(channel).await?;
-            channel.add_user_to_group(db, &user, creator).await?;
+            channel.add_user_to_group(db, amqp, &user, creator).await?;
             if let Channel::Group { recipients, .. } = &channel {
                 Ok(Json(InviteJoinResponse::Group {
                     users: User::fetch_many_ids_as_mutuals(db, &user, recipients).await?,

crates/delta/src/routes/root.rs

@@ -114,7 +114,7 @@ pub async fn root() -> Result<Json<RevoltConfig>> {
         },
         ws: config.hosts.events,
         app: config.hosts.app,
-        vapid: config.api.vapid.public_key,
+        vapid: config.pushd.vapid.public_key,
         build: BuildInformation {
             commit_sha: option_env!("VERGEN_GIT_SHA")
                 .unwrap_or_else(|| "<failed to generate>")

crates/delta/src/routes/users/add_friend.rs

@@ -1,5 +1,5 @@
 use revolt_database::util::reference::Reference;
-use revolt_database::{Database, User};
+use revolt_database::{Database, User, AMQP};
 use revolt_models::v0;
 use revolt_result::{create_error, Result};
 use rocket::serde::json::Json;
@@ -12,6 +12,7 @@ use rocket::State;
 #[put("/<target>/friend")]
 pub async fn add(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     mut user: User,
     target: Reference,
 ) -> Result<Json<v0::User>> {
@@ -21,6 +22,6 @@ pub async fn add(
         return Err(create_error!(IsBot));
     }
 
-    user.add_friend(db, &mut target).await?;
+    user.add_friend(db, amqp, &mut target).await?;
     Ok(Json(target.into(db, &user).await))
 }

crates/delta/src/routes/users/send_friend_request.rs

@@ -1,5 +1,5 @@
-use revolt_database::util::reference::Reference;
-use revolt_database::{Database, User};
+// use revolt_database::util::reference::Reference;
+use revolt_database::{Database, User, AMQP};
 use revolt_models::v0;
 use revolt_result::{create_error, Result};
 use rocket::serde::json::Json;
@@ -12,6 +12,7 @@ use rocket::State;
 #[post("/friend", data = "<data>")]
 pub async fn send_friend_request(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     mut user: User,
     data: Json<v0::DataSendFriendRequest>,
 ) -> Result<Json<v0::User>> {
@@ -22,7 +23,7 @@ pub async fn send_friend_request(
             return Err(create_error!(IsBot));
         }
 
-        user.add_friend(db, &mut target).await?;
+        user.add_friend(db, amqp, &mut target).await?;
         Ok(Json(target.into(db, &user).await))
     } else {
         Err(create_error!(InvalidProperty))

crates/delta/src/routes/webhooks/webhook_execute.rs

@@ -1,7 +1,7 @@
 use revolt_config::config;
 use revolt_database::{
     util::{idempotency::IdempotencyKey, reference::Reference},
-    Database, Message,
+    Database, Message, AMQP,
 };
 use revolt_models::v0;
 use revolt_permissions::{ChannelPermission, PermissionValue};
@@ -17,6 +17,7 @@ use validator::Validate;
 #[post("/<webhook_id>/<token>", data = "<data>")]
 pub async fn webhook_execute(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     webhook_id: Reference,
     token: String,
     data: Json<v0::DataMessageSend>,
@@ -56,6 +57,7 @@ pub async fn webhook_execute(
     Ok(Json(
         Message::create_from_api(
             db,
+            Some(amqp),
             channel,
             data,
             v0::MessageAuthor::Webhook(&webhook.into()),

crates/delta/src/routes/webhooks/webhook_execute_github.rs

@@ -1,4 +1,4 @@
-use revolt_database::{util::reference::Reference, Database, Message};
+use revolt_database::{util::reference::Reference, Database, Message, AMQP};
 use revolt_models::v0::{MessageAuthor, SendableEmbed, Webhook};
 use revolt_result::{create_error, Error, Result};
 use revolt_rocket_okapi::{
@@ -635,7 +635,10 @@ impl<'r> FromRequest<'r> for EventHeader<'r> {
     async fn from_request(request: &'r Request<'_>) -> rocket::request::Outcome<Self, Self::Error> {
         let headers = request.headers();
         let Some(event) = headers.get_one("X-GitHub-Event") else {
-            return rocket::request::Outcome::Failure((Status::BadRequest, create_error!(InvalidOperation)))
+            return rocket::request::Outcome::Error((
+                Status::BadRequest,
+                create_error!(InvalidOperation),
+            ));
         };
 
         rocket::request::Outcome::Success(Self(event))
@@ -747,6 +750,7 @@ fn convert_event(data: &str, event_name: &str) -> Result<Event> {
 #[post("/<webhook_id>/<token>/github", data = "<data>")]
 pub async fn webhook_execute_github(
     db: &State<Database>,
+    amqp: &State<AMQP>,
     webhook_id: Reference,
     token: String,
     event: EventHeader<'_>,
@@ -784,7 +788,9 @@ pub async fn webhook_execute_github(
             r#ref,
             ..
         }) => {
-            let Some(branch) = r#ref.split('/').nth(2) else { return Ok(()) };
+            let Some(branch) = r#ref.split('/').nth(2) else {
+                return Ok(());
+            };
 
             if forced {
                 let description = format!(
@@ -1074,6 +1080,7 @@ pub async fn webhook_execute_github(
     message
         .send(
             db,
+            Some(amqp),
             MessageAuthor::Webhook(&webhook.into()),
             None,
             None,

crates/delta/src/util/ratelimiter.rs

@@ -235,7 +235,7 @@ impl<'r> FromRequest<'r> for Ratelimiter {
 
         match ratelimiter {
             Ok(ratelimiter) => Outcome::Success(*ratelimiter),
-            Err(ratelimiter) => Outcome::Failure((Status::TooManyRequests, *ratelimiter)),
+            Err(ratelimiter) => Outcome::Error((Status::TooManyRequests, *ratelimiter)),
         }
     }
 }
@@ -264,7 +264,7 @@ impl Fairing for RatelimitFairing {
 
     async fn on_request(&self, request: &mut Request<'_>, _: &mut Data<'_>) {
         use rocket::outcome::Outcome;
-        if let Outcome::Failure(_) = request.guard::<Ratelimiter>().await {
+        if let Outcome::Error(_) = request.guard::<Ratelimiter>().await {
             info!(
                 "User rate-limited on route {}! (IP = {:?})",
                 request.uri(),
@@ -278,7 +278,7 @@ impl Fairing for RatelimitFairing {
 
     async fn on_response<'r>(&self, request: &'r Request<'_>, response: &mut Response<'r>) {
         let guard = request.guard::<Ratelimiter>().await;
-        let (Outcome::Success(ratelimiter) | Outcome::Failure((_, ratelimiter))) = guard else {
+        let (Outcome::Success(ratelimiter) | Outcome::Error((_, ratelimiter))) = guard else {
             unreachable!()
         };
         let Ratelimiter {
@@ -293,7 +293,7 @@ impl Fairing for RatelimitFairing {
         response.set_raw_header("X-RateLimit-Remaining", remaining.to_string());
         response.set_raw_header("X-RateLimit-Reset-After", reset.to_string());
 
-        if guard.is_failure() {
+        if guard.is_error() {
             response.set_status(Status::TooManyRequests);
         }
     }
@@ -313,7 +313,7 @@ impl<'r> FromRequest<'r> for RatelimitInformation {
     async fn from_request(request: &'r rocket::Request<'_>) -> Outcome<Self, Self::Error> {
         let info = match request.guard::<Ratelimiter>().await {
             Outcome::Success(ratelimiter) => RatelimitInformation::Success(ratelimiter),
-            Outcome::Failure((_, ratelimiter)) => RatelimitInformation::Failure {
+            Outcome::Error((_, ratelimiter)) => RatelimitInformation::Failure {
                 retry_after: ratelimiter.reset,
             },
             _ => unreachable!(),

crates/delta/src/util/test.rs

@@ -5,7 +5,7 @@ use authifier::{
 use futures::StreamExt;
 use rand::Rng;
 use redis_kiss::redis::aio::PubSub;
-use revolt_database::{events::client::EventV1, Database, User};
+use revolt_database::{events::client::EventV1, Database, User, AMQP};
 use revolt_models::v0;
 use rocket::local::asynchronous::Client;
 
@@ -13,13 +13,14 @@ pub struct TestHarness {
     pub client: Client,
     authifier: Authifier,
     pub db: Database,
+    pub amqp: AMQP,
     sub: PubSub,
     event_buffer: Vec<(String, EventV1)>,
 }
 
 impl TestHarness {
     pub async fn new() -> TestHarness {
-        dotenv::dotenv().ok();
+        let config = revolt_config::config().await;
 
         let client = Client::tracked(crate::web().await)
             .await
@@ -43,10 +44,25 @@ impl TestHarness {
             .expect("`Authifier`")
             .clone();
 
+        let connection = amqprs::connection::Connection::open(
+            &amqprs::connection::OpenConnectionArguments::new(
+                &config.rabbit.host,
+                config.rabbit.port,
+                &config.rabbit.username,
+                &config.rabbit.password,
+            ),
+        )
+        .await
+        .unwrap();
+        let channel = connection.open_channel(None).await.unwrap();
+
+        let amqp = AMQP::new(connection, channel);
+
         TestHarness {
             client,
             authifier,
             db,
+            amqp,
             sub,
             event_buffer: vec![],
         }

crates/services/autumn/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-autumn"
-version = "0.7.14"
+version = "0.8.0"
 edition = "2021"
 
 [dependencies]
@@ -13,7 +13,7 @@ webp = "0.3.0"
 sha2 = "0.10.8"
 jxl-oxide = "0.8.1"
 kamadak-exif = "0.5.4"
-# revolt_little_exif = "0.4.0"
+# revolt_little_exif = "0.5.1"
 image = { version = "0.25.2" } # avif encode requires dav1d system library: features = ["avif-native"]
 
 # File processing
@@ -42,12 +42,12 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 # Core crates
-revolt-files = { version = "0.7.16", path = "../../core/files" }
-revolt-config = { version = "0.7.16", path = "../../core/config" }
-revolt-database = { version = "0.7.16", path = "../../core/database", features = [
+revolt-files = { version = "0.8.0", path = "../../core/files" }
+revolt-config = { version = "0.8.0", path = "../../core/config" }
+revolt-database = { version = "0.8.0", path = "../../core/database", features = [
     "axum-impl",
 ] }
-revolt-result = { version = "0.7.16", path = "../../core/result", features = [
+revolt-result = { version = "0.8.0", path = "../../core/result", features = [
     "utoipa",
     "axum",
 ] }

crates/services/autumn/src/api.rs

@@ -11,12 +11,13 @@ use axum::{
     Json, Router,
 };
 use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart};
-use image::ImageReader;
 use lazy_static::lazy_static;
 use revolt_config::{config, report_internal_error};
 use revolt_database::{iso8601_timestamp::Timestamp, Database, FileHash, Metadata, User};
-use revolt_files::{fetch_from_s3, upload_to_s3, AUTHENTICATION_TAG_SIZE_BYTES};
-use revolt_result::{create_error, Result};
+use revolt_files::{
+    create_thumbnail, decode_image, fetch_from_s3, upload_to_s3, AUTHENTICATION_TAG_SIZE_BYTES,
+};
+use revolt_result::{create_error, Error, Result};
 use serde::{Deserialize, Serialize};
 use sha2::Digest;
 use tempfile::NamedTempFile;
@@ -54,8 +55,17 @@ lazy_static! {
     /// Short-lived file cache to allow us to populate different CDN regions without increasing bandwidth to S3 provider
     /// Uploads will also be stored here to prevent immediately queued downloads from doing the entire round-trip
     static ref S3_CACHE: moka::future::Cache<String, Result<Vec<u8>>> = moka::future::Cache::builder()
-        .max_capacity(10_000) // TODO config
-        .time_to_live(Duration::from_secs(60)) // TODO config
+        .weigher(|_key, value: &Result<Vec<u8>>| -> u32 {
+            std::mem::size_of::<Result<Vec<u8>>>() as u32 + if let Ok(vec) = value {
+                vec.len().try_into().unwrap_or(u32::MAX)
+            } else {
+                std::mem::size_of::<Error>() as u32
+            }
+        })
+        // TODO config
+        // .max_capacity(1024 * 1024 * 1024) // Cache up to 1GiB in memory
+        .max_capacity(512 * 1024 * 1024) // Cache up to 512MiB in memory
+        .time_to_live(Duration::from_secs(5 * 60)) // For up to 5 minutes
         .build();
 }
 
@@ -206,16 +216,27 @@ async fn upload_file(
     };
 
     // Find an existing hash and use that if possible
-    if let Ok(file_hash) = db
+    let file_hash_exists = if let Ok(file_hash) = db
         .fetch_attachment_hash(&format!("{original_hash:02x}"))
         .await
     {
-        let tag: &'static str = tag.into();
-        db.insert_attachment(&file_hash.into_file(id.clone(), tag.to_owned(), filename, user.id))
+        if !file_hash.iv.is_empty() {
+            let tag: &'static str = tag.into();
+            db.insert_attachment(&file_hash.into_file(
+                id.clone(),
+                tag.to_owned(),
+                filename,
+                user.id,
+            ))
             .await?;
 
-        return Ok(Json(UploadResponse { id }));
-    }
+            return Ok(Json(UploadResponse { id }));
+        }
+
+        true
+    } else {
+        false
+    };
 
     // Determine the mime type for the file
     let mime_type = determine_mime_type(&mut file.contents, &buf, &filename);
@@ -273,7 +294,10 @@ async fn upload_file(
         size: new_file_size as isize,
     };
 
-    db.insert_attachment_hash(&file_hash).await?;
+    // Add attachment hash if it doesn't exist
+    if !file_hash_exists {
+        db.insert_attachment_hash(&file_hash).await?;
+    }
 
     // Upload the file to S3 and commit nonce to database
     let upload_start = Instant::now();
@@ -352,30 +376,12 @@ async fn fetch_preview(
     // Original image data
     let data = retrieve_file_by_hash(&hash).await?;
 
-    // Dimensions we need to resize to
-    let config = config().await;
-    let [w, h] = config.files.preview.get(tag).unwrap();
-
-    // Read the image and resize it
-    // TODO: use jxl_oxide to process image/jxl files
-    let image = report_internal_error!(report_internal_error!(ImageReader::new(Cursor::new(
-        data
-    ))
-    .with_guessed_format())?
-    .decode())?
-    //.resize(width as u32, height as u32, image::imageops::FilterType::Gaussian)
-    // resize is about 2.5x slower,
-    // thumbnail doesn't have terrible quality
-    // so we use thumbnail
-    .thumbnail(*w as u32, *h as u32);
-
-    // Encode it into WEBP
-    let encoder = webp::Encoder::from_image(&image).expect("Could not create encoder.");
-    let data = if config.files.webp_quality != 100.0 {
-        encoder.encode(config.files.webp_quality).to_vec()
-    } else {
-        encoder.encode_lossless().to_vec()
-    };
+    // Read image and create thumbnail
+    let data = create_thumbnail(
+        decode_image(&mut Cursor::new(data), &file.content_type)?,
+        tag,
+    )
+    .await;
 
     Ok((
         [

crates/services/autumn/src/exif.rs

@@ -17,23 +17,24 @@ pub async fn strip_metadata(
 ) -> Result<(Vec<u8>, Metadata)> {
     match &metadata {
         Metadata::Image { width, height } => match mime {
-            // little_exif does not appear to parse JPEGs correctly? had 2/2 files fail
-            /* "image/jpeg" | "image/png" => {
-                // use little_exif to strip metadata except for orientation and colour profile
-                // PNGs must also be re-encoded to mitigate CVE-2023-21036
-                let metadata = revolt_little_exif::metadata::Metadata::new_from_path_with_filetype(
-                    file.path(),
-                    match mime {
-                        "image/jpeg" => revolt_little_exif::filetype::FileExtension::JPEG,
-                        "image/png" => revolt_little_exif::filetype::FileExtension::PNG {
-                            as_zTXt_chunk: true,
-                        },
-                        _ => unreachable!(),
-                    },
-                )
-                .unwrap();
-                dbg!(metadata.data());
-            } */
+            // // little_exif does not appear to parse JPEGs correctly? had 2/2 files fail
+            // "image/jpeg" | "image/png" => {
+            //     // use little_exif to strip metadata except for orientation and colour profile
+            //     // PNGs must also be re-encoded to mitigate CVE-2023-21036
+            //     let metadata = revolt_little_exif::metadata::Metadata::new_from_path_with_filetype(
+            //         file.path(),
+            //         match mime {
+            //             "image/jpeg" => revolt_little_exif::filetype::FileExtension::JPEG,
+            //             "image/png" => revolt_little_exif::filetype::FileExtension::PNG {
+            //                 as_zTXt_chunk: true,
+            //             },
+            //             _ => unreachable!(),
+            //         },
+            //     )
+            //     .unwrap();
+            //     dbg!(metadata.data());
+            //     todo!()
+            // }
             // Apply orientation manually & strip all other EXIF data
             "image/jpeg" | "image/png" | "image/avif" | "image/tiff" => {
                 // Create a reader

crates/services/autumn/src/metadata.rs

@@ -1,6 +1,7 @@
 use std::io::Cursor;
 
 use revolt_database::Metadata;
+use revolt_files::{image_size, video_size};
 use tempfile::NamedTempFile;
 
 /// Intersection of what infer can detect and what image-rs supports
@@ -21,32 +22,19 @@ static SUPPORTED_IMAGE_MIME: [&str; 9] = [
 /// Generate metadata from file, using mime type as a hint
 pub fn generate_metadata(f: &NamedTempFile, mime_type: &str) -> Metadata {
     if SUPPORTED_IMAGE_MIME.contains(&mime_type) {
-        if let Ok(size) = imagesize::size(f.path())
-            .inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}"))
-        {
-            if let (Ok(width), Ok(height)) = (size.width.try_into(), size.height.try_into()) {
-                return Metadata::Image { width, height };
-            }
-        }
-
-        Metadata::File
+        image_size(f)
+            .map(|(width, height)| Metadata::Image {
+                width: width as isize,
+                height: height as isize,
+            })
+            .unwrap_or_default()
     } else if mime_type.starts_with("video/") {
-        if let Ok(data) = ffprobe::ffprobe(f.path())
-            .inspect_err(|err| tracing::error!("Failed to ffprobe file! {err:?}"))
-        {
-            // Use first valid stream
-            for stream in data.streams {
-                if let (Some(w), Some(h)) = (stream.width, stream.height) {
-                    if let (Ok(width), Ok(height)) = (w.try_into(), h.try_into()) {
-                        return Metadata::Video { width, height };
-                    }
-                }
-            }
-
-            Metadata::File
-        } else {
-            Metadata::File
-        }
+        video_size(f)
+            .map(|(width, height)| Metadata::Video {
+                width: width as isize,
+                height: height as isize,
+            })
+            .unwrap_or_default()
     } else if mime_type.starts_with("audio/") {
         Metadata::Audio
     } else if mime_type == "plain/text" {

crates/services/autumn/src/mime_type.rs

@@ -19,7 +19,11 @@ pub fn determine_mime_type(f: &mut NamedTempFile, buf: &[u8], file_name: &str) -
 
     // See if the file is actually just plain Unicode/ASCII text
     if mime_type == "application/octet-stream" && simdutf8::basic::from_utf8(buf).is_ok() {
-        return "plain/text";
+        if file_name.to_lowercase().ends_with(".svg") {
+            return "image/svg+xml";
+        } else {
+            return "plain/text";
+        }
     }
 
     mime_type

crates/services/january/Cargo.toml

@@ -1,19 +1,26 @@
 [package]
 name = "revolt-january"
-version = "0.7.14"
+version = "0.8.0"
 edition = "2021"
 
 [dependencies]
 # Utility
 mime = "0.3.17"
+regex = "1.11.0"
+tempfile = "3.13.0"
 lazy_static = "1.5.0"
 moka = { version = "0.12.8", features = ["future"] }
 
+# Web scraping
+scraper = "0.20.0"
+encoding_rs = "0.8.34"
+
 # Serialisation
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.68"
 
 # Async runtime
+async-recursion = "1.1.1"
 tokio = { version = "1.0", features = ["full"] }
 
 # Web requests
@@ -24,12 +31,13 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 # Core crates
-revolt-config = { version = "0.7.16", path = "../../core/config" }
-revolt-models = { version = "0.7.16", path = "../../core/models" }
-revolt-result = { version = "0.7.16", path = "../../core/result", features = [
+revolt-config = { version = "0.8.0", path = "../../core/config" }
+revolt-models = { version = "0.8.0", path = "../../core/models" }
+revolt-result = { version = "0.8.0", path = "../../core/result", features = [
     "utoipa",
     "axum",
 ] }
+revolt-files = { version = "0.8.0", path = "../../core/files" }
 
 # Axum / web server
 axum = { version = "0.7.5" }

crates/services/january/src/api.rs

@@ -1,16 +1,14 @@
-use axum::{body::Bytes, extract::Query, routing::get, Json, Router};
+use axum::{extract::Query, response::IntoResponse, routing::get, Json, Router};
+use reqwest::header;
 use revolt_models::v0::Embed;
-use revolt_result::Result;
+use revolt_result::{create_error, Result};
 use serde::{Deserialize, Serialize};
 use utoipa::ToSchema;
 
-use axum_extra::{
-    headers::{authorization::Bearer, Authorization},
-    TypedHeader,
-};
-
 use crate::requests::Request;
 
+pub static CACHE_CONTROL: &str = "public, max-age=600, immutable";
+
 pub async fn router() -> Router {
     Router::new()
         .route("/", get(root))
@@ -59,8 +57,17 @@ struct UrlQuery {
         ("url" = String, Query, description = "URL to fetch")
     ),
 )]
-async fn proxy(Query(UrlQuery { url }): Query<UrlQuery>) -> Result<Bytes> {
-    Request::proxy_file(&url).await
+async fn proxy(Query(UrlQuery { url }): Query<UrlQuery>) -> Result<impl IntoResponse> {
+    Request::proxy_file(&url).await.map(|(content_type, data)| {
+        (
+            [
+                (header::CONTENT_TYPE, content_type),
+                (header::CONTENT_DISPOSITION, "inline".to_owned()),
+                (header::CACHE_CONTROL, CACHE_CONTROL.to_owned()),
+            ],
+            data,
+        )
+    })
 }
 
 /// Generate embed for a given URL
@@ -79,7 +86,11 @@ async fn proxy(Query(UrlQuery { url }): Query<UrlQuery>) -> Result<Bytes> {
 )]
 async fn embed(
     Query(UrlQuery { url }): Query<UrlQuery>,
-    TypedHeader(Authorization(_bearer)): TypedHeader<Authorization<Bearer>>,
-) -> Result<Json<Embed>> {
-    Request::generate_embed(&url).await.map(Json)
+    // TypedHeader(Authorization(_bearer)): TypedHeader<Authorization<Bearer>>,
+) -> Result<impl IntoResponse> {
+    match Request::generate_embed(url).await {
+        Ok(Embed::None) => Err(create_error!(NoEmbedData)),
+        result => result,
+    }
+    .map(Json)
 }

crates/services/january/src/main.rs

@@ -11,6 +11,7 @@ use utoipa_scalar::{Scalar, Servable as ScalarServable};
 
 mod api;
 pub mod requests;
+pub mod website_embed;
 
 #[tokio::main]
 async fn main() -> Result<(), std::io::Error> {
@@ -65,6 +66,8 @@ async fn main() -> Result<(), std::io::Error> {
         .nest("/", api::router().await);
 
     // Configure TCP listener and bind
+    tracing::info!("Listening on 0.0.0.0:14705");
+    tracing::info!("Play around with the API: http://localhost:14705/scalar");
     let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, 14705));
     let listener = TcpListener::bind(&address).await?;
     axum::serve(listener, app.into_make_service()).await

crates/services/january/src/requests.rs

@@ -1,15 +1,23 @@
-use std::time::Duration;
-
-use axum::body::Bytes;
+use encoding_rs::{Encoding, UTF_8_INIT};
 use lazy_static::lazy_static;
 use mime::Mime;
-use reqwest::{header::CONTENT_TYPE, redirect, Client, Response};
-use revolt_models::v0::Embed;
-use revolt_result::{create_error, Result};
+use regex::Regex;
+use reqwest::{
+    header::{self, CONTENT_TYPE},
+    redirect, Client, Response,
+};
+use revolt_config::report_internal_error;
+use revolt_files::{create_thumbnail, decode_image, image_size_vec, is_valid_image, video_size};
+use revolt_models::v0::{Embed, Image, ImageSize, Video};
+use revolt_result::{create_error, Error, Result};
+use std::{
+    io::{Cursor, Write},
+    time::Duration,
+};
 
 lazy_static! {
+    /// Request client
     static ref CLIENT: Client = reqwest::Client::builder()
-        .user_agent("Mozilla/5.0 (compatible; January/2.0; +https://github.com/revoltchat/backend)")
         .timeout(Duration::from_secs(10)) // TODO config
         .connect_timeout(Duration::from_secs(5)) // TODO config
         .redirect(redirect::Policy::custom(|attempt| {
@@ -24,16 +32,32 @@ lazy_static! {
         .build()
         .expect("reqwest Client");
 
+    /// Spoof User Agent as Discord
+    static ref RE_USER_AGENT_SPOOFING_AS_DISCORD: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:vx|fx)?twitter|(?:fixv|fixup)?x|(?:old\\.|new\\.|www\\.)reddit).com").expect("valid regex");
+
+    /// Regex for matching new Reddit URLs
+    static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex");
+
     /// Cache for proxy results
-    static ref PROXY_CACHE: moka::future::Cache<String, Result<Bytes>> = moka::future::Cache::builder()
-        .max_capacity(10_000) // TODO config
-        .time_to_live(Duration::from_secs(60)) // TODO config
+    static ref PROXY_CACHE: moka::future::Cache<String, Result<(String, Vec<u8>)>> = moka::future::Cache::builder()
+        .weigher(|_key, value: &Result<(String, Vec<u8>)>| -> u32 {
+            std::mem::size_of::<Result<(String, Vec<u8>)>>() as u32 + if let Ok((url, vec)) = value {
+                url.len().try_into().unwrap_or(u32::MAX) +
+                vec.len().try_into().unwrap_or(u32::MAX)
+            } else {
+                std::mem::size_of::<Error>() as u32
+            }
+        })
+        // TODO config
+        .max_capacity(512 * 1024 * 1024) // Cache up to 512MiB in memory
+        .time_to_live(Duration::from_secs(60)) // For up to 1 minute
         .build();
 
     /// Cache for embed results
-    static ref EMBED_CACHE: moka::future::Cache<String, Result<Embed>> = moka::future::Cache::builder()
-        .max_capacity(1_000) // TODO config
-        .time_to_live(Duration::from_secs(60)) // TODO config
+    static ref EMBED_CACHE: moka::future::Cache<String, Embed> = moka::future::Cache::builder()
+        // TODO config
+        .max_capacity(10_000) // Cache up to 10k embeds
+        .time_to_live(Duration::from_secs(60)) // For up to 1 minute
         .build();
 }
 
@@ -45,20 +69,188 @@ pub struct Request {
 
 impl Request {
     /// Proxy a given URL
-    pub async fn proxy_file(url: &str) -> Result<Bytes> {
+    pub async fn proxy_file(url: &str) -> Result<(String, Vec<u8>)> {
         if let Some(hit) = PROXY_CACHE.get(url).await {
             hit
         } else {
-            todo!()
+            let Request { response, mime } = Request::new(url).await?;
+
+            if matches!(mime.type_(), mime::IMAGE | mime::VIDEO) {
+                let bytes = report_internal_error!(response.bytes().await);
+
+                let result = match bytes {
+                    Ok(bytes) => {
+                        if matches!(mime.type_(), mime::IMAGE) {
+                            let reader = &mut Cursor::new(&bytes);
+
+                            if matches!(mime.subtype(), mime::GIF) {
+                                if is_valid_image(reader, "image/gif") {
+                                    Ok(("image/gif".to_owned(), bytes.to_vec()))
+                                } else {
+                                    Err(create_error!(FileTypeNotAllowed))
+                                }
+                            } else {
+                                Ok((
+                                    "image/webp".to_owned(),
+                                    create_thumbnail(
+                                        decode_image(reader, mime.as_ref())?,
+                                        "attachments",
+                                    )
+                                    .await,
+                                ))
+                            }
+                        } else {
+                            let mut file = report_internal_error!(tempfile::NamedTempFile::new())?;
+                            report_internal_error!(file.write_all(&bytes))?;
+
+                            if video_size(&file).is_some() {
+                                Ok((mime.to_string(), bytes.to_vec()))
+                            } else {
+                                Err(create_error!(FileTypeNotAllowed))
+                            }
+                        }
+                    }
+                    Err(err) => Err(err),
+                };
+
+                PROXY_CACHE.insert(url.to_owned(), result.clone()).await;
+                result
+            } else {
+                Err(create_error!(FileTypeNotAllowed))
+            }
+        }
+    }
+
+    /// Fetch metadata for an image
+    pub async fn fetch_image_metadata(
+        url: &str,
+        request: Option<Request>,
+    ) -> Result<Option<Image>> {
+        if let Some(hit) = EMBED_CACHE.get(url).await {
+            match hit {
+                Embed::Image(img) => Ok(Some(img)),
+                _ => Ok(None),
+            }
+        } else {
+            let request = if let Some(request) = request {
+                request
+            } else {
+                let request = Request::new(url).await?;
+                if matches!(request.mime.type_(), mime::IMAGE) {
+                    request
+                } else {
+                    return Err(create_error!(FileTypeNotAllowed));
+                }
+            };
+
+            if let Some((width, height)) = image_size_vec(
+                &report_internal_error!(request.response.bytes().await)?,
+                request.mime.as_ref(),
+            ) {
+                Ok(Some(Image {
+                    url: url.to_owned(),
+                    width,
+                    height,
+                    size: ImageSize::Large,
+                }))
+            } else {
+                Ok(None)
+            }
+        }
+    }
+
+    /// Fetch metadata for an video
+    pub async fn fetch_video_metadata(
+        url: &str,
+        request: Option<Request>,
+    ) -> Result<Option<Video>> {
+        if let Some(hit) = EMBED_CACHE.get(url).await {
+            match hit {
+                Embed::Video(vid) => Ok(Some(vid)),
+                _ => Ok(None),
+            }
+        } else {
+            let response = if let Some(Request { response, .. }) = request {
+                response
+            } else {
+                let Request { response, mime } = Request::new(url).await?;
+                if matches!(mime.type_(), mime::VIDEO) {
+                    response
+                } else {
+                    return Err(create_error!(FileTypeNotAllowed));
+                }
+            };
+
+            let mut file = report_internal_error!(tempfile::NamedTempFile::new())?;
+            report_internal_error!(
+                file.write_all(&report_internal_error!(response.bytes().await)?)
+            )?;
+
+            if let Some((width, height)) = video_size(&file) {
+                Ok(Some(Video {
+                    url: url.to_owned(),
+                    width: width as usize,
+                    height: height as usize,
+                }))
+            } else {
+                Ok(None)
+            }
         }
     }
 
     /// Generate embed for a given URL
-    pub async fn generate_embed(url: &str) -> Result<Embed> {
-        if let Some(hit) = EMBED_CACHE.get(url).await {
-            hit
+    pub async fn generate_embed(mut url: String) -> Result<Embed> {
+        // Re-map certain links for better metadata generation
+        if RE_URL_NEW_REDDIT.is_match(&url) {
+            url = RE_URL_NEW_REDDIT
+                // Reddit has a bunch of clickbait-y marketing on the new URLs, so we use the old site instead
+                .replace(&url, "https://old.reddit.com")
+                .to_string();
+        }
+
+        // Generate the actual embed
+        if let Some(hit) = EMBED_CACHE.get(&url).await {
+            Ok(hit)
         } else {
-            todo!()
+            let request = Request::new(&url).await?;
+            let embed = match (request.mime.type_(), request.mime.subtype()) {
+                (_, mime::HTML) => {
+                    let content_type = request
+                        .response
+                        .headers()
+                        .get(header::CONTENT_TYPE)
+                        .and_then(|value| value.to_str().ok())
+                        .and_then(|value| value.parse::<Mime>().ok());
+
+                    let encoding_name = content_type
+                        .as_ref()
+                        .and_then(|mime| mime.get_param("charset").map(|charset| charset.as_str()))
+                        .unwrap_or("utf-8");
+
+                    let encoding =
+                        Encoding::for_label(encoding_name.as_bytes()).unwrap_or(&UTF_8_INIT);
+
+                    let bytes = report_internal_error!(request.response.bytes().await)?;
+                    let (text, _, _) = encoding.decode(&bytes);
+
+                    crate::website_embed::create_website_embed(&url, &text)
+                        .await
+                        .map(Embed::Website)
+                        .unwrap_or_default()
+                }
+                (mime::IMAGE, _) => Request::fetch_image_metadata(&url, Some(request))
+                    .await
+                    .map(|res| res.map(Embed::Image).unwrap_or_default())
+                    .unwrap_or_default(),
+                (mime::VIDEO, _) => Request::fetch_video_metadata(&url, Some(request))
+                    .await
+                    .map(|res| res.map(Embed::Video).unwrap_or_default())
+                    .unwrap_or_default(),
+                _ => Embed::None,
+            };
+
+            EMBED_CACHE.insert(url.to_owned(), embed.clone()).await;
+            Ok(embed)
         }
     }
 
@@ -66,6 +258,15 @@ impl Request {
     pub async fn new(url: &str) -> Result<Request> {
         let response = CLIENT
             .get(url)
+            .header(
+                "User-Agent",
+                if RE_USER_AGENT_SPOOFING_AS_DISCORD.is_match(url) {
+                    "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"
+                } else {
+                    "Mozilla/5.0 (compatible; January/2.0; +https://github.com/revoltchat/backend)"
+                },
+            )
+            .header("Accept-Language", "en-US,en;q=0.5")
             .send()
             .await
             .map_err(|_| create_error!(ProxyError))?;
@@ -88,4 +289,13 @@ impl Request {
 
         Ok(Request { response, mime })
     }
+
+    /// Check if something exists
+    pub async fn exists(url: &str) -> bool {
+        if let Ok(response) = CLIENT.head(url).send().await {
+            response.status().is_success()
+        } else {
+            false
+        }
+    }
 }

crates/services/january/src/website_embed.rs

@@ -0,0 +1,335 @@
+use std::collections::HashMap;
+
+use lazy_static::lazy_static;
+use regex::Regex;
+use revolt_models::v0::{
+    BandcampType, Image, ImageSize, LightspeedType, Special, TwitchType, Video, WebsiteMetadata,
+};
+use scraper::{Html, Selector};
+
+/// Create website metadata from URL and document
+pub async fn create_website_embed(original_url: &str, document: &str) -> Option<WebsiteMetadata> {
+    let (mut meta, mut link) = {
+        let document = Html::parse_document(document);
+
+        // create selectors
+        let meta_selector = Selector::parse("meta").ok()?;
+        let link_selector = Selector::parse("link").ok()?;
+
+        // extract meta tags
+        let mut meta = HashMap::new();
+        for el in document.select(&meta_selector) {
+            let node = el.value();
+
+            if let (Some(property), Some(content)) = (
+                node.attr("property").or_else(|| node.attr("name")),
+                node.attr("content"),
+            ) {
+                meta.insert(property.to_string(), content.to_string());
+            }
+        }
+
+        // extract rel links
+        let mut link = HashMap::new();
+        for el in document.select(&link_selector) {
+            let node = el.value();
+
+            if let (Some(property), Some(content)) = (node.attr("rel"), node.attr("href")) {
+                link.insert(property.to_string(), content.to_string());
+            }
+        }
+
+        (meta, link)
+    };
+
+    // build metadata
+    let mut metadata = WebsiteMetadata {
+        title: meta
+            .remove("og:title")
+            .or_else(|| meta.remove("twitter:title"))
+            .or_else(|| meta.remove("title"))
+            .map(|s| s.trim().to_owned()),
+        description: meta
+            .remove("og:description")
+            .or_else(|| meta.remove("twitter:description"))
+            .or_else(|| meta.remove("description"))
+            .map(|s| s.trim().to_owned()),
+        image: meta
+            .remove("og:image")
+            .or_else(|| meta.remove("og:image:secure_url"))
+            .or_else(|| meta.remove("twitter:image"))
+            .or_else(|| meta.remove("twitter:image:src"))
+            .map(|s| s.trim().to_owned())
+            .map(|mut url| {
+                // If relative URL, prepend root URL. Also if root URL ends with a slash, remove it.
+                if let Some(ch) = url.chars().next() {
+                    if ch == '/' {
+                        url = format!("{}{}", &original_url.trim_end_matches('/'), url);
+                    }
+                }
+
+                let mut size = ImageSize::Preview;
+                if let Some(card) = meta.remove("twitter:card") {
+                    if &card == "summary_large_image" {
+                        size = ImageSize::Large;
+                    }
+                }
+
+                Image {
+                    url: url.to_owned(),
+                    width: meta
+                        .remove("og:image:width")
+                        .unwrap_or_default()
+                        .parse()
+                        .unwrap_or(0),
+                    height: meta
+                        .remove("og:image:height")
+                        .unwrap_or_default()
+                        .parse()
+                        .unwrap_or(0),
+                    size,
+                }
+            }),
+        video: meta
+            .remove("og:video")
+            .or_else(|| meta.remove("og:video:url"))
+            .or_else(|| meta.remove("og:video:secure_url"))
+            .map(|s| s.trim().to_owned())
+            .map(|mut url| {
+                // If relative URL, prepend root URL. Also if root URL ends with a slash, remove it.
+                if let Some(ch) = url.chars().next() {
+                    if ch == '/' {
+                        url = format!("{}{}", &original_url.trim_end_matches('/'), url);
+                    }
+                }
+
+                Video {
+                    url: url.to_owned(),
+                    width: meta
+                        .remove("og:video:width")
+                        .unwrap_or_default()
+                        .parse()
+                        .unwrap_or(0),
+                    height: meta
+                        .remove("og:video:height")
+                        .unwrap_or_default()
+                        .parse()
+                        .unwrap_or(0),
+                }
+            }),
+        icon_url: link
+            .remove("apple-touch-icon")
+            .or_else(|| link.remove("icon"))
+            .map(|s| s.trim().to_owned())
+            .map(|mut v| {
+                // If relative URL, prepend root URL.
+                if let Some(ch) = v.chars().next() {
+                    if ch == '/' {
+                        v = format!("{}{}", &original_url.trim_end_matches('/'), v);
+                    }
+                }
+
+                v
+            }),
+        colour: meta.remove("theme-color").map(|s| s.trim().to_owned()),
+        site_name: meta.remove("og:site_name").map(|s| s.trim().to_owned()),
+        url: meta
+            .remove("og:url")
+            .or_else(|| Some(original_url.to_owned())),
+        original_url: Some(original_url.to_owned()),
+        special: None,
+    };
+
+    // populate extra metadata for popular websites
+    populate_special(original_url.to_owned(), &mut metadata).await;
+
+    // fetch video size if missing
+    if metadata.special.is_none() {
+        if let Some(Video { width, height, url }) = &metadata.video {
+            if width == &0 || height == &0 {
+                metadata.video =
+                    match crate::requests::Request::fetch_video_metadata(url, None).await {
+                        Ok(Some(video)) => Some(video),
+                        _ => None,
+                    }
+            }
+        }
+    }
+
+    // remove image if video exists
+    if metadata.video.is_some() {
+        metadata.image.take();
+    }
+
+    // fetch image size if missing
+    if metadata.special.is_none() {
+        if let Some(Image {
+            width, height, url, ..
+        }) = &metadata.image
+        {
+            if width == &0 || height == &0 {
+                metadata.image =
+                    match crate::requests::Request::fetch_image_metadata(url, None).await {
+                        Ok(Some(image)) => Some(image),
+                        _ => None,
+                    }
+            }
+        }
+    }
+
+    // truncate data
+    metadata.truncate();
+
+    // if it's empty, don't return anything
+    if metadata.is_empty() {
+        None
+    } else {
+        Some(metadata)
+    }
+}
+
+pub async fn populate_special(original_url: String, metadata: &mut WebsiteMetadata) {
+    lazy_static! {
+        static ref RE_YOUTUBE: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www|m)\\.)?(?:(?:youtube\\.com|youtu.be))(?:/(?:[\\w\\-]+\\?v=|embed/|v/)?)([\\w\\-]+)(?:\\S+)?$").unwrap();
+
+        static ref RE_LIGHTSPEED: Regex = Regex::new("^(?:https?://)?(?:[\\w]+\\.)?lightspeed\\.tv/([a-z0-9_]{4,25})").unwrap();
+
+        static ref RE_TWITCH: Regex = Regex::new("^(?:https?://)?(?:www\\.|go\\.)?twitch\\.tv/([a-z0-9_]+)($|\\?)").unwrap();
+        static ref RE_TWITCH_VOD: Regex = Regex::new("^(?:https?://)?(?:www\\.|go\\.)?twitch\\.tv/videos/([0-9]+)($|\\?)").unwrap();
+        static ref RE_TWITCH_CLIP: Regex = Regex::new("^(?:https?://)?(?:www\\.|go\\.)?twitch\\.tv/(?:[a-z0-9_]+)/clip/([A-z0-9_-]+)($|\\?)").unwrap();
+
+        static ref RE_SPOTIFY: Regex = Regex::new("^(?:https?://)?open.spotify.com/(track|user|artist|album|playlist)/([A-z0-9]+)").unwrap();
+        static ref RE_SOUNDCLOUD: Regex = Regex::new("^(?:https?://)?soundcloud.com/([a-zA-Z0-9-]+)/([A-z0-9-]+)").unwrap();
+        static ref RE_BANDCAMP: Regex = Regex::new("^(?:https?://)?(?:[A-z0-9_-]+).bandcamp.com/(track|album)/([A-z0-9_-]+)").unwrap();
+        static ref RE_APPLE_MUSIC: Regex = Regex::new("^(?:https?://)?music\\.apple\\.com/(?:[a-z]{2}/)?album/(?:[a-zA-Z0-9-]+)/(\\d+)(?:\\?i=(\\d+))?").unwrap();
+
+        static ref RE_STREAMABLE: Regex = Regex::new("^(?:https?://)?(?:www\\.)?streamable\\.com/([\\w\\d-]+)").unwrap();
+
+        static ref RE_GIF: Regex = Regex::new("^(?:https?://)?(www\\.)?(gifbox\\.me/view|yiffbox\\.me/view|tenor\\.com/view|giphy\\.com/gifs|gfycat\\.com|redgifs\\.com/watch)/[\\w\\d-]+").unwrap();
+    }
+
+    let url = metadata
+        .url
+        .as_ref()
+        .or(metadata.original_url.as_ref())
+        .unwrap_or(&original_url)
+        .as_str();
+
+    metadata.special = if let Some(captures) = RE_STREAMABLE.captures_iter(url).next() {
+        Some(Special::Streamable {
+            id: captures[1].to_string(),
+        })
+    } else if let Some(captures) = RE_YOUTUBE.captures_iter(url).next() {
+        let id = captures[1].to_string();
+
+        lazy_static! {
+            static ref RE_TIMESTAMP: Regex = Regex::new("(?:\\?|&)(?:t|start)=([\\w]+)").unwrap();
+        }
+
+        // YouTube now blocks datacentre IPs from fetching information
+        // This is a fallback to prevent the embed from looking weird
+        if metadata.video.is_none() {
+            metadata.title.replace("YouTube".to_owned());
+            metadata.description.take();
+            metadata.colour.take();
+            metadata.icon_url.take();
+            metadata.site_name.take();
+
+            // Verify the video exists
+            if !crate::requests::Request::exists(&format!(
+                "http://img.youtube.com/vi/{}/sddefault.jpg",
+                id
+            ))
+            .await
+            {
+                return;
+            }
+        }
+
+        if let Some(timestamp_captures) = RE_TIMESTAMP.captures_iter(url).next() {
+            Some(Special::YouTube {
+                id,
+                timestamp: Some(timestamp_captures[1].to_string()),
+            })
+        } else {
+            Some(Special::YouTube {
+                id,
+                timestamp: None,
+            })
+        }
+    } else if let Some(captures) = RE_LIGHTSPEED.captures_iter(url).next() {
+        Some(Special::Lightspeed {
+            id: captures[1].to_string(),
+            content_type: LightspeedType::Channel,
+        })
+    } else if let Some(captures) = RE_TWITCH.captures_iter(url).next() {
+        Some(Special::Twitch {
+            id: captures[1].to_string(),
+            content_type: TwitchType::Channel,
+        })
+    } else if let Some(captures) = RE_TWITCH_VOD.captures_iter(url).next() {
+        Some(Special::Twitch {
+            id: captures[1].to_string(),
+            content_type: TwitchType::Video,
+        })
+    } else if let Some(captures) = RE_TWITCH_CLIP.captures_iter(url).next() {
+        Some(Special::Twitch {
+            id: captures[1].to_string(),
+            content_type: TwitchType::Clip,
+        })
+    } else if let Some(captures) = RE_SPOTIFY.captures_iter(url).next() {
+        Some(Special::Spotify {
+            content_type: captures[1].to_string(),
+            id: captures[2].to_string(),
+        })
+    } else if RE_SOUNDCLOUD.is_match(url) {
+        Some(Special::Soundcloud)
+    } else if RE_BANDCAMP.is_match(url) {
+        lazy_static! {
+            static ref RE_TRACK: Regex = Regex::new("track=(\\d+)").unwrap();
+            static ref RE_ALBUM: Regex = Regex::new("album=(\\d+)").unwrap();
+        }
+
+        if let Some(video) = &metadata.video {
+            if let Some(captures) = RE_TRACK.captures_iter(&video.url).next() {
+                Some(Special::Bandcamp {
+                    content_type: BandcampType::Track,
+                    id: captures[1].to_string(),
+                })
+            } else {
+                RE_ALBUM
+                    .captures_iter(&video.url)
+                    .next()
+                    .map(|captures| Special::Bandcamp {
+                        content_type: BandcampType::Album,
+                        id: captures[1].to_string(),
+                    })
+            }
+        } else {
+            None
+        }
+    } else if RE_GIF.is_match(url) {
+        Some(Special::GIF)
+    } else {
+        RE_APPLE_MUSIC
+            .captures_iter(url)
+            .next()
+            .map(|captures| Special::AppleMusic {
+                album_id: captures[1].to_string(),
+                track_id: captures.get(2).map(|m| m.as_str().to_string()),
+            })
+    };
+
+    // add colours for popular websites
+    if let Some(special) = &metadata.special {
+        match special {
+            Special::YouTube { .. } => metadata.colour = Some("#FF424F".to_string()),
+            Special::Twitch { .. } => metadata.colour = Some("#7B68EE".to_string()),
+            Special::Lightspeed { .. } => metadata.colour = Some("#7445D9".to_string()),
+            Special::Spotify { .. } => metadata.colour = Some("#1ABC9C".to_string()),
+            Special::Soundcloud { .. } => metadata.colour = Some("#FF7F50".to_string()),
+            Special::AppleMusic { .. } => metadata.colour = Some("#FA233B".to_string()),
+            _ => {}
+        }
+    }
+}

doc/src/SUMMARY.md

@@ -2,6 +2,7 @@
 
 - [Introduction](./hello.md)
 - [Project Structure]()
+- [Creating new API features](./new_features.md)
 - [Testing]()
   - [Writing a new database test]()
   - [Writing a new API test]()

doc/src/new_features.md

@@ -0,0 +1,15 @@
+# New API features
+
+New API features must be documented where appropriate, this document aims to cover everywhere you need to update for new features.
+
+Before writing new API features, generally a good idea to:
+
+- Consult with other developers in the [Revolt Developers space](https://rvlt.gg/API)
+- If it's a relatively big feature, also [write an RFC](https://github.com/revoltchat/rfcs)
+
+When your feature is ready to release, ensure to:
+
+- Update backend documentation (what you're reading now!) if applicable
+- Update the [developers documentation](https://github.com/revoltchat/wiki) if applicable
+- Update the Feature Matrix (or ask someone that can to do so)
+- Ensure it is properly listed in the backend release changelog