feat: add crond container definitions

closes #376

.github/workflows/docker.yaml

@@ -27,7 +27,7 @@ permissions:
 
 jobs:
   base:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     name: Build base image
     steps:
       # Configure build environment
@@ -57,7 +57,7 @@ jobs:
 
   publish:
     needs: [base]
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     if: github.event_name != 'pull_request'
     strategy:
       matrix:
@@ -110,6 +110,10 @@ jobs:
               "pushd": {
                 "path": "crates/daemons/pushd",
                 "tag": "${{ github.repository_owner }}/pushd"
+              },
+              "crond": {
+                "path": "crates/daemons/crond",
+                "tag": "${{ github.repository_owner }}/crond"
               }
             }
           export_to: output

.gitignore

@@ -1,5 +1,6 @@
 Rocket.toml
 Revolt.*.toml
+compose.override.yml
 
 target
 .data

Cargo.lock

@@ -2410,9 +2410,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.21"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e"
+checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -2441,9 +2441,9 @@ checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.21"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6"
+checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -5523,7 +5523,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-autumn"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "axum",
  "axum-macros",
@@ -5560,7 +5560,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-bonfire"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "async-channel 2.3.1",
  "async-std",
@@ -5590,7 +5590,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-config"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "async-std",
  "cached",
@@ -5604,9 +5604,21 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "revolt-crond"
+version = "0.8.2"
+dependencies = [
+ "log",
+ "revolt-config",
+ "revolt-database",
+ "revolt-files",
+ "revolt-result",
+ "tokio 1.40.0",
+]
+
 [[package]]
 name = "revolt-database"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "amqprs",
  "async-lock 2.8.0",
@@ -5655,7 +5667,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-delta"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "amqprs",
  "async-channel 1.6.1",
@@ -5702,7 +5714,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-files"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "aes-gcm",
  "aws-config",
@@ -5725,7 +5737,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-january"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "async-recursion",
  "axum",
@@ -5753,7 +5765,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-models"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "indexmap 1.9.3",
  "iso8601-timestamp 0.2.11",
@@ -5771,7 +5783,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-nodejs-bindings"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "async-std",
  "neon",
@@ -5784,7 +5796,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-permissions"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "async-std",
  "async-trait",
@@ -5799,7 +5811,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-presence"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "async-std",
  "log",
@@ -5810,7 +5822,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-pushd"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "amqprs",
  "async-trait",
@@ -5834,7 +5846,7 @@ dependencies = [
 
 [[package]]
 name = "revolt-result"
-version = "0.8.0"
+version = "0.8.2"
 dependencies = [
  "axum",
  "revolt_okapi",

Cargo.toml

@@ -7,7 +7,7 @@ members = [
     "crates/core/*",
     "crates/services/*",
     "crates/bindings/*",
-    "crates/daemons/pushd",
+    "crates/daemons/*",
 ]
 
 [patch.crates-io]
@@ -16,5 +16,9 @@ redis23 = { package = "redis", version = "0.23.1", git = "https://github.com/rev
 # authifier = { package = "authifier", version = "1.0.8", path = "../authifier/crates/authifier" }
 # rocket_authifier = { package = "rocket_authifier", version = "1.0.8", path = "../authifier/crates/rocket_authifier" }
 
-[profile.release]
+# I'm 99% sure this is overloading the GitHub worker
-lto = true
+# hence builds have been failing since, let's just
+# disable it for now. In the future, we could use this
+# if we were rolling our own CI.
+# [profile.release]
+# lto = true

Dockerfile

@@ -29,6 +29,7 @@ COPY crates/core/presence/Cargo.toml ./crates/core/presence/
 COPY crates/core/result/Cargo.toml ./crates/core/result/
 COPY crates/services/autumn/Cargo.toml ./crates/services/autumn/
 COPY crates/services/january/Cargo.toml ./crates/services/january/
+COPY crates/daemons/crond/Cargo.toml ./crates/daemons/crond/
 COPY crates/daemons/pushd/Cargo.toml ./crates/daemons/pushd/
 RUN sh /tmp/build-image-layer.sh deps
 

Dockerfile.useCurrentArch

@@ -25,6 +25,7 @@ COPY crates/core/presence/Cargo.toml ./crates/core/presence/
 COPY crates/core/result/Cargo.toml ./crates/core/result/
 COPY crates/services/autumn/Cargo.toml ./crates/services/autumn/
 COPY crates/services/january/Cargo.toml ./crates/services/january/
+COPY crates/daemons/crond/Cargo.toml ./crates/daemons/crond/
 COPY crates/daemons/pushd/Cargo.toml ./crates/daemons/pushd/
 RUN sh /tmp/build-image-layer.sh deps
 

README.md

@@ -26,6 +26,7 @@ The services and libraries that power the Revolt service.<br/>
 | `services/january` | [crates/services/january](crates/services/january) | Proxy server                             | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue)                                                                                                                                                                                                                                |
 | `services/autumn`  | [crates/services/autumn](crates/services/autumn)   | File server                              | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue)                                                                                                                                                                                                                                |
 | `bindings/node`    | [crates/bindings/node](crates/bindings/node)       | Node.js bindings for the Revolt software | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue)                                                                                                                                                                                                                                |
+| `daemons/crond`    | [crates/daemons/crond](crates/daemons/crond)       | Timed data clean up daemon server        | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue)                                                                                                                                                                                                                                |
 | `daemons/pushd`    | [crates/daemons/pushd](crates/daemons/pushd)       | Push notification daemon server          | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue)                                                                                                                                                                                                                                |
 
 </div>
@@ -91,6 +92,38 @@ If you'd like to change anything, create a `Revolt.overrides.toml` file and spec
 > proxy = "https://abc@your.sentry/1"
 > ```
 
+> [!TIP]
+> If you have port conflicts on common services, you can try the following:
+>
+> ```yaml
+> # compose.override.yml
+> services:
+>   redis:
+>     ports: !override
+>       - "14079:6379"
+>
+>   database:
+>     ports: !override
+>       - "14017:27017"
+>
+>   rabbit:
+>     ports: !override
+>       - "14072:5672"
+>       - "14672:15672"
+> ```
+>
+> And corresponding Revolt configuration:
+>
+> ```toml
+> # Revolt.overrides.toml
+> [database]
+> mongodb = "mongodb://127.0.0.1:14017"
+> redis = "redis://127.0.0.1:14079/"
+>
+> [rabbit]
+> port = 14072
+> ```
+
 Then continue:
 
 ```bash

Revolt.toml

@@ -4,10 +4,13 @@
 [database]
 # MongoDB connection URL
 # Defaults to the container name specified in self-hosted
-mongodb = "mongodb://127.0.0.1:14017"
+mongodb = "mongodb://127.0.0.1:27017"
 # Redis connection URL
 # Defaults to the container name specified in self-hosted
-redis = "redis://127.0.0.1:14079/"
+redis = "redis://127.0.0.1:6379/"
+
+[rabbit]
+host = "127.0.0.1"
 
 [hosts]
 # Web locations of various services

crates/bindings/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-nodejs-bindings"
-version = "0.8.0"
+version = "0.8.2"
 description = "Node.js bindings for the Revolt software"
 authors = ["Paul Makles <me@insrt.uk>"]
 license = "MIT"
@@ -20,6 +20,6 @@ serde = { version = "1", features = ["derive"] }
 
 async-std = "1.12.0"
 
-revolt-config = { version = "0.8.0", path = "../../core/config" }
+revolt-config = { version = "0.8.2", path = "../../core/config" }
-revolt-result = { version = "0.8.0", path = "../../core/result" }
+revolt-result = { version = "0.8.2", path = "../../core/result" }
-revolt-database = { version = "0.8.0", path = "../../core/database" }
+revolt-database = { version = "0.8.2", path = "../../core/database" }

crates/bonfire/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-bonfire"
-version = "0.8.0"
+version = "0.8.2"
 license = "AGPL-3.0-or-later"
 edition = "2021"
 
@@ -41,7 +41,7 @@ revolt-result = { path = "../core/result" }
 revolt-models = { path = "../core/models" }
 revolt-config = { path = "../core/config" }
 revolt-database = { path = "../core/database" }
-revolt-permissions = { version = "0.8.0", path = "../core/permissions" }
+revolt-permissions = { version = "0.8.2", path = "../core/permissions" }
 revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] }
 
 # redis

crates/core/config/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-config"
-version = "0.8.0"
+version = "0.8.2"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -34,4 +34,4 @@ pretty_env_logger = "0.4.0"
 sentry = "0.31.5"
 
 # Core
-revolt-result = { version = "0.8.0", path = "../result", optional = true }
+revolt-result = { version = "0.8.2", path = "../result", optional = true }

crates/core/config/Revolt.toml

@@ -258,3 +258,4 @@ api = ""
 events = ""
 files = ""
 proxy = ""
+crond = ""

crates/core/config/src/lib.rs

@@ -324,6 +324,7 @@ pub struct Sentry {
     pub events: String,
     pub files: String,
     pub proxy: String,
+    pub crond: String,
 }
 
 #[derive(Deserialize, Debug, Clone)]

crates/core/database/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-database"
-version = "0.8.0"
+version = "0.8.2"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -24,15 +24,15 @@ default = ["mongodb", "async-std-runtime", "tasks"]
 
 [dependencies]
 # Core
-revolt-config = { version = "0.8.0", path = "../config", features = [
+revolt-config = { version = "0.8.2", path = "../config", features = [
     "report-macros",
 ] }
-revolt-result = { version = "0.8.0", path = "../result" }
+revolt-result = { version = "0.8.2", path = "../result" }
-revolt-models = { version = "0.8.0", path = "../models", features = [
+revolt-models = { version = "0.8.2", path = "../models", features = [
     "validator",
 ] }
-revolt-presence = { version = "0.8.0", path = "../presence" }
+revolt-presence = { version = "0.8.2", path = "../presence" }
-revolt-permissions = { version = "0.8.0", path = "../permissions", features = [
+revolt-permissions = { version = "0.8.2", path = "../permissions", features = [
     "serde",
     "bson",
 ] }

crates/core/database/src/models/channel_unreads/ops/mongodb.rs

@@ -30,7 +30,7 @@ impl AbstractChannelUnreads for MongoDb {
                 doc! {
                     "$pull": {
                         "mentions": {
-                            "$lt": message_id
+                            "$lte": message_id
                         }
                     },
                     "$set": {

crates/core/database/src/models/file_hashes/ops.rs

@@ -15,4 +15,7 @@ pub trait AbstractAttachmentHashes: Sync + Send {
 
     /// Update an attachment hash nonce value.
     async fn set_attachment_hash_nonce(&self, hash: &str, nonce: &str) -> Result<()>;
+
+    /// Delete attachment hash by id.
+    async fn delete_attachment_hash(&self, id: &str) -> Result<()>;
 }

crates/core/database/src/models/file_hashes/ops/mongodb.rs

@@ -48,4 +48,9 @@ impl AbstractAttachmentHashes for MongoDb {
             .map(|_| ())
             .map_err(|_| create_database_error!("update_one", COL))
     }
+
+    /// Delete attachment hash by id.
+    async fn delete_attachment_hash(&self, id: &str) -> Result<()> {
+        query!(self, delete_one_by_id, COL, id).map(|_| ())
+    }
 }

crates/core/database/src/models/file_hashes/ops/reference.rs

@@ -23,8 +23,8 @@ impl AbstractAttachmentHashes for ReferenceDb {
         let hashes = self.file_hashes.lock().await;
         hashes
             .values()
+            .find(|&hash| hash.id == hash_value || hash.processed_hash == hash_value)
             .cloned()
-            .find(|hash| hash.id == hash_value || hash.processed_hash == hash_value)
             .ok_or(create_error!(NotFound))
     }
 
@@ -38,4 +38,14 @@ impl AbstractAttachmentHashes for ReferenceDb {
             Err(create_error!(NotFound))
         }
     }
+
+    /// Delete attachment hash by id.
+    async fn delete_attachment_hash(&self, id: &str) -> Result<()> {
+        let mut file_hashes = self.file_hashes.lock().await;
+        if file_hashes.remove(id).is_some() {
+            Ok(())
+        } else {
+            Err(create_error!(NotFound))
+        }
+    }
 }

crates/core/database/src/models/files/ops.rs

@@ -15,6 +15,15 @@ pub trait AbstractAttachments: Sync + Send {
     /// Fetch an attachment by its id.
     async fn fetch_attachment(&self, tag: &str, file_id: &str) -> Result<File>;
 
+    /// Fetch all deleted attachments.
+    async fn fetch_deleted_attachments(&self) -> Result<Vec<File>>;
+
+    /// Fetch all dangling attachments.
+    async fn fetch_dangling_files(&self) -> Result<Vec<File>>;
+
+    /// Count references to a given hash.
+    async fn count_file_hash_references(&self, hash: &str) -> Result<usize>;
+
     /// Find an attachment by its details and mark it as used by a given parent.
     async fn find_and_use_attachment(
         &self,
@@ -32,4 +41,7 @@ pub trait AbstractAttachments: Sync + Send {
 
     /// Mark multiple attachments as having been deleted.
     async fn mark_attachments_as_deleted(&self, ids: &[String]) -> Result<()>;
+
+    /// Delete the attachment entry.
+    async fn delete_attachment(&self, id: &str) -> Result<()>;
 }

crates/core/database/src/models/files/ops/mongodb.rs

@@ -32,6 +32,51 @@ impl AbstractAttachments for MongoDb {
         .ok_or_else(|| create_error!(NotFound))
     }
 
+    /// Fetch all deleted attachments.
+    async fn fetch_deleted_attachments(&self) -> Result<Vec<File>> {
+        query!(
+            self,
+            find,
+            COL,
+            doc! {
+                "deleted": true,
+                "reported": {
+                    "$ne": true
+                }
+            }
+        )
+    }
+
+    /// Fetch all dangling attachments.
+    async fn fetch_dangling_files(&self) -> Result<Vec<File>> {
+        query!(
+            self,
+            find,
+            COL,
+            doc! {
+                "used_for.type": {
+                    "$exists": 0
+                },
+                "deleted": {
+                    "$ne": true
+                }
+            }
+        )
+    }
+
+    /// Count references to a given hash.
+    async fn count_file_hash_references(&self, hash: &str) -> Result<usize> {
+        query!(
+            self,
+            count_documents,
+            COL,
+            doc! {
+                "hash": hash
+            }
+        )
+        .map(|count| count as usize)
+    }
+
     /// Find an attachment by its details and mark it as used by a given parent.
     async fn find_and_use_attachment(
         &self,
@@ -114,7 +159,7 @@ impl AbstractAttachments for MongoDb {
     /// Mark multiple attachments as having been deleted.
     async fn mark_attachments_as_deleted(&self, ids: &[String]) -> Result<()> {
         self.col::<Document>(COL)
-            .update_one(
+            .update_many(
                 doc! {
                     "_id": {
                         "$in": ids
@@ -129,7 +174,12 @@ impl AbstractAttachments for MongoDb {
             )
             .await
             .map(|_| ())
-            .map_err(|_| create_database_error!("update_one", COL))
+            .map_err(|_| create_database_error!("update_many", COL))
+    }
+
+    /// Delete the attachment entry.
+    async fn delete_attachment(&self, id: &str) -> Result<()> {
+        query!(self, delete_one_by_id, COL, id).map(|_| ())
     }
 }
 

crates/core/database/src/models/files/ops/reference.rs

@@ -33,6 +33,41 @@ impl AbstractAttachments for ReferenceDb {
         }
     }
 
+    /// Fetch all deleted attachments.
+    async fn fetch_deleted_attachments(&self) -> Result<Vec<File>> {
+        let files = self.files.lock().await;
+        Ok(files
+            .values()
+            .filter(|file| {
+                // file has been marked as deleted
+                file.deleted.is_some_and(|v| v)
+                    // and it has not been reported
+                    && !file.reported.is_some_and(|v| v)
+            })
+            .cloned()
+            .collect())
+    }
+
+    /// Fetch all dangling attachments.
+    async fn fetch_dangling_files(&self) -> Result<Vec<File>> {
+        let files = self.files.lock().await;
+        Ok(files
+            .values()
+            .filter(|file| file.used_for.is_none() && !file.deleted.is_some_and(|v| v))
+            .cloned()
+            .collect())
+    }
+
+    /// Count references to a given hash.
+    async fn count_file_hash_references(&self, hash: &str) -> Result<usize> {
+        let files = self.files.lock().await;
+        Ok(files
+            .values()
+            .filter(|file| file.hash.as_ref().is_some_and(|h| h == hash))
+            .cloned()
+            .count())
+    }
+
     /// Find an attachment by its details and mark it as used by a given parent.
     async fn find_and_use_attachment(
         &self,
@@ -96,4 +131,14 @@ impl AbstractAttachments for ReferenceDb {
 
         Ok(())
     }
+
+    /// Delete the attachment entry.
+    async fn delete_attachment(&self, id: &str) -> Result<()> {
+        let mut files = self.files.lock().await;
+        if files.remove(id).is_some() {
+            Ok(())
+        } else {
+            Err(create_error!(NotFound))
+        }
+    }
 }

crates/core/database/src/models/messages/ops/mongodb.rs

@@ -95,7 +95,7 @@ impl AbstractMessages for MongoDb {
                         COL,
                         older_message_filter,
                         FindOptions::builder()
-                            .limit(limit / 2)
+                            .limit(limit / 2 + 1)
                             .sort(doc! {
                                 "_id": -1_i32
                             })

crates/core/database/src/models/users/model.rs

@@ -152,7 +152,7 @@ pub static DISCRIMINATOR_SEARCH_SPACE: Lazy<HashSet<String>> = Lazy::new(|| {
         .collect::<HashSet<String>>();
 
     for discrim in [
-        123, 1234, 1111, 2222, 3333, 4444, 5555, 6666, 7777, 8888, 9999,
+        123, 1234, 1111, 2222, 3333, 4444, 5555, 6666, 7777, 8888, 9999, 1488,
     ] {
         set.remove(&format!("{:0>4}", discrim));
     }
@@ -293,7 +293,14 @@ impl User {
         }
 
         // Ensure none of the following substrings show up in the username
-        const BLOCKED_SUBSTRINGS: &[&str] = &["```"];
+        const BLOCKED_SUBSTRINGS: &[&str] = &[
+            "```",
+            "discord.gg",
+            "rvlt.gg",
+            "guilded.gg",
+            "https://",
+            "http://",
+        ];
 
         for substr in BLOCKED_SUBSTRINGS {
             if username_lowercase.contains(substr) {

crates/core/files/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-files"
-version = "0.8.0"
+version = "0.8.2"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -20,10 +20,10 @@ typenum = "1.17.0"
 aws-config = "1.5.5"
 aws-sdk-s3 = { version = "1.46.0", features = ["behavior-version-latest"] }
 
-revolt-config = { version = "0.8.0", path = "../config", features = [
+revolt-config = { version = "0.8.2", path = "../config", features = [
     "report-macros",
 ] }
-revolt-result = { version = "0.8.0", path = "../result" }
+revolt-result = { version = "0.8.2", path = "../result" }
 
 # image processing
 jxl-oxide = "0.8.1"

crates/core/files/src/lib.rs

@@ -113,6 +113,23 @@ pub async fn upload_to_s3(bucket_id: &str, path: &str, buf: &[u8]) -> Result<Str
     Ok(BASE64_STANDARD.encode(nonce))
 }
 
+/// Delete a file from S3 by path
+pub async fn delete_from_s3(bucket_id: &str, path: &str) -> Result<()> {
+    let config = config().await;
+    let client = create_client(config.files.s3);
+
+    report_internal_error!(
+        client
+            .delete_object()
+            .bucket(bucket_id)
+            .key(path)
+            .send()
+            .await
+    )?;
+
+    Ok(())
+}
+
 /// Determine size of image at temp file
 pub fn image_size(f: &NamedTempFile) -> Option<(usize, usize)> {
     if let Ok(size) = imagesize::size(f.path())

crates/core/models/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-models"
-version = "0.8.0"
+version = "0.8.2"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -20,8 +20,8 @@ default = ["serde", "partials", "rocket"]
 
 [dependencies]
 # Core
-revolt-config = { version = "0.8.0", path = "../config" }
+revolt-config = { version = "0.8.2", path = "../config" }
-revolt-permissions = { version = "0.8.0", path = "../permissions" }
+revolt-permissions = { version = "0.8.2", path = "../permissions" }
 
 # Utility
 regex = "1"

crates/core/models/src/v0/messages.rs

@@ -280,7 +280,7 @@ auto_derived!(
     pub struct OptionsQueryMessages {
         /// Maximum number of messages to fetch
         ///
-        /// For fetching nearby messages, this is \`(limit + 1)\`.
+        /// For fetching nearby messages, this is \`(limit + 2)\`.
         #[cfg_attr(feature = "validator", validate(range(min = 1, max = 100)))]
         pub limit: Option<i64>,
         /// Message id before which messages should be fetched

crates/core/permissions/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-permissions"
-version = "0.8.0"
+version = "0.8.2"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]
@@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] }
 
 [dependencies]
 # Core
-revolt-result = { version = "0.8.0", path = "../result" }
+revolt-result = { version = "0.8.2", path = "../result" }
 
 # Utility
 auto_ops = "0.3.0"

crates/core/presence/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-presence"
-version = "0.8.0"
+version = "0.8.2"
 edition = "2021"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <me@insrt.uk>"]

crates/core/result/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-result"
-version = "0.8.0"
+version = "0.8.2"
 edition = "2021"
 license = "MIT"
 authors = ["Paul Makles <me@insrt.uk>"]

crates/core/result/src/axum.rs

@@ -42,6 +42,7 @@ impl IntoResponse for Error {
             ErrorType::InvalidRole => StatusCode::NOT_FOUND,
             ErrorType::Banned => StatusCode::FORBIDDEN,
             ErrorType::AlreadyInServer => StatusCode::CONFLICT,
+            ErrorType::CannotTimeoutYourself => StatusCode::BAD_REQUEST,
 
             ErrorType::TooManyServers { .. } => StatusCode::BAD_REQUEST,
             ErrorType::TooManyEmbeds { .. } => StatusCode::BAD_REQUEST,

crates/core/result/src/lib.rs

@@ -116,6 +116,7 @@ pub enum ErrorType {
         max: usize,
     },
     AlreadyInServer,
+    CannotTimeoutYourself,
 
     // ? Bot related errors
     ReachedMaximumBots,

crates/core/result/src/rocket.rs

@@ -47,6 +47,7 @@ impl<'r> Responder<'r, 'static> for Error {
             ErrorType::InvalidRole => Status::NotFound,
             ErrorType::Banned => Status::Forbidden,
             ErrorType::AlreadyInServer => Status::Conflict,
+            ErrorType::CannotTimeoutYourself => Status::BadRequest,
 
             ErrorType::TooManyServers { .. } => Status::BadRequest,
             ErrorType::TooManyEmbeds { .. } => Status::BadRequest,

crates/daemons/crond/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "revolt-crond"
+version = "0.8.2"
+license = "AGPL-3.0-or-later"
+authors = ["Paul Makles <me@insrt.uk>"]
+edition = "2021"
+description = "Revolt Daemon Service: Timed data clean up tasks"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+# Utility
+log = "0.4"
+
+# Async
+tokio = { version = "1" }
+
+# Core
+revolt-database = { version = "0.8.2", path = "../../core/database" }
+revolt-result = { version = "0.8.2", path = "../../core/result" }
+revolt-config = { version = "0.8.2", path = "../../core/config" }
+revolt-files = { version = "0.8.2", path = "../../core/files" }

crates/daemons/crond/Dockerfile

@@ -0,0 +1,10 @@
+# Build Stage
+FROM ghcr.io/revoltchat/base:latest AS builder
+FROM debian:12 AS debian
+
+# Bundle Stage
+FROM gcr.io/distroless/cc-debian12:nonroot
+COPY --from=builder /home/rust/src/target/release/revolt-crond ./
+
+USER nonroot
+CMD ["./revolt-crond"]

crates/daemons/crond/src/main.rs

@@ -0,0 +1,19 @@
+use revolt_config::configure;
+use revolt_database::DatabaseInfo;
+use revolt_result::Result;
+use tasks::{file_deletion, prune_dangling_files};
+use tokio::try_join;
+
+pub mod tasks;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    configure!(crond);
+
+    let db = DatabaseInfo::Auto.connect().await.expect("database");
+    try_join!(
+        file_deletion::task(db.clone()),
+        prune_dangling_files::task(db)
+    )
+    .map(|_| ())
+}

crates/daemons/crond/src/tasks/file_deletion.rs

@@ -0,0 +1,39 @@
+use std::time::Duration;
+
+use log::info;
+use revolt_database::Database;
+use revolt_files::delete_from_s3;
+use revolt_result::Result;
+use tokio::time::sleep;
+
+pub async fn task(db: Database) -> Result<()> {
+    loop {
+        let files = db.fetch_deleted_attachments().await?;
+
+        for file in files {
+            let count = db
+                .count_file_hash_references(file.hash.as_ref().expect("no `hash` present"))
+                .await?;
+
+            // No other files reference this file on disk anymore
+            if count <= 1 {
+                let file_hash = db
+                    .fetch_attachment_hash(file.hash.as_ref().expect("no `hash` present"))
+                    .await?;
+
+                // Delete from S3
+                delete_from_s3(&file_hash.bucket_id, &file_hash.path).await?;
+
+                // Delete the hash
+                db.delete_attachment_hash(&file_hash.id).await?;
+                info!("Deleted file hash {}", file_hash.id);
+            }
+
+            // Delete the file
+            db.delete_attachment(&file.id).await?;
+            info!("Deleted file {}", file.id);
+        }
+
+        sleep(Duration::from_secs(60)).await;
+    }
+}

crates/daemons/crond/src/tasks/mod.rs

@@ -0,0 +1,2 @@
+pub mod file_deletion;
+pub mod prune_dangling_files;

crates/daemons/crond/src/tasks/prune_dangling_files.rs

@@ -0,0 +1,36 @@
+use std::time::Duration;
+
+use revolt_database::{iso8601_timestamp::Timestamp, Database};
+use revolt_result::Result;
+use tokio::time::sleep;
+
+use log::info;
+
+pub async fn task(db: Database) -> Result<()> {
+    loop {
+        // This could just be a single database query
+        // ... but timestamps are inconsistently serialised
+        // ... sometimes they are dates/numbers, hard to query
+        // ... in the future, we could use Postgres instead! :D
+        // ...
+        // ... on the plus side, it's still only 2 queries
+
+        let files = db.fetch_dangling_files().await?;
+        let file_ids: Vec<String> = files
+            .into_iter()
+            .filter(|file| {
+                file.uploaded_at.is_some_and(|uploaded_at| {
+                    Timestamp::now_utc().duration_since(uploaded_at) > Duration::from_secs(60 * 60)
+                })
+            })
+            .map(|file| file.id)
+            .collect();
+
+        if !file_ids.is_empty() {
+            db.mark_attachments_as_deleted(&file_ids).await?;
+            info!("Marked {} dangling files for deletion", file_ids.len());
+        }
+
+        sleep(Duration::from_secs(60)).await;
+    }
+}

crates/daemons/pushd/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "revolt-pushd"
-version = "0.8.0"
+version = "0.8.2"
 edition = "2021"
 
 [dependencies]
-revolt-config = { version = "0.8.0", path = "../../core/config" }
+revolt-config = { version = "0.8.2", path = "../../core/config" }
-revolt-database = { version = "0.8.0", path = "../../core/database" }
+revolt-database = { version = "0.8.2", path = "../../core/database" }
-revolt-models = { version = "0.8.0", path = "../../core/models", features = [
+revolt-models = { version = "0.8.2", path = "../../core/models", features = [
     "validator",
 ] }
 

crates/delta/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-delta"
-version = "0.8.0"
+version = "0.8.2"
 license = "AGPL-3.0-or-later"
 authors = ["Paul Makles <paulmakles@gmail.com>"]
 edition = "2018"

crates/delta/src/main.rs

@@ -75,7 +75,15 @@ pub async fn web() -> Rocket<Build> {
     // Configure Swagger
     let swagger = revolt_rocket_okapi::swagger_ui::make_swagger_ui(
         &revolt_rocket_okapi::swagger_ui::SwaggerUIConfig {
-            url: "../openapi.json".to_owned(),
+            url: "/openapi.json".to_owned(),
+            ..Default::default()
+        },
+    )
+    .into();
+
+    let swagger_0_8 = revolt_rocket_okapi::swagger_ui::make_swagger_ui(
+        &revolt_rocket_okapi::swagger_ui::SwaggerUIConfig {
+            url: "/0.8/openapi.json".to_owned(),
             ..Default::default()
         },
     )
@@ -89,8 +97,12 @@ pub async fn web() -> Rocket<Build> {
         &config.rabbit.password,
     ))
     .await
-    .unwrap();
+    .expect("Failed to connect to RabbitMQ");
-    let channel = connection.open_channel(None).await.unwrap();
+
+    let channel = connection
+        .open_channel(None)
+        .await
+        .expect("Failed to open RabbitMQ channel");
 
     channel
         .exchange_declare(
@@ -116,6 +128,7 @@ pub async fn web() -> Rocket<Build> {
         .mount("/", rocket_cors::catch_all_options_routes())
         .mount("/", util::ratelimiter::routes())
         .mount("/swagger/", swagger)
+        .mount("/0.8/swagger/", swagger_0_8)
         .manage(authifier)
         .manage(db)
         .manage(amqp)

crates/delta/src/routes/channels/webhook_fetch_all.rs

@@ -22,7 +22,7 @@ pub async fn fetch_webhooks(
     let mut query = DatabasePermissionQuery::new(db, &user).channel(&channel);
     calculate_channel_permissions(&mut query)
         .await
-        .throw_if_lacking_channel_permission(ChannelPermission::ViewChannel)?;
+        .throw_if_lacking_channel_permission(ChannelPermission::ManageWebhooks)?;
 
     Ok(Json(
         db.fetch_webhooks_for_channel(channel.id())

crates/delta/src/routes/mod.rs

@@ -61,6 +61,47 @@ pub fn mount(config: Settings, mut rocket: Rocket<Build>) -> Rocket<Build> {
         };
     }
 
+    if config.features.webhooks_enabled {
+        mount_endpoints_and_merged_docs! {
+            rocket, "/0.8".to_owned(), settings,
+            "/" => (vec![], custom_openapi_spec()),
+            "" => openapi_get_routes_spec![root::root],
+            "/users" => users::routes(),
+            "/bots" => bots::routes(),
+            "/channels" => channels::routes(),
+            "/servers" => servers::routes(),
+            "/invites" => invites::routes(),
+            "/custom" => customisation::routes(),
+            "/safety" => safety::routes(),
+            "/auth/account" => rocket_authifier::routes::account::routes(),
+            "/auth/session" => rocket_authifier::routes::session::routes(),
+            "/auth/mfa" => rocket_authifier::routes::mfa::routes(),
+            "/onboard" => onboard::routes(),
+            "/push" => push::routes(),
+            "/sync" => sync::routes(),
+            "/webhooks" => webhooks::routes()
+        };
+    } else {
+        mount_endpoints_and_merged_docs! {
+            rocket, "/0.8".to_owned(), settings,
+            "/" => (vec![], custom_openapi_spec()),
+            "" => openapi_get_routes_spec![root::root],
+            "/users" => users::routes(),
+            "/bots" => bots::routes(),
+            "/channels" => channels::routes(),
+            "/servers" => servers::routes(),
+            "/invites" => invites::routes(),
+            "/custom" => customisation::routes(),
+            "/safety" => safety::routes(),
+            "/auth/account" => rocket_authifier::routes::account::routes(),
+            "/auth/session" => rocket_authifier::routes::session::routes(),
+            "/auth/mfa" => rocket_authifier::routes::mfa::routes(),
+            "/onboard" => onboard::routes(),
+            "/push" => push::routes(),
+            "/sync" => sync::routes()
+        };
+    }
+
     rocket
 }
 
@@ -190,10 +231,15 @@ fn custom_openapi_spec() -> OpenApi {
                 ..Default::default()
             },
             Server {
-                url: "http://local.revolt.chat:8000".to_owned(),
+                url: "http://local.revolt.chat:14702".to_owned(),
                 description: Some("Local Revolt Environment".to_owned()),
                 ..Default::default()
             },
+            Server {
+                url: "http://local.revolt.chat:14702/0.8".to_owned(),
+                description: Some("Local Revolt Environment (v0.8)".to_owned()),
+                ..Default::default()
+            },
         ],
         external_docs: Some(ExternalDocs {
             url: "https://developers.revolt.chat".to_owned(),

crates/delta/src/routes/servers/member_edit.rs

@@ -15,12 +15,12 @@ use validator::Validate;
 ///
 /// Edit a member by their id.
 #[openapi(tag = "Server Members")]
-#[patch("/<server>/members/<target>", data = "<data>")]
+#[patch("/<server>/members/<member>", data = "<data>")]
 pub async fn edit(
     db: &State<Database>,
     user: User,
     server: Reference,
-    target: Reference,
+    member: Reference,
     data: Json<v0::DataMemberEdit>,
 ) -> Result<Json<v0::Member>> {
     let data = data.into_inner();
@@ -30,9 +30,9 @@ pub async fn edit(
         })
     })?;
 
-    // Fetch server and target member
+    // Fetch server and member
     let mut server = server.as_server(db).await?;
-    let mut member = target.as_member(db, &server.id).await?;
+    let mut member = member.as_member(db, &server.id).await?;
 
     // Fetch our currrent permissions
     let mut query = DatabasePermissionQuery::new(db, &user).server(&server);
@@ -84,6 +84,10 @@ pub async fn edit(
             .map(|x| x.contains(&v0::FieldsMember::Timeout))
             .unwrap_or_default()
     {
+        if data.timeout.is_some() && member.id.user == user.id {
+            return Err(create_error!(CannotTimeoutYourself));
+        }
+
         permissions.throw_if_lacking_channel_permission(ChannelPermission::TimeoutMembers)?;
     }
 

crates/delta/src/util/ratelimiter.rs

@@ -94,14 +94,28 @@ pub struct Ratelimiter {
 ///
 /// Optionally, include a resource id to hash against.
 fn resolve_bucket<'r>(request: &'r rocket::Request<'_>) -> (&'r str, Option<&'r str>) {
-    if let Some(segment) = request.routed_segment(0) {
+    let (segment, resource, extra) = if matches!(request.routed_segment(0), Some("0.8")) {
-        let resource = request.routed_segment(1);
+        (
+            request.routed_segment(1),
+            request.routed_segment(2),
+            request.routed_segment(3),
+        )
+    } else {
+        (
+            request.routed_segment(0),
+            request.routed_segment(1),
+            request.routed_segment(2),
+        )
+    };
+
+    if let Some(segment) = segment {
+        let resource = resource;
 
         let method = request.method();
         match (segment, resource, method) {
             ("users", target, Method::Patch) => ("user_edit", target),
             ("users", _, _) => {
-                if let Some("default_avatar") = request.routed_segment(2) {
+                if let Some("default_avatar") = extra {
                     return ("default_avatar", None);
                 }
 
@@ -110,7 +124,7 @@ fn resolve_bucket<'r>(request: &'r rocket::Request<'_>) -> (&'r str, Option<&'r
             ("bots", _, _) => ("bots", None),
             ("channels", Some(id), _) => {
                 if request.method() == Method::Post {
-                    if let Some("messages") = request.routed_segment(2) {
+                    if let Some("messages") = extra {
                         return ("messaging", Some(id));
                     }
                 }

crates/services/autumn/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-autumn"
-version = "0.8.0"
+version = "0.8.2"
 edition = "2021"
 
 [dependencies]
@@ -42,12 +42,12 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 # Core crates
-revolt-files = { version = "0.8.0", path = "../../core/files" }
+revolt-files = { version = "0.8.2", path = "../../core/files" }
-revolt-config = { version = "0.8.0", path = "../../core/config" }
+revolt-config = { version = "0.8.2", path = "../../core/config" }
-revolt-database = { version = "0.8.0", path = "../../core/database", features = [
+revolt-database = { version = "0.8.2", path = "../../core/database", features = [
     "axum-impl",
 ] }
-revolt-result = { version = "0.8.0", path = "../../core/result", features = [
+revolt-result = { version = "0.8.2", path = "../../core/result", features = [
     "utoipa",
     "axum",
 ] }

crates/services/january/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "revolt-january"
-version = "0.8.0"
+version = "0.8.2"
 edition = "2021"
 
 [dependencies]
@@ -31,13 +31,13 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 # Core crates
-revolt-config = { version = "0.8.0", path = "../../core/config" }
+revolt-config = { version = "0.8.2", path = "../../core/config" }
-revolt-models = { version = "0.8.0", path = "../../core/models" }
+revolt-models = { version = "0.8.2", path = "../../core/models" }
-revolt-result = { version = "0.8.0", path = "../../core/result", features = [
+revolt-result = { version = "0.8.2", path = "../../core/result", features = [
     "utoipa",
     "axum",
 ] }
-revolt-files = { version = "0.8.0", path = "../../core/files" }
+revolt-files = { version = "0.8.2", path = "../../core/files" }
 
 # Axum / web server
 axum = { version = "0.7.5" }

default.nix

@@ -1,26 +1,10 @@
-let
+{ pkgs ? import <nixpkgs> {} }:
-  # Pinned nixpkgs, deterministic. Last updated: 28-07-2024.
-  pkgs = import (fetchTarball("https://github.com/NixOS/nixpkgs/archive/9b34ca580417e1ebc56c4df57d8b387dad686665.tar.gz")) {};
-
-  # Rolling updates, not deterministic.
-  # pkgs = import (fetchTarball("channel:nixpkgs-unstable")) {};
-in pkgs.mkShell {
-  name = "revoltEnv";
-
-  # LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
-  #   pkgs.gcc-unwrapped
-  #   pkgs.zlib
-  #   pkgs.glib
-  #   pkgs.libGL
-  # ];
 
+pkgs.mkShell rec {
   buildInputs = [
     # Tools
     pkgs.git
 
-    # Database
-    # pkgs.mongodb
-
     # Cargo
     pkgs.cargo
     pkgs.cargo-nextest

scripts/build-image-layer.sh

@@ -34,12 +34,14 @@ deps() {
     crates/core/result/src \
     crates/services/autumn/src \
     crates/services/january/src \
+    crates/daemons/crond/src \
     crates/daemons/pushd/src
   echo 'fn main() { panic!("stub"); }' |
     tee crates/bonfire/src/main.rs |
     tee crates/delta/src/main.rs |
     tee crates/services/autumn/src/main.rs |
     tee crates/services/january/src/main.rs |
+    tee crates/daemons/crond/src/main.rs |
     tee crates/daemons/pushd/src/main.rs
   echo '' |
     tee crates/bindings/node/src/lib.rs |
@@ -62,6 +64,7 @@ apps() {
   touch -am \
     crates/bonfire/src/main.rs \
     crates/delta/src/main.rs \
+    crates/daemons/crond/src/main.rs \
     crates/daemons/pushd/src/main.rs \
     crates/core/config/src/lib.rs \
     crates/core/database/src/lib.rs \

scripts/publish-debug-image.sh

@@ -25,6 +25,8 @@ docker build -t ghcr.io/revoltchat/server:$TAG - < crates/delta/Dockerfile
 docker build -t ghcr.io/revoltchat/bonfire:$TAG - < crates/bonfire/Dockerfile
 docker build -t ghcr.io/revoltchat/autumn:$TAG - < crates/services/autumn/Dockerfile
 docker build -t ghcr.io/revoltchat/january:$TAG - < crates/services/january/Dockerfile
+docker build -t ghcr.io/revoltchat/crond:$TAG - < crates/daemons/crond/Dockerfile
+docker build -t ghcr.io/revoltchat/pushd:$TAG - < crates/daemons/pushd/Dockerfile
 
 if [ "$DEBUG" = "true" ]; then
   git restore Cargo.toml
@@ -34,3 +36,5 @@ docker push ghcr.io/revoltchat/server:$TAG
 docker push ghcr.io/revoltchat/bonfire:$TAG
 docker push ghcr.io/revoltchat/autumn:$TAG
 docker push ghcr.io/revoltchat/january:$TAG
+docker push ghcr.io/revoltchat/crond:$TAG
+docker push ghcr.io/revoltchat/pushd:$TAG