ci: bring our own runner

trying to fix build cancellations

.github/workflows/docker.yaml

@@ -27,12 +27,14 @@ permissions:
 
 jobs:
   base:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     name: Build base image
     steps:
       # Configure build environment
       - name: Checkout
         uses: actions/checkout@v3
+      - name: Fix https://github.com/actions/runner-images/issues/9959
+        run: sudo apt-get autopurge -y needrestart
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
@@ -57,7 +59,7 @@ jobs:
 
   publish:
     needs: [base]
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     if: github.event_name != 'pull_request'
     strategy:
       matrix:
@@ -67,6 +69,8 @@ jobs:
       # Configure build environment
       - name: Checkout
         uses: actions/checkout@v3
+      - name: Fix https://github.com/actions/runner-images/issues/9959
+        run: sudo apt-get autopurge -y needrestart
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 

.github/workflows/rust.yaml

@@ -67,7 +67,7 @@ jobs:
         if: github.event_name != 'pull_request' && github.ref_name == 'main'
         uses: nev7n/wait_for_response@v1
         with:
-          url: "http://localhost:8000/"
+          url: "http://localhost:14702/"
 
       - name: Checkout API repository
         if: github.event_name != 'pull_request' && github.ref_name == 'main'
@@ -79,7 +79,7 @@ jobs:
 
       - name: Download OpenAPI specification
         if: github.event_name != 'pull_request' && github.ref_name == 'main'
-        run: curl http://localhost:8000/openapi.json -o api/OpenAPI.json
+        run: curl http://localhost:14702/openapi.json -o api/OpenAPI.json
 
       - name: Commit changes
         if: github.event_name != 'pull_request' && github.ref_name == 'main'

.gitignore

@@ -7,3 +7,5 @@ target
 
 .vercel
 .DS_Store
+
+.idea

Cargo.toml

@@ -13,3 +13,8 @@ members = [
 [patch.crates-io]
 redis22 = { package = "redis", version = "0.22.3", git = "https://github.com/revoltchat/redis-rs", rev = "1a41faf356fd21aebba71cea7eb7eb2653e5f0ef" }
 redis23 = { package = "redis", version = "0.23.1", git = "https://github.com/revoltchat/redis-rs", rev = "f8ca28ab85da59d2ccde526b4d2fb390eff5a5f9" }
+# authifier = { package = "authifier", version = "1.0.8", path = "../authifier/crates/authifier" }
+# rocket_authifier = { package = "rocket_authifier", version = "1.0.8", path = "../authifier/crates/rocket_authifier" }
+
+[profile.release]
+lto = true

Dockerfile.useCurrentArch

@@ -25,6 +25,7 @@ COPY crates/core/presence/Cargo.toml ./crates/core/presence/
 COPY crates/core/result/Cargo.toml ./crates/core/result/
 COPY crates/services/autumn/Cargo.toml ./crates/services/autumn/
 COPY crates/services/january/Cargo.toml ./crates/services/january/
+COPY crates/daemons/pushd/Cargo.toml ./crates/daemons/pushd/
 RUN sh /tmp/build-image-layer.sh deps
 
 # Build all apps

crates/core/config/src/lib.rs

@@ -23,6 +23,17 @@ macro_rules! report_error {
     };
 }
 
+#[cfg(feature = "report-macros")]
+#[macro_export]
+macro_rules! capture_internal_error {
+    ( $expr: expr ) => {
+        $crate::capture_message(
+            &format!("{:?} ({}:{}:{})", $expr, file!(), line!(), column!()),
+            $crate::Level::Error,
+        );
+    };
+}
+
 #[cfg(feature = "report-macros")]
 #[macro_export]
 macro_rules! report_internal_error {

crates/core/database/src/lib.rs

@@ -25,6 +25,26 @@ pub use mongodb;
 #[macro_use]
 extern crate bson;
 
+#[macro_export]
+#[cfg(debug_assertions)]
+macro_rules! query {
+    ( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
+        Ok($self.$type($collection, $($rest),+).await.unwrap())
+    };
+}
+
+#[macro_export]
+#[cfg(not(debug_assertions))]
+macro_rules! query {
+    ( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
+        $self.$type($collection, $($rest),+).await
+            .map_err(|err| {
+                revolt_config::capture_internal_error!(err);
+                create_database_error!(stringify!($type), $collection)
+            })
+    };
+}
+
 macro_rules! database_derived {
     ( $( $item:item )+ ) => {
         $(

crates/core/database/src/models/file_hashes/model.rs

@@ -45,7 +45,7 @@ auto_derived!(
         Image {
             width: isize,
             height: isize,
-            // animated: bool // TODO: https://docs.rs/image/latest/image/trait.AnimationDecoder.html
+            // animated: bool // TODO: https://docs.rs/image/latest/image/trait.AnimationDecoder.html for APNG support
         },
         /// File is a video with specific dimensions
         Video { width: isize, height: isize },

crates/core/database/src/models/messages/ops/mongodb.rs

@@ -171,7 +171,7 @@ impl AbstractMessages for MongoDb {
         self.find_with_options(
             COL,
             doc! {
-                "ids": {
+                "_id": {
                     "$in": ids
                 }
             },

crates/core/result/src/lib.rs

@@ -189,23 +189,6 @@ macro_rules! create_database_error {
     };
 }
 
-#[macro_export]
-#[cfg(debug_assertions)]
-macro_rules! query {
-    ( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
-        Ok($self.$type($collection, $($rest),+).await.unwrap())
-    };
-}
-
-#[macro_export]
-#[cfg(not(debug_assertions))]
-macro_rules! query {
-    ( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
-        $self.$type($collection, $($rest),+).await
-            .map_err(|_| create_database_error!(stringify!($type), $collection))
-    };
-}
-
 #[cfg(test)]
 mod tests {
     use crate::ErrorType;

crates/daemons/pushd/Dockerfile

@@ -1,9 +1,11 @@
 # Build Stage
 FROM ghcr.io/revoltchat/base:latest AS builder
+FROM debian:12 AS debian
 
 # Bundle Stage
 FROM gcr.io/distroless/cc-debian12:nonroot
 COPY --from=builder /home/rust/src/target/release/revolt-pushd ./
+COPY --from=debian /usr/bin/uname /usr/bin/uname
 
 USER nonroot
 CMD ["./revolt-pushd"]

crates/delta/Dockerfile

@@ -1,11 +1,13 @@
 # Build Stage
 FROM ghcr.io/revoltchat/base:latest AS builder
+FROM debian:12 AS debian
 
 # Bundle Stage
 FROM gcr.io/distroless/cc-debian12:nonroot
 COPY --from=builder /home/rust/src/target/release/revolt-delta ./
+COPY --from=debian /usr/bin/uname /usr/bin/uname
 
-EXPOSE 8000
-ENV ROCKET_ADDRESS 0.0.0.0
+EXPOSE 14702
+ENV ROCKET_ADDRESS=0.0.0.0
 USER nonroot
 CMD ["./revolt-delta"]

crates/services/autumn/src/api.rs

@@ -64,7 +64,8 @@ lazy_static! {
         })
         // TODO config
         // .max_capacity(1024 * 1024 * 1024) // Cache up to 1GiB in memory
-        .max_capacity(512 * 1024 * 1024) // Cache up to 512MiB in memory
+        // .max_capacity(512 * 1024 * 1024) // Cache up to 512MiB in memory
+        .max_capacity(2 * 1024 * 1024 * 1024) // Cache up to 2GiB in memory
         .time_to_live(Duration::from_secs(5 * 60)) // For up to 5 minutes
         .build();
 }
@@ -215,6 +216,27 @@ async fn upload_file(
         nanoid::nanoid!(42)
     };
 
+    // Determine the mime type for the file
+    let mime_type = determine_mime_type(&mut file.contents, &buf, &filename);
+
+    // Check blocklist for mime type
+    if config
+        .files
+        .blocked_mime_types
+        .iter()
+        .any(|m| m == mime_type)
+    {
+        return Err(create_error!(FileTypeNotAllowed));
+    }
+
+    // Determine metadata for the file
+    let metadata = generate_metadata(&file.contents, mime_type);
+
+    // Block non-images for non-attachment uploads
+    if !matches!(tag, Tag::attachments) && !matches!(metadata, Metadata::Image { .. }) {
+        return Err(create_error!(FileTypeNotAllowed));
+    }
+
     // Find an existing hash and use that if possible
     let file_hash_exists = if let Ok(file_hash) = db
         .fetch_attachment_hash(&format!("{original_hash:02x}"))
@@ -238,22 +260,6 @@ async fn upload_file(
         false
     };
 
-    // Determine the mime type for the file
-    let mime_type = determine_mime_type(&mut file.contents, &buf, &filename);
-
-    // Check blocklist for mime type
-    if config
-        .files
-        .blocked_mime_types
-        .iter()
-        .any(|m| m == mime_type)
-    {
-        return Err(create_error!(FileTypeNotAllowed));
-    }
-
-    // Determine metadata for the file
-    let metadata = generate_metadata(&file.contents, mime_type);
-
     // Strip metadata
     let (buf, metadata) = strip_metadata(file.contents, buf, metadata, mime_type).await?;
 
@@ -321,21 +327,23 @@ pub static CACHE_CONTROL: &str = "public, max-age=604800, must-revalidate";
 
 /// Fetch preview of file
 ///
-/// This route will only return image content.
+/// This route will only return image content. <br>
 /// For all other file types, please use the fetch route (you will receive a redirect if you try to use this route anyways!).
 ///
 /// Depending on the given tag, the file will be re-processed to fit the criteria:
 ///
-/// | Tag | Image Resolution <sup>†</sup> |
-/// | :-: | --- |
-/// | attachments | Up to 1280px on any axis |
-/// | avatars | Up to 128px on any axis |
-/// | backgrounds | Up to 1280x720px |
-/// | icons | Up to 128px on any axis |
-/// | banners | Up to 480px on any axis |
-/// | emojis | Up to 128px on any axis |
+/// | Tag | Image Resolution <sup>†</sup> | Animations stripped by preview <sup>‡</sup> |
+/// | :-: | --- | :-: |
+/// | attachments | Up to 1280px on any axis | ❌ |
+/// | avatars | Up to 128px on any axis | ✅ |
+/// | backgrounds | Up to 1280x720px | ❌ |
+/// | icons | Up to 128px on any axis | ✅ |
+/// | banners | Up to 480px on any axis | ❌ |
+/// | emojis | Up to 128px on any axis | ❌ |
 ///
 /// <sup>†</sup> aspect ratio will always be preserved
+///
+/// <sup>‡</sup> to fetch animated variant, suffix `/{file_name}` or `/original` to the path
 #[utoipa::path(
     get,
     path = "/{tag}/{file_id}",
@@ -351,8 +359,8 @@ async fn fetch_preview(
     State(db): State<Database>,
     Path((tag, file_id)): Path<(Tag, String)>,
 ) -> Result<Response> {
-    let tag: &'static str = tag.into();
-    let file = db.fetch_attachment(tag, &file_id).await?;
+    let tag_str: &'static str = tag.clone().into();
+    let file = db.fetch_attachment(tag_str, &file_id).await?;
 
     // Ignore deleted files
     if file.deleted.is_some_and(|v| v) {
@@ -366,10 +374,14 @@ async fn fetch_preview(
 
     let hash = file.as_hash(&db).await?;
 
-    // Only process image files
-    if !matches!(hash.metadata, Metadata::Image { .. }) {
+    let is_animated = hash.content_type == "image/gif"; // TODO: extract this data from files
+
+    // Only process image files and don't process GIFs if not avatar or icon
+    if !matches!(hash.metadata, Metadata::Image { .. })
+        || (is_animated && !matches!(tag, Tag::avatars | Tag::icons))
+    {
         return Ok(
-            Redirect::permanent(&format!("/{tag}/{file_id}/{}", file.filename)).into_response(),
+            Redirect::permanent(&format!("/{tag_str}/{file_id}/{}", file.filename)).into_response(),
         );
     }
 
@@ -379,7 +391,7 @@ async fn fetch_preview(
     // Read image and create thumbnail
     let data = create_thumbnail(
         decode_image(&mut Cursor::new(data), &file.content_type)?,
-        tag,
+        tag_str,
     )
     .await;
 
@@ -397,6 +409,8 @@ async fn fetch_preview(
 /// Fetch original file
 ///
 /// Content disposition header will be set to 'attachment' to prevent browser from rendering anything.
+///
+/// Using `original` as the file name parameter will redirect you to the original file.
 #[utoipa::path(
     get,
     path = "/{tag}/{file_id}/{file_name}",
@@ -413,7 +427,8 @@ async fn fetch_file(
     State(db): State<Database>,
     Path((tag, file_id, file_name)): Path<(Tag, String, String)>,
 ) -> Result<Response> {
-    let file = db.fetch_attachment(tag.into(), &file_id).await?;
+    let tag: &'static str = tag.clone().into();
+    let file = db.fetch_attachment(tag, &file_id).await?;
 
     // Ignore deleted files
     if file.deleted.is_some_and(|v| v) {
@@ -427,6 +442,12 @@ async fn fetch_file(
 
     // Ensure filename is correct
     if file_name != file.filename {
+        if file_name == "original" {
+            return Ok(
+                Redirect::permanent(&format!("/{tag}/{file_id}/{}", file.filename)).into_response(),
+            );
+        }
+
         return Err(create_error!(NotFound));
     }