jmug/stoatchat
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/admin-api

Browse Source 
# Conflicts:
#	crates/core/database/src/util/mod.rs
#	crates/core/result/src/axum.rs
This commit is contained in:
ispik
2026-05-23 13:26:21 +03:00
parent 2d8b7481bc 7937179db7
commit 93a00ffbf8
94 changed files with 7108 additions and 3102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
.github/workflows/docker-cleanup.yaml vendored Normal file
View File

@@ -0,0 +1,46 @@
name: Docker PR Image Cleanup
on:
pull_request:
types:
- closed
permissions:
contents: read
packages: write
concurrency:
group: docker-cleanup-${{ github.event.pull_request.number }}
cancel-in-progress: false
jobs:
cleanup:
runs-on: ubuntu-latest
if: ${{ !github.event.pull_request.head.repo.fork }}
strategy:
fail-fast: false
matrix:
package:
- base
- api
- events
- file-server
- proxy
- gifbox
- crond
- pushd
- voice-ingress
steps:
- env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ORG: stoatchat
PACKAGE: ${{ matrix.package }}
TAG: pr-${{ github.event.pull_request.number }}
run: |
set -euo pipefail
gh api --paginate \
"/orgs/${ORG}/packages/container/${PACKAGE}/versions" \
--jq ".[] | select(.metadata.container.tags | index(\"${TAG}\")) | .id" \
| while read -r id; do
gh api -X DELETE "/orgs/${ORG}/packages/container/${PACKAGE}/versions/${id}"
done

37
.github/workflows/docker.yaml vendored
View File

@@ -5,8 +5,6 @@ on:
tags:
- "*"
pull_request:
paths:
- "Dockerfile"
workflow_dispatch:
permissions:
@@ -19,9 +17,9 @@ concurrency:
jobs:
base:
name: Test base image build
name: Test base image build (fork)
runs-on: arc-runner-set
if: github.event_name == 'pull_request'
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork
steps:
# Configure build environment
- name: Checkout
@@ -42,7 +40,7 @@ jobs:
publish:
runs-on: arc-runner-set
if: github.event_name != 'pull_request'
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }}
name: Publish Docker images
steps:
# Configure build environment
@@ -59,6 +57,15 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Determine base image tag
id: base
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
echo "tag=pr-${{ github.event.number }}" >> "$GITHUB_OUTPUT"
else
echo "tag=latest" >> "$GITHUB_OUTPUT"
fi
# Build the image
- name: Build base image
uses: docker/build-push-action@v4
@@ -66,7 +73,9 @@ jobs:
context: .
push: true
platforms: linux/amd64,linux/arm64
tags: ghcr.io/${{ github.repository_owner }}/base:latest
tags: ghcr.io/${{ github.repository_owner }}/base:${{ steps.base.outputs.tag }}
cache-from: type=gha,scope=buildx-base-multi-arch
cache-to: type=gha,scope=buildx-base-multi-arch,mode=max
# stoatchat/api
- name: Docker meta
@@ -84,7 +93,7 @@ jobs:
file: crates/delta/Dockerfile
tags: ${{ steps.meta-delta.outputs.tags }}
build-args: |
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:${{ steps.base.outputs.tag }}
labels: ${{ steps.meta-delta.outputs.labels }}
# stoatchat/events
@@ -103,7 +112,7 @@ jobs:
file: crates/bonfire/Dockerfile
tags: ${{ steps.meta-bonfire.outputs.tags }}
build-args: |
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:${{ steps.base.outputs.tag }}
labels: ${{ steps.meta-bonfire.outputs.labels }}
# stoatchat/file-server
@@ -122,7 +131,7 @@ jobs:
file: crates/services/autumn/Dockerfile
tags: ${{ steps.meta-autumn.outputs.tags }}
build-args: |
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:${{ steps.base.outputs.tag }}
labels: ${{ steps.meta-autumn.outputs.labels }}
# stoatchat/proxy
@@ -141,7 +150,7 @@ jobs:
file: crates/services/january/Dockerfile
tags: ${{ steps.meta-january.outputs.tags }}
build-args: |
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:${{ steps.base.outputs.tag }}
labels: ${{ steps.meta-january.outputs.labels }}
# stoatchat/gifbox
@@ -160,7 +169,7 @@ jobs:
file: crates/services/gifbox/Dockerfile
tags: ${{ steps.meta-gifbox.outputs.tags }}
build-args: |
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:${{ steps.base.outputs.tag }}
labels: ${{ steps.meta-gifbox.outputs.labels }}
# stoatchat/crond
@@ -179,7 +188,7 @@ jobs:
file: crates/daemons/crond/Dockerfile
tags: ${{ steps.meta-crond.outputs.tags }}
build-args: |
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:${{ steps.base.outputs.tag }}
labels: ${{ steps.meta-crond.outputs.labels }}
# stoatchat/pushd
@@ -198,7 +207,7 @@ jobs:
file: crates/daemons/pushd/Dockerfile
tags: ${{ steps.meta-pushd.outputs.tags }}
build-args: |
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:${{ steps.base.outputs.tag }}
labels: ${{ steps.meta-pushd.outputs.labels }}
# stoatchat/voice-ingress
@@ -217,5 +226,5 @@ jobs:
file: crates/daemons/voice-ingress/Dockerfile
tags: ${{ steps.meta-voice-ingress.outputs.tags }}
build-args: |
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:${{ steps.base.outputs.tag }}
labels: ${{ steps.meta-voice-ingress.outputs.labels }}

2
.github/workflows/publish-crates.yml vendored
View File

@@ -21,4 +21,6 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Publish
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
run: mise publish --workspace

2
.github/workflows/rust.yaml vendored
View File

@@ -37,6 +37,7 @@ jobs:
- name: Reference Test
env:
TEST_DB: REFERENCE
continue-on-error: ${{ github.ref_name == 'main' }}
run: |
mise test
@@ -44,6 +45,7 @@ jobs:
env:
TEST_DB: MONGODB
MONGODB: mongodb://localhost
continue-on-error: ${{ github.ref_name == 'main' }}
run: |
mise test

2
.release-please-manifest.json
View File

@@ -1,3 +1,3 @@
{
".": "0.12.1"
".": "0.13.7"
}

92
CHANGELOG.md
View File

@@ -1,5 +1,97 @@
# Changelog
## [0.13.7](https://github.com/stoatchat/stoatchat/compare/v0.13.6...v0.13.7) (2026-05-21)
### Bug Fixes
* sanitize emoji input to handle variation selectors ([#774](https://github.com/stoatchat/stoatchat/issues/774)) ([2d308e0](https://github.com/stoatchat/stoatchat/commit/2d308e03d58c19f27b5b4d65dc2a15ef20b56190))
* update mention count badge for channel acks ([#769](https://github.com/stoatchat/stoatchat/issues/769)) ([0d9ae50](https://github.com/stoatchat/stoatchat/commit/0d9ae508d9d2199f0e408b8ca634d20489be6f61))
## [0.13.6](https://github.com/stoatchat/stoatchat/compare/v0.13.5...v0.13.6) (2026-05-18)
### Features
* Update FCM payload for android notifications ([#766](https://github.com/stoatchat/stoatchat/issues/766)) ([acbc087](https://github.com/stoatchat/stoatchat/commit/acbc087982e9aeb05cabc5ab4c9b1291f67490ad))
* user slowmode events ([#760](https://github.com/stoatchat/stoatchat/issues/760)) ([af0d8aa](https://github.com/stoatchat/stoatchat/commit/af0d8aad14dc68d88159d0e1c714077d362e21e4))
### Bug Fixes
* include `minio` region as tests need it ([#761](https://github.com/stoatchat/stoatchat/issues/761)) ([298742d](https://github.com/stoatchat/stoatchat/commit/298742dbad4eafae356f976c56b9db23904b0c3a))
* set env var for publishing crates ([#768](https://github.com/stoatchat/stoatchat/issues/768)) ([018afaf](https://github.com/stoatchat/stoatchat/commit/018afaf38f6330d92dad2a68b640c0cb3f6b639a))
* Use proper headers to determine IP when not behind cloudflare ([#764](https://github.com/stoatchat/stoatchat/issues/764)) ([494c8b7](https://github.com/stoatchat/stoatchat/commit/494c8b7cabaae2a51039a7a5b559d5e2e5279554))
* voice ingress crashing due to new Result in AMQP::new_auto() ([#765](https://github.com/stoatchat/stoatchat/issues/765)) ([2871632](https://github.com/stoatchat/stoatchat/commit/2871632382395cb20cbe0047c542d3ac31ff3f03))
### Miscellaneous Chores
* switch to lapin ([#767](https://github.com/stoatchat/stoatchat/issues/767)) ([5b19853](https://github.com/stoatchat/stoatchat/commit/5b1985381ae829a92c80a19e91a414cd9dc4de93))
## [0.13.5](https://github.com/stoatchat/stoatchat/compare/v0.13.4...v0.13.5) (2026-05-17)
### Bug Fixes
* dont panic on hash missing when deleting files ([#755](https://github.com/stoatchat/stoatchat/issues/755)) ([c902077](https://github.com/stoatchat/stoatchat/commit/c902077cf51076fee11712eb732dc8a8f786fc4b))
## [0.13.4](https://github.com/stoatchat/stoatchat/compare/v0.13.3...v0.13.4) (2026-05-16)
### Bug Fixes
* add TLS feature to livekit-api crate ([#753](https://github.com/stoatchat/stoatchat/issues/753)) ([6cfee1f](https://github.com/stoatchat/stoatchat/commit/6cfee1f601c1e084df7c8f1e7a5e8a560d1dd514))
## [0.13.3](https://github.com/stoatchat/stoatchat/compare/v0.13.2...v0.13.3) (2026-05-15)
### Bug Fixes
* don't automatically set up rabbitmq in delta ([#749](https://github.com/stoatchat/stoatchat/issues/749)) ([7647cfc](https://github.com/stoatchat/stoatchat/commit/7647cfc8d93aba99f5faef13eb3d970097540d76))
* don't declare queues which seem to cause the backend to crash in prod ([7647cfc](https://github.com/stoatchat/stoatchat/commit/7647cfc8d93aba99f5faef13eb3d970097540d76))
## [0.13.2](https://github.com/stoatchat/stoatchat/compare/v0.13.1...v0.13.2) (2026-05-11)
### Bug Fixes
* update default exchange to `revolt.default` ([#746](https://github.com/stoatchat/stoatchat/issues/746)) ([fcb8091](https://github.com/stoatchat/stoatchat/commit/fcb8091cd7a00d7f26c798daa33aae4b923b2a8b))
## [0.13.1](https://github.com/stoatchat/stoatchat/compare/v0.13.0...v0.13.1) (2026-05-10)
### Bug Fixes
* amqprs startup bug ([#744](https://github.com/stoatchat/stoatchat/issues/744)) ([1100eaf](https://github.com/stoatchat/stoatchat/commit/1100eaf46f849f2509ae01ac497556ca33bde778))
## [0.13.0](https://github.com/stoatchat/stoatchat/compare/v0.12.1...v0.13.0) (2026-05-08)
### Features
* add embed support for YouTube Shorts ([#734](https://github.com/stoatchat/stoatchat/issues/734)) ([d46c7f7](https://github.com/stoatchat/stoatchat/commit/d46c7f7f3c04524c0639c3e0a122626f8e0b3bf7))
* add emoji rename endpoint ([#714](https://github.com/stoatchat/stoatchat/issues/714)) ([23ad135](https://github.com/stoatchat/stoatchat/commit/23ad1359834bb7d07a460b8678d6a6ebffc73eb0))
* add legal links to root payload ([#733](https://github.com/stoatchat/stoatchat/issues/733)) ([21d8201](https://github.com/stoatchat/stoatchat/commit/21d82018cf84ab0fdd10613d254b9562aea8eea3))
* add role icon support ([#724](https://github.com/stoatchat/stoatchat/issues/724)) ([841985d](https://github.com/stoatchat/stoatchat/commit/841985d3b994df1c6eefab2fc7ecbd77ab22c493))
* Add webhook endpoints for editing and deleting messages ([#682](https://github.com/stoatchat/stoatchat/issues/682)) ([6f3441c](https://github.com/stoatchat/stoatchat/commit/6f3441cf4acac2a8e6e1bf07a279a153b80f7956))
* automatically sanitise usernames on create/update ([#689](https://github.com/stoatchat/stoatchat/issues/689)) ([e937697](https://github.com/stoatchat/stoatchat/commit/e93769786c7669485a659ee471630740d3cea702))
* blacklist private ip ranges and add january domain blocklist ([#731](https://github.com/stoatchat/stoatchat/issues/731)) ([6b41db9](https://github.com/stoatchat/stoatchat/commit/6b41db984bb491b2e58324309cc70d8c14e0b814))
* Rewrite acks ([#741](https://github.com/stoatchat/stoatchat/issues/741)) ([ab5bd47](https://github.com/stoatchat/stoatchat/commit/ab5bd47a39ee889de0b5ae6e7b560620853daead))
### Bug Fixes
* add new_user_hours to configuration limits ([#729](https://github.com/stoatchat/stoatchat/issues/729)) ([279f5d5](https://github.com/stoatchat/stoatchat/commit/279f5d5fd7af2df55902c706859ec07f569cdb1e))
* add reconnection policy to Redis subscriber to prevent ghost state ([#708](https://github.com/stoatchat/stoatchat/issues/708)) ([057f2bb](https://github.com/stoatchat/stoatchat/commit/057f2bb8b359f8b942741a30ff54eeb8fbe3e0b1))
* docker compose file had personal url in it ([#742](https://github.com/stoatchat/stoatchat/issues/742)) ([0719985](https://github.com/stoatchat/stoatchat/commit/0719985ac5636590f91e6f9ec4b68f3eded70c13))
* don't strip ICC from exif ([#735](https://github.com/stoatchat/stoatchat/issues/735)) ([d76a711](https://github.com/stoatchat/stoatchat/commit/d76a71141f3e508f6308ba52fa28eaeb56fb3438))
* dont send notification in fcm ([#721](https://github.com/stoatchat/stoatchat/issues/721)) ([89171e9](https://github.com/stoatchat/stoatchat/commit/89171e9bd0f15711157e78c6eec0fe7b480de93a))
* encode filenames in redirects ([#737](https://github.com/stoatchat/stoatchat/issues/737)) ([9fd7128](https://github.com/stoatchat/stoatchat/commit/9fd7128f800badbd184baf943d4f799e601201e4))
* january ip redirects & domain resolver ([#738](https://github.com/stoatchat/stoatchat/issues/738)) ([356491e](https://github.com/stoatchat/stoatchat/commit/356491e934b274f9e895df883dd63ef0b3123510))
* update message length validation to remove upper limit ([#723](https://github.com/stoatchat/stoatchat/issues/723)) ([ed4fd5e](https://github.com/stoatchat/stoatchat/commit/ed4fd5ebfe6d0ea534a0898da4afdc1f4e2cd6c5))
* use correct response for NoEffect errors ([#732](https://github.com/stoatchat/stoatchat/issues/732)) ([5378cd2](https://github.com/stoatchat/stoatchat/commit/5378cd22b4c7d85f44c31a6af0dda00941b80d5c))
## [0.12.1](https://github.com/stoatchat/stoatchat/compare/v0.12.0...v0.12.1) (2026-04-10)

1014
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

24
Cargo.toml
View File

@@ -134,6 +134,7 @@ kamadak-exif = "0.5.4"
webp = "0.3.0"
image = "0.25.2" # avif encode requires dav1d system library: features = ["avif-native"]
thumbhash = "0.1.0"
lcms2 = "6.1.1" # for color profile processing
# File processing
revolt_clamav-client = "0.1.5"
@@ -158,7 +159,7 @@ opentelemetry-appender-tracing = "0.31.1"
authifier = "1.0.16"
# RabbitMQ
amqprs = "1.7.0"
lapin = "4.7.1"
# Voice
livekit-api = "0.4.4"
@@ -184,18 +185,19 @@ url = "2.2.2"
impl_ops = "0.1.1"
lazy_static = "1.5.0"
mime = "0.3.17"
futures-lite = "2.6.1"
# Build Dependencies
vergen = "7.5.0"
# Local packages
revolt-coalesced = { version = "0.12.0", path = "crates/core/coalesced" }
revolt-config = { version = "0.12.0", path = "crates/core/config" }
revolt-database = { version = "0.12.0", path = "crates/core/database" }
revolt-files = { version = "0.12.0", path = "crates/core/files" }
revolt-models = { version = "0.12.0", path = "crates/core/models" }
revolt-parser = { version = "0.12.0", path = "crates/core/parser" }
revolt-permissions = { version = "0.12.0", path = "crates/core/permissions" }
revolt-presence = { version = "0.12.0", path = "crates/core/presence" }
revolt-ratelimits = { version = "0.12.0", path = "crates/core/ratelimits" }
revolt-result = { version = "0.12.0", path = "crates/core/result" }
revolt-coalesced = { version = "0.13.7", path = "crates/core/coalesced" }
revolt-config = { version = "0.13.7", path = "crates/core/config" }
revolt-database = { version = "0.13.7", path = "crates/core/database" }
revolt-files = { version = "0.13.7", path = "crates/core/files" }
revolt-models = { version = "0.13.7", path = "crates/core/models" }
revolt-parser = { version = "0.13.7", path = "crates/core/parser" }
revolt-permissions = { version = "0.13.7", path = "crates/core/permissions" }
revolt-presence = { version = "0.13.7", path = "crates/core/presence" }
revolt-ratelimits = { version = "0.13.7", path = "crates/core/ratelimits" }
revolt-result = { version = "0.13.7", path = "crates/core/result" }

15
compose.yml
View File

@@ -8,10 +8,20 @@ services:
# MongoDB
database:
image: mongo
command: mongod --replSet rs0
ports:
- "27017:27017"
volumes:
- ./.data/db:/data/db
extra_hosts:
- "host.docker.internal:host-gateway"
healthcheck:
test: echo "try { rs.status() } catch (err) { rs.initiate({_id:'rs0',members:[{_id:0,host:'127.0.0.1:27017'}]}) }" | mongosh --port 27017 --quiet
interval: 5s
timeout: 30s
start_period: 0s
start_interval: 1s
retries: 30
ulimits:
nofile:
soft: 65536
@@ -19,11 +29,12 @@ services:
# MinIO
minio:
image: minio/minio
command: server /data
image: firstfinger/minio:latest
#command: server /data
environment:
MINIO_ROOT_USER: minioautumn
MINIO_ROOT_PASSWORD: minioautumn
MINIO_REGION: minio
volumes:
- ./.data/minio:/data
ports:

2
crates/bonfire/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-bonfire"
version = "0.12.1"
version = "0.13.7"
license = "AGPL-3.0-or-later"
edition = "2021"
publish = false

1
crates/bonfire/src/events/impl.rs
View File

@@ -1,6 +1,7 @@
use std::collections::{HashMap, HashSet};
use futures::future::join_all;
use redis_kiss::AsyncCommands;
use revolt_database::{
events::client::{EventV1, ReadyPayloadFields},
util::permissions::DatabasePermissionQuery,

52
crates/bonfire/src/websocket.rs
View File

@@ -13,7 +13,7 @@ use futures::{
stream::{SplitSink, SplitStream},
FutureExt, SinkExt, StreamExt, TryStreamExt,
};
use redis_kiss::{PayloadType, REDIS_PAYLOAD_TYPE, REDIS_URI};
use redis_kiss::{get_connection, AsyncCommands, PayloadType, REDIS_PAYLOAD_TYPE, REDIS_URI};
use revolt_config::report_internal_error;
use revolt_database::{
events::{client::EventV1, server::ClientMessage},
@@ -32,6 +32,7 @@ use sentry::Level;
use crate::config::{ProtocolConfiguration, WebsocketHandshakeCallback};
use crate::events::state::{State, SubscriptionStateChange};
use revolt_models::v0;
type WsReader = SplitStream<WebSocketStream<TcpStream>>;
type WsWriter = SplitSink<WebSocketStream<TcpStream>, async_tungstenite::tungstenite::Message>;
@@ -128,6 +129,14 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr)
return;
}
let slowmodes = fetch_user_slowmodes(&user_id).await.unwrap_or_default();
if !slowmodes.is_empty() {
let event = EventV1::UserSlowmodes { slowmodes };
if report_internal_error!(write.send(config.encode(&event)).await).is_err() {
return;
}
}
// Create presence session.
let (first_session, session_id) = create_session(&user_id, 0).await;
@@ -527,4 +536,43 @@ async fn worker(
}
}
}
}
}
async fn fetch_user_slowmodes(user_id: &str) -> Option<Vec<v0::ChannelSlowmode>> {
let mut conn = get_connection().await.ok()?.into_inner();
let idx_key = format!("slowmode_idx:{}", user_id);
let channel_ids: Vec<String> = conn.smembers(&idx_key).await.unwrap_or_default();
if channel_ids.is_empty() {
return Some(vec![]);
}
// Bulk fetch all TTLs in one round trip
let mut pipe = redis_kiss::redis::pipe();
for channel_id in &channel_ids {
pipe.ttl(format!("slowmode:{}:{}", user_id, channel_id));
}
let ttls: Vec<i64> = pipe.query_async(&mut conn).await.unwrap_or_default();
// Partition into alive/expired in one pass
let mut slowmodes = vec![];
let mut expired = vec![];
for (channel_id, ttl) in channel_ids.iter().zip(ttls.iter()) {
if *ttl > 0 {
slowmodes.push(v0::ChannelSlowmode {
channel_id: channel_id.clone(),
duration: *ttl as u64,
retry_after: *ttl as u64,
});
} else {
expired.push(channel_id.as_str());
}
}
// Bulk remove all expired members in one SREM call
if !expired.is_empty() {
conn.srem::<_, _, ()>(&idx_key, expired).await.ok();
}
Some(slowmodes)
}

2
crates/core/coalesced/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-coalesced"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>", "Zomatree <me@zomatree.live>"]

2
crates/core/config/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-config"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]

12
crates/core/config/Revolt.toml
View File

@@ -30,6 +30,10 @@ host = "rabbit"
port = 5672
username = "rabbituser"
password = "rabbitpass"
default_exchange = "revolt.default"
[rabbit.queues]
acks = "internal.ack"
[api]
@@ -135,6 +139,8 @@ pkcs8 = ""
key_id = ""
team_id = ""
[january]
blocked_domains = []
[files]
# Encryption key for stored files
@@ -321,6 +327,12 @@ emojis = 500_000
# default: 5
process_message_delay_limit = 5
[features.legal_links]
# URLs for legal documents
terms_of_service = ""
privacy_policy = ""
guidelines = ""
[sentry]
# Configuration for Sentry error reporting
api = ""

24
crates/core/config/src/lib.rs
View File

@@ -122,12 +122,19 @@ pub struct Database {
pub redis_pubsub: Option<String>,
}
#[derive(Deserialize, Debug, Clone)]
pub struct RabbitQueues {
pub acks: String,
}
#[derive(Deserialize, Debug, Clone)]
pub struct Rabbit {
pub host: String,
pub port: u16,
pub username: String,
pub password: String,
pub default_exchange: String,
pub queues: RabbitQueues,
}
#[derive(Deserialize, Debug, Clone)]
@@ -303,6 +310,11 @@ impl Pushd {
}
}
#[derive(Deserialize, Debug, Clone)]
pub struct January {
pub blocked_domains: Vec<String>,
}
#[derive(Deserialize, Debug, Clone)]
pub struct FilesLimit {
pub min_file_size: usize,
@@ -378,6 +390,16 @@ pub struct FeaturesLimitsCollection {
pub roles: HashMap<String, FeaturesLimits>,
}
#[derive(Deserialize, Debug, Clone)]
pub struct LegalLinks {
/// Terms of Service URL
pub terms_of_service: String,
/// Privacy Policy URL
pub privacy_policy: String,
/// Guidelines URL
pub guidelines: String,
}
#[derive(Deserialize, Debug, Clone)]
pub struct FeaturesAdvanced {
#[serde(default)]
@@ -395,6 +417,7 @@ impl Default for FeaturesAdvanced {
#[derive(Deserialize, Debug, Clone)]
pub struct Features {
pub limits: FeaturesLimitsCollection,
pub legal_links: LegalLinks,
pub webhooks_enabled: bool,
pub mass_mentions_send_notifications: bool,
pub mass_mentions_enabled: bool,
@@ -424,6 +447,7 @@ pub struct Settings {
pub hosts: Hosts,
pub api: Api,
pub pushd: Pushd,
pub january: January,
pub files: Files,
pub features: Features,
pub sentry: Sentry,

7
crates/core/database/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-database"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -38,6 +38,7 @@ revolt-models = { workspace = true, features = ["validator"] }
revolt-presence = { workspace = true }
revolt-permissions = { workspace = true, features = ["serde", "bson"] }
revolt-parser = { workspace = true }
revolt-coalesced = { workspace = true }
# Utility
log = { workspace = true }
@@ -94,9 +95,9 @@ revolt_rocket_okapi = { workspace = true, optional = true }
authifier = { workspace = true }
# RabbitMQ
amqprs = { workspace = true }
lapin = { workspace = true, features = ["tokio"] }
# Voice
livekit-api = { workspace = true, optional = true }
livekit-api = { workspace = true, features = ["rustls-tls-native-roots"], optional = true }
livekit-protocol = { workspace = true, optional = true }
livekit-runtime = { workspace = true, features = ["tokio"], optional = true }

277
crates/core/database/src/amqp/amqp.rs
View File

@@ -1,58 +1,77 @@
use std::collections::HashSet;
use std::sync::Arc;
use crate::events::rabbit::*;
use crate::User;
use amqprs::channel::{BasicPublishArguments, ExchangeDeclareArguments};
use amqprs::connection::OpenConnectionArguments;
use amqprs::{channel::Channel, connection::Connection, error::Error as AMQPError};
use amqprs::{BasicProperties, FieldTable};
use lapin::{
options::BasicPublishOptions,
protocol::basic::AMQPProperties,
types::{AMQPValue, FieldTable},
Channel, Connection, ConnectionProperties, Error as AMQPError,
};
use revolt_models::v0::PushNotification;
use revolt_presence::filter_online;
use revolt_result::Result;
use serde_json::to_string;
#[derive(Clone)]
pub struct AMQP {
friend_request_accepted: Arc<Channel>,
friend_request_received: Arc<Channel>,
generic_message: Arc<Channel>,
message_sent: Arc<Channel>,
mass_mention_message_sent: Arc<Channel>,
ack_notification_message: Arc<Channel>,
dm_call_updated: Arc<Channel>,
process_ack: Arc<Channel>,
#[allow(unused)]
connection: Connection,
channel: Channel,
connection: Arc<Connection>,
}
impl AMQP {
pub fn new(connection: Connection, channel: Channel) -> AMQP {
AMQP {
pub async fn new(connection: Arc<Connection>) -> Self {
Self {
friend_request_accepted: Self::create_channel(&connection).await,
friend_request_received: Self::create_channel(&connection).await,
generic_message: Self::create_channel(&connection).await,
message_sent: Self::create_channel(&connection).await,
mass_mention_message_sent: Self::create_channel(&connection).await,
ack_notification_message: Self::create_channel(&connection).await,
dm_call_updated: Self::create_channel(&connection).await,
process_ack: Self::create_channel(&connection).await,
connection,
channel,
}
}
pub async fn new_auto() -> AMQP {
pub async fn new_auto() -> Self {
let config = revolt_config::config().await;
let connection = Connection::open(&OpenConnectionArguments::new(
&config.rabbit.host,
config.rabbit.port,
&config.rabbit.username,
&config.rabbit.password,
))
.await
.expect("Failed to connect to RabbitMQ");
let channel = connection
.open_channel(None)
.await
.expect("Failed to open RabbitMQ channel");
channel
.exchange_declare(
ExchangeDeclareArguments::new(&config.pushd.exchange, "direct")
.durable(true)
.finish(),
let connection = Arc::new(
Connection::connect(
&format!(
"amqp://{}:{}@{}:{}",
&config.rabbit.username,
&config.rabbit.password,
&config.rabbit.host,
&config.rabbit.port,
),
ConnectionProperties::default(),
)
.await
.expect("Failed to declare exchange");
.expect("Failed to connect to RabbitMQ"),
);
AMQP::new(connection, channel)
Self::new(connection).await
}
async fn create_channel(connection: &Connection) -> Arc<Channel> {
Arc::new(
connection
.create_channel()
.await
.expect("Failed to create channel"),
)
}
pub async fn friend_request_accepted(
@@ -72,19 +91,20 @@ impl AMQP {
config.pushd.get_fr_accepted_routing_key(),
payload
);
self.channel
self.friend_request_accepted
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_fr_accepted_routing_key(),
),
config.pushd.exchange.clone().into(),
config.pushd.get_fr_accepted_routing_key().into(),
BasicPublishOptions::default(),
payload.as_bytes(),
AMQPProperties::default()
.with_content_type("application/json".into())
.with_delivery_mode(2),
)
.await
.await?;
Ok(())
}
pub async fn friend_request_received(
@@ -105,19 +125,19 @@ impl AMQP {
payload
);
self.channel
self.friend_request_received
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_fr_received_routing_key(),
),
config.pushd.exchange.clone().into(),
config.pushd.get_fr_received_routing_key().into(),
BasicPublishOptions::default(),
payload.as_bytes(),
AMQPProperties::default()
.with_content_type("application/json".into())
.with_delivery_mode(2),
)
.await
.await?;
Ok(())
}
pub async fn generic_message(
@@ -142,19 +162,19 @@ impl AMQP {
payload
);
self.channel
self.generic_message
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_generic_routing_key(),
),
config.pushd.exchange.clone().into(),
config.pushd.get_generic_routing_key().into(),
BasicPublishOptions::default(),
payload.as_bytes(),
AMQPProperties::default()
.with_content_type("application/json".into())
.with_delivery_mode(2),
)
.await
.await?;
Ok(())
}
pub async fn message_sent(
@@ -185,19 +205,19 @@ impl AMQP {
payload
);
self.channel
self.message_sent
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_message_routing_key(),
),
config.pushd.exchange.clone().into(),
config.pushd.get_message_routing_key().into(),
BasicPublishOptions::default(),
payload.as_bytes(),
AMQPProperties::default()
.with_content_type("application/json".into())
.with_delivery_mode(2),
)
.await
.await?;
Ok(())
}
pub async fn mass_mention_message_sent(
@@ -220,19 +240,24 @@ impl AMQP {
routing_key, payload
);
self.channel
self.mass_mention_message_sent
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(&config.pushd.exchange, routing_key.as_str()),
config.pushd.exchange.clone().into(),
routing_key.into(),
BasicPublishOptions::default(),
payload.as_bytes(),
AMQPProperties::default()
.with_content_type("application/json".into())
.with_delivery_mode(2),
)
.await
.await?;
Ok(())
}
pub async fn ack_message(
/// # Sends an ack to pushd to update badges on iPhones.
/// Not to be confused with the process_ack function, which handles sending all acks to crond for processing.
pub async fn ack_notification_message(
&self,
user_id: String,
channel_id: String,
@@ -252,23 +277,25 @@ impl AMQP {
config.pushd.ack_queue, payload
);
let mut headers = FieldTable::new();
let mut headers = FieldTable::default();
headers.insert(
"x-deduplication-header".try_into().unwrap(),
format!("{}-{}", &user_id, &channel_id).into(),
"x-deduplication-header".into(),
AMQPValue::LongString(format!("{}-{}", &user_id, &channel_id).into()),
);
self.channel
self.ack_notification_message
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
//.with_headers(headers)
.finish(),
payload.into(),
BasicPublishArguments::new(&config.pushd.exchange, &config.pushd.ack_queue),
config.pushd.exchange.clone().into(),
config.pushd.ack_queue.into(),
BasicPublishOptions::default(),
payload.as_bytes(),
AMQPProperties::default()
.with_content_type("application/json".into())
.with_delivery_mode(2),
)
.await
.await?;
Ok(())
}
/// # DM Call Update
@@ -302,18 +329,54 @@ impl AMQP {
payload
);
self.channel
self.dm_call_updated
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_dm_call_routing_key(),
),
config.pushd.exchange.clone().into(),
config.pushd.get_dm_call_routing_key().into(),
BasicPublishOptions::default(),
payload.as_bytes(),
AMQPProperties::default()
.with_content_type("application/json".into())
.with_delivery_mode(2),
)
.await
.await?;
Ok(())
}
/// # Send an ack to crond for processing
pub async fn process_ack(
&self,
user_id: &str,
channel_id: Option<&str>,
server_id: Option<&str>,
) -> Result<(), AMQPError> {
let config = revolt_config::config().await;
let payload = AckEventPayload {
user_id: user_id.to_string(),
channel_id: channel_id.map(|value| value.to_string()),
server_id: server_id.map(|value| value.to_string()),
};
let payload = to_string(&payload).unwrap();
info!(
"Sending ack processor event on exchange {}, channel {}: {}",
config.rabbit.default_exchange, config.rabbit.queues.acks, payload
);
self.process_ack
.basic_publish(
config.rabbit.default_exchange.clone().into(),
config.rabbit.queues.acks.into(),
BasicPublishOptions::default(),
payload.as_bytes(),
AMQPProperties::default()
.with_content_type("application/json".into())
.with_delivery_mode(2),
)
.await?;
Ok(())
}
}

106
crates/core/database/src/events/client.rs
View File

@@ -3,7 +3,12 @@ use revolt_result::Error;
use serde::{Deserialize, Serialize};
use revolt_models::v0::{
AppendMessage, Channel, ChannelUnread, ChannelVoiceState, Emoji, FieldsChannel, FieldsMember, FieldsMessage, FieldsRole, FieldsServer, FieldsUser, FieldsWebhook, Member, MemberCompositeKey, Message, PartialChannel, PartialMember, PartialMessage, PartialRole, PartialServer, PartialUser, PartialUserVoiceState, PartialWebhook, PolicyChange, RemovalIntention, Report, Server, User, UserSettings, UserVoiceState, Webhook
AppendMessage, Channel, ChannelSlowmode, ChannelUnread, ChannelVoiceState, Emoji,
FieldsChannel, FieldsMember, FieldsMessage, FieldsRole, FieldsServer, FieldsUser,
FieldsWebhook, Member, MemberCompositeKey, Message, PartialChannel, PartialEmoji,
PartialMember, PartialMessage, PartialRole, PartialServer, PartialUser, PartialUserVoiceState,
PartialWebhook, PolicyChange, RemovalIntention, Report, Server, User, UserSettings,
UserVoiceState, Webhook,
};
use crate::Database;
@@ -51,9 +56,13 @@ impl Default for ReadyPayloadFields {
#[serde(tag = "type")]
pub enum EventV1 {
/// Multiple events
Bulk { v: Vec<EventV1> },
Bulk {
v: Vec<EventV1>,
},
/// Error event
Error { data: Error },
Error {
data: Error,
},
/// Successfully authenticated
Authenticated,
@@ -84,7 +93,9 @@ pub enum EventV1 {
},
/// Ping response
Pong { data: Ping },
Pong {
data: Ping,
},
/// New message
Message(Message),
@@ -105,7 +116,10 @@ pub enum EventV1 {
},
/// Delete message
MessageDelete { id: String, channel: String },
MessageDelete {
id: String,
channel: String,
},
/// New reaction to a message
MessageReact {
@@ -131,7 +145,10 @@ pub enum EventV1 {
},
/// Bulk delete messages
BulkMessageDelete { channel: String, ids: Vec<String> },
BulkMessageDelete {
channel: String,
ids: Vec<String>,
},
/// New server
ServerCreate {
@@ -139,7 +156,7 @@ pub enum EventV1 {
server: Server,
channels: Vec<Channel>,
emojis: Vec<Emoji>,
voice_states: Vec<ChannelVoiceState>
voice_states: Vec<ChannelVoiceState>,
},
/// Update existing server
@@ -151,7 +168,9 @@ pub enum EventV1 {
},
/// Delete server
ServerDelete { id: String },
ServerDelete {
id: String,
},
/// Update existing server member
ServerMemberUpdate {
@@ -187,10 +206,16 @@ pub enum EventV1 {
},
/// Server role deleted
ServerRoleDelete { id: String, role_id: String },
ServerRoleDelete {
id: String,
role_id: String,
},
/// Server roles ranks updated
ServerRoleRanksUpdate { id: String, ranks: Vec<String> },
ServerRoleRanksUpdate {
id: String,
ranks: Vec<String>,
},
/// Update existing user
UserUpdate {
@@ -202,9 +227,15 @@ pub enum EventV1 {
},
/// Relationship with another user changed
UserRelationship { id: String, user: User },
UserRelationship {
id: String,
user: User,
},
/// Settings updated remotely
UserSettingsUpdate { id: String, update: UserSettings },
UserSettingsUpdate {
id: String,
update: UserSettings,
},
/// User has been platform banned or deleted their account
///
@@ -215,12 +246,23 @@ pub enum EventV1 {
/// - Server Memberships
///
/// User flags are specified to explain why a wipe is occurring though not all reasons will necessarily ever appear.
UserPlatformWipe { user_id: String, flags: i32 },
UserPlatformWipe {
user_id: String,
flags: i32,
},
/// New emoji
EmojiCreate(Emoji),
/// Update existing emoji
EmojiUpdate {
id: String,
data: PartialEmoji,
},
/// Delete emoji
EmojiDelete { id: String },
EmojiDelete {
id: String,
},
/// New report
ReportCreate(Report),
@@ -236,19 +278,33 @@ pub enum EventV1 {
},
/// Delete channel
ChannelDelete { id: String },
ChannelDelete {
id: String,
},
/// User joins a group
ChannelGroupJoin { id: String, user: String },
ChannelGroupJoin {
id: String,
user: String,
},
/// User leaves a group
ChannelGroupLeave { id: String, user: String },
ChannelGroupLeave {
id: String,
user: String,
},
/// User started typing in a channel
ChannelStartTyping { id: String, user: String },
ChannelStartTyping {
id: String,
user: String,
},
/// User stopped typing in a channel
ChannelStopTyping { id: String, user: String },
ChannelStopTyping {
id: String,
user: String,
},
/// User acknowledged message in channel
ChannelAck {
@@ -268,7 +324,9 @@ pub enum EventV1 {
},
/// Delete webhook
WebhookDelete { id: String },
WebhookDelete {
id: String,
},
/// Auth events
Auth(AuthifierEvent),
@@ -286,7 +344,7 @@ pub enum EventV1 {
user: String,
from: String,
to: String,
state: UserVoiceState
state: UserVoiceState,
},
UserVoiceStateUpdate {
id: String,
@@ -298,7 +356,11 @@ pub enum EventV1 {
from: String,
to: String,
token: String,
}
},
/// User's active slowmodes
UserSlowmodes {
slowmodes: Vec<ChannelSlowmode>,
},
}
impl EventV1 {

8
crates/core/database/src/events/rabbit.rs
View File

@@ -78,3 +78,11 @@ pub struct AckPayload {
pub channel_id: String,
pub message_id: String,
}
/// This is not the same as the AckPayload above, as the state for this event is stored in redis to allow for state updates while the event is queued.
#[derive(Serialize, Deserialize, Debug)]
pub struct AckEventPayload {
pub user_id: String,
pub channel_id: Option<String>,
pub server_id: Option<String>,
}

19
crates/core/database/src/models/channels/model.rs
View File

@@ -1,6 +1,7 @@
#![allow(deprecated)]
use std::{borrow::Cow, collections::HashMap};
use redis_kiss::get_connection;
use revolt_config::config;
use revolt_models::v0::{self, MessageAuthor};
use revolt_permissions::OverrideField;
@@ -212,7 +213,7 @@ impl Channel {
role_permissions: HashMap::new(),
nsfw: data.nsfw.unwrap_or(false),
voice: data.voice.map(|voice| voice.into()),
slowmode: None
slowmode: None,
},
v0::LegacyServerChannelType::Voice => Channel::TextChannel {
id: id.clone(),
@@ -225,7 +226,7 @@ impl Channel {
role_permissions: HashMap::new(),
nsfw: data.nsfw.unwrap_or(false),
voice: Some(data.voice.unwrap_or_default().into()),
slowmode: None
slowmode: None,
},
};
@@ -643,7 +644,7 @@ impl Channel {
}
/// Acknowledge a message
pub async fn ack(&self, user: &str, message: &str) -> Result<()> {
pub async fn ack(&self, user: &str, message: &str, amqp: &AMQP) -> Result<()> {
EventV1::ChannelAck {
id: self.id().to_string(),
user: user.to_string(),
@@ -652,17 +653,7 @@ impl Channel {
.private(user.to_string())
.await;
#[cfg(feature = "tasks")]
crate::tasks::ack::queue_ack(
self.id().to_string(),
user.to_string(),
crate::tasks::ack::AckEvent::AckMessage {
id: message.to_string(),
},
)
.await;
Ok(())
crate::util::acker::ack_channel(user, self.id(), message, amqp).await
}
/// Remove user from a group

32
crates/core/database/src/models/emojis/model.rs
View File

@@ -2,6 +2,7 @@ use std::collections::HashSet;
use std::str::FromStr;
use once_cell::sync::Lazy;
use revolt_models::v0;
use revolt_result::Result;
use ulid::Ulid;
@@ -11,7 +12,7 @@ use crate::Database;
static PERMISSIBLE_EMOJIS: Lazy<HashSet<String>> = Lazy::new(|| {
include_str!("unicode_emoji.txt")
.split('\n')
.map(|x| x.into())
.map(|x| x.replace('\u{FE0F}', ""))
.collect()
});
@@ -41,6 +42,12 @@ auto_derived!(
Server { id: String },
Detached,
}
/// Partial representation of an emoji
pub struct PartialEmoji {
#[serde(skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
}
);
#[allow(clippy::disallowed_methods)]
@@ -75,13 +82,34 @@ impl Emoji {
db.detach_emoji(&self).await
}
/// Update an emoji
pub async fn update(&mut self, db: &Database, partial: PartialEmoji) -> Result<()> {
if let Some(name) = partial.name.clone() {
self.name = name;
}
db.update_emoji(&self.id, &partial).await?;
EventV1::EmojiUpdate {
id: self.id.clone(),
data: v0::PartialEmoji {
name: partial.name.clone(),
},
}
.p(self.parent().to_string())
.await;
Ok(())
}
/// Check whether we can use a given emoji
pub async fn can_use(db: &Database, emoji: &str) -> Result<bool> {
if Ulid::from_str(emoji).is_ok() {
db.fetch_emoji(emoji).await?;
Ok(true)
} else {
Ok(PERMISSIBLE_EMOJIS.contains(emoji))
let sanitized_emoji = emoji.replace('\u{FE0F}', "");
Ok(PERMISSIBLE_EMOJIS.contains(&sanitized_emoji))
}
}
}

5
crates/core/database/src/models/emojis/ops.rs
View File

@@ -1,6 +1,6 @@
use revolt_result::Result;
use crate::Emoji;
use crate::{Emoji, PartialEmoji};
#[cfg(feature = "mongodb")]
mod mongodb;
@@ -20,6 +20,9 @@ pub trait AbstractEmojis: Sync + Send {
/// Fetch emoji by their parent ids
async fn fetch_emoji_by_parent_ids(&self, parent_ids: &[String]) -> Result<Vec<Emoji>>;
/// Update emoji with new information
async fn update_emoji(&self, emoji_id: &str, partial: &PartialEmoji) -> Result<()>;
/// Detach an emoji by its id
async fn detach_emoji(&self, emoji: &Emoji) -> Result<()>;
}

7
crates/core/database/src/models/emojis/ops/mongodb.rs
View File

@@ -1,7 +1,7 @@
use bson::Document;
use revolt_result::Result;
use crate::Emoji;
use crate::{Emoji, PartialEmoji};
use crate::MongoDb;
use super::AbstractEmojis;
@@ -46,6 +46,11 @@ impl AbstractEmojis for MongoDb {
)
}
/// Update emoji with new information
async fn update_emoji(&self, emoji_id: &str, partial: &PartialEmoji) -> Result<()> {
query!(self, update_one_by_id, COL, emoji_id, partial, vec![], None).map(|_| ())
}
/// Detach an emoji by its id
async fn detach_emoji(&self, emoji: &Emoji) -> Result<()> {
self.col::<Document>(COL)

15
crates/core/database/src/models/emojis/ops/reference.rs
View File

@@ -1,6 +1,6 @@
use revolt_result::Result;
use crate::Emoji;
use crate::{Emoji, PartialEmoji};
use crate::EmojiParent;
use crate::ReferenceDb;
@@ -54,6 +54,19 @@ impl AbstractEmojis for ReferenceDb {
.collect())
}
/// Update emoji with new information
async fn update_emoji(&self, emoji_id: &str, partial: &PartialEmoji) -> Result<()> {
let mut emojis = self.emojis.lock().await;
if let Some(emoji) = emojis.get_mut(emoji_id) {
if let Some(name) = partial.name.clone() {
emoji.name = name;
}
Ok(())
} else {
Err(create_error!(NotFound))
}
}
/// Detach an emoji by its id
async fn detach_emoji(&self, emoji: &Emoji) -> Result<()> {
let mut emojis = self.emojis.lock().await;

5180
crates/core/database/src/models/emojis/unicode_emoji.txt
View File

File diff suppressed because it is too large Load Diff

20
crates/core/database/src/models/files/model.rs
View File

@@ -70,6 +70,7 @@ auto_derived!(
LegacyGroupIcon,
ChannelIcon,
ServerIcon,
RoleIcon,
}
/// Information about what the file was used for
@@ -239,4 +240,23 @@ impl File {
)
.await
}
/// Use a file for a role icon
pub async fn use_role_icon(
db: &Database,
id: &str,
parent: &str,
uploader_id: &str,
) -> Result<File> {
db.find_and_use_attachment(
id,
"icons",
FileUsedFor {
id: parent.to_owned(),
object_type: FileUsedForType::RoleIcon,
},
uploader_id.to_owned(),
)
.await
}
}

7
crates/core/database/src/models/servers/model.rs
View File

@@ -86,6 +86,9 @@ auto_derived_partial!(
/// Ranking of this role
#[serde(default)]
pub rank: i64,
/// Custom icon attachment
#[serde(skip_serializing_if = "Option::is_none")]
pub icon: Option<File>,
},
"PartialRole"
);
@@ -129,6 +132,7 @@ auto_derived!(
/// Optional fields on server object
pub enum FieldsRole {
Colour,
Icon,
}
);
@@ -305,6 +309,7 @@ impl Role {
colour: self.colour,
hoist: Some(self.hoist),
rank: Some(self.rank),
icon: self.icon,
}
}
@@ -318,6 +323,7 @@ impl Role {
colour: None,
hoist: false,
permissions: Default::default(),
icon: None,
};
db.insert_role(&server.id, &role).await?;
@@ -367,6 +373,7 @@ impl Role {
pub fn remove_field(&mut self, field: &FieldsRole) {
match field {
FieldsRole::Colour => self.colour = None,
FieldsRole::Icon => self.icon = None,
}
}

1
crates/core/database/src/models/servers/ops/mongodb.rs
View File

@@ -172,6 +172,7 @@ impl IntoDocumentPath for FieldsRole {
fn as_path(&self) -> Option<&'static str> {
Some(match self {
FieldsRole::Colour => "colour",
FieldsRole::Icon => "icon",
})
}
}

10
crates/core/database/src/tasks/ack.rs
View File

@@ -105,7 +105,11 @@ pub async fn handle_ack_event(
if mentions_acked > 0 {
if let Err(err) = amqp
.ack_message(user.to_string(), channel.to_string(), id.to_owned())
.ack_notification_message(
user.to_string(),
channel.to_string(),
id.to_owned(),
)
.await
{
revolt_config::capture_error(&err);
@@ -192,9 +196,7 @@ pub async fn handle_ack_event(
.expect("Failed to fetch channel from db");
if let TextChannel { server, .. } = channel {
if let Err(err) =
amqp.mass_mention_message_sent(server, mass_mentions).await
{
if let Err(err) = amqp.mass_mention_message_sent(server, mass_mentions).await {
revolt_config::capture_error(&err);
}
} else {

77
crates/core/database/src/util/acker.rs Normal file
View File

@@ -0,0 +1,77 @@
use redis_kiss::{get_connection, AsyncCommands};
use revolt_permissions::{calculate_channel_permissions, ChannelPermission};
use revolt_result::{Result, ToRevoltError};
use crate::{events::client::EventV1, Channel, Database, Server, User, AMQP};
pub async fn ack_channel(user: &str, channel: &str, message: &str, amqp: &AMQP) -> Result<()> {
let mut redis = get_connection()
.await
.map_err(|_| create_error!(InternalError))?;
let old: Option<String> = redis
.getset(format!("acker:{user}+{channel}"), message)
.await
.to_internal_error()?;
if old.is_none() || old.unwrap() == message {
amqp.process_ack(user, Some(channel), None)
.await
.to_internal_error()?;
}
Ok(())
}
pub async fn ack_server(user: &User, server: &Server, db: &Database, amqp: &AMQP) -> Result<()> {
let mut redis = get_connection()
.await
.map_err(|_| create_error!(InternalError))?;
let channels = db.fetch_channels(&server.channels).await?;
let query = crate::util::permissions::DatabasePermissionQuery::new(db, user).server(server);
for channel in channels {
let channel_id = channel.id();
let mut q = query.clone().channel(&channel);
if calculate_channel_permissions(&mut q)
.await
.has_channel_permission(ChannelPermission::ViewChannel)
{
let channel_last_msg = match &channel {
Channel::TextChannel {
last_message_id, ..
} => last_message_id,
_ => unreachable!(),
}
.clone();
if let Some(channel_last_msg) = channel_last_msg {
let old: Option<String> = redis
.getset(
format!("acker:{}+{}", user.id, channel_id),
&channel_last_msg,
)
.await
.to_internal_error()?;
if old.is_none() || old.unwrap() == channel_last_msg {
amqp.process_ack(&user.id, Some(channel_id), Some(&server.id))
.await
.to_internal_error()?;
EventV1::ChannelAck {
id: channel_id.to_string(),
user: user.id.clone(),
message_id: channel_last_msg,
}
.private(user.id.clone())
.await;
}
}
}
}
Ok(())
}

16
crates/core/database/src/util/bridge/v0.rs
View File

@@ -190,7 +190,7 @@ impl From<crate::Channel> for Channel {
role_permissions,
nsfw,
voice,
slowmode
slowmode,
} => Channel::TextChannel {
id,
server,
@@ -202,7 +202,7 @@ impl From<crate::Channel> for Channel {
role_permissions,
nsfw,
voice: voice.map(|voice| voice.into()),
slowmode
slowmode,
},
}
}
@@ -256,7 +256,7 @@ impl From<Channel> for crate::Channel {
role_permissions,
nsfw,
voice,
slowmode
slowmode,
} => crate::Channel::TextChannel {
id,
server,
@@ -268,7 +268,7 @@ impl From<Channel> for crate::Channel {
role_permissions,
nsfw,
voice: voice.map(|voice| voice.into()),
slowmode
slowmode,
},
}
}
@@ -307,7 +307,7 @@ impl From<PartialChannel> for crate::PartialChannel {
default_permissions: value.default_permissions,
last_message_id: value.last_message_id,
voice: value.voice.map(|voice| voice.into()),
slowmode: value.slowmode
slowmode: value.slowmode,
}
}
}
@@ -927,6 +927,7 @@ impl From<crate::Role> for Role {
colour: value.colour,
hoist: value.hoist,
rank: value.rank,
icon: value.icon.map(|f| f.into()),
}
}
}
@@ -940,6 +941,7 @@ impl From<Role> for crate::Role {
colour: value.colour,
hoist: value.hoist,
rank: value.rank,
icon: value.icon.map(|f| f.into()),
}
}
}
@@ -953,6 +955,7 @@ impl From<crate::PartialRole> for PartialRole {
colour: value.colour,
hoist: value.hoist,
rank: value.rank,
icon: value.icon.map(|f| f.into()),
}
}
}
@@ -966,6 +969,7 @@ impl From<PartialRole> for crate::PartialRole {
colour: value.colour,
hoist: value.hoist,
rank: value.rank,
icon: value.icon.map(|f| f.into()),
}
}
}
@@ -974,6 +978,7 @@ impl From<crate::FieldsRole> for FieldsRole {
fn from(value: crate::FieldsRole) -> Self {
match value {
crate::FieldsRole::Colour => FieldsRole::Colour,
crate::FieldsRole::Icon => FieldsRole::Icon,
}
}
}
@@ -982,6 +987,7 @@ impl From<FieldsRole> for crate::FieldsRole {
fn from(value: FieldsRole) -> Self {
match value {
FieldsRole::Colour => crate::FieldsRole::Colour,
FieldsRole::Icon => crate::FieldsRole::Icon,
}
}
}

1
crates/core/database/src/util/mod.rs
View File

@@ -1,3 +1,4 @@
pub mod acker;
pub mod basic;
pub mod bridge;
pub mod bulk_permissions;

2
crates/core/files/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-files"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <me@insrt.uk>"]

2
crates/core/models/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-models"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]

6
crates/core/models/src/v0/channels.rs
View File

@@ -314,6 +314,12 @@ auto_derived!(
/// Only used when the user is the first one connected.
pub recipients: Option<Vec<String>>,
}
pub struct ChannelSlowmode {
pub channel_id: String,
pub duration: u64,
pub retry_after: u64,
}
);
impl Channel {

18
crates/core/models/src/v0/emojis.rs
View File

@@ -54,4 +54,22 @@ auto_derived!(
#[serde(default)]
pub nsfw: bool,
}
/// Partial emoji representation
#[derive(Default)]
pub struct PartialEmoji {
#[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
pub name: Option<String>,
}
/// Edit emoji information
#[cfg_attr(feature = "validator", derive(Validate))]
pub struct DataEditEmoji {
/// Emoji name
#[cfg_attr(
feature = "validator",
validate(length(min = 1, max = 32), regex = "RE_EMOJI")
)]
pub name: Option<String>,
}
);

9
crates/core/models/src/v0/servers.rs
View File

@@ -106,6 +106,9 @@ auto_derived_partial!(
/// Ranking of this role
#[cfg_attr(feature = "serde", serde(default))]
pub rank: i64,
/// Role icon
#[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
pub icon: Option<File>,
},
"PartialRole"
);
@@ -123,6 +126,7 @@ auto_derived!(
/// Optional fields on server object
pub enum FieldsRole {
Colour,
Icon,
}
/// Channel category
@@ -278,6 +282,11 @@ auto_derived!(
///
/// **Removed** - no effect, use the edit server role positions route
pub rank: Option<i64>,
/// Role icon
///
/// Provide an Autumn attachment Id.
#[cfg_attr(feature = "validator", validate(length(min = 1, max = 128)))]
pub icon: Option<String>,
/// Fields to remove from role object
#[cfg_attr(feature = "serde", serde(default))]
pub remove: Vec<FieldsRole>,

2
crates/core/parser/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-parser"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "MIT"
authors = ["Zomatree <me@zomatree.live>", "Paul Makles <me@insrt.uk>"]

2
crates/core/permissions/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-permissions"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]

2
crates/core/presence/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-presence"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <me@insrt.uk>"]

2
crates/core/ratelimits/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-ratelimits"
version = "0.12.1"
version = "0.13.7"
edition = "2024"
license = "MIT"
authors = ["Zomatree <me@zomatree.live>", "Paul Makles <me@insrt.uk>"]

6
crates/core/ratelimits/src/rocket.rs
View File

@@ -1,12 +1,12 @@
use async_trait::async_trait;
use log::info;
use revolt_config::config;
use rocket::fairing::{Fairing, Info, Kind};
use rocket::http::uri::Origin;
use rocket::http::{Method, Status};
use rocket::request::{FromRequest, Outcome};
use rocket::serde::json::Json;
use rocket::{Data, Request, Response, State};
use revolt_config::config;
use revolt_rocket_okapi::r#gen::OpenApiGenerator;
use revolt_rocket_okapi::request::{OpenApiFromRequest, RequestHeaderInput};
@@ -28,8 +28,8 @@ pub type RatelimitStorage = crate::ratelimiter::RatelimitStorage<RocketRequestKi
/// Find the remote IP of the client
fn to_ip(request: &'_ rocket::Request<'_>) -> String {
request
.remote()
.map(|x| x.ip().to_string())
.client_ip()
.map(|r| r.to_string())
.unwrap_or_default()
}

2
crates/core/result/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-result"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]

3
crates/core/result/src/axum.rs
View File

@@ -24,6 +24,7 @@ impl IntoResponse for Error {
ErrorType::UnknownChannel => StatusCode::NOT_FOUND,
ErrorType::UnknownMessage => StatusCode::NOT_FOUND,
ErrorType::UnknownAttachment => StatusCode::BAD_REQUEST,
ErrorType::CannotDeleteMessage => StatusCode::FORBIDDEN,
ErrorType::CannotEditMessage => StatusCode::FORBIDDEN,
ErrorType::CannotJoinCall => StatusCode::BAD_REQUEST,
ErrorType::TooManyAttachments { .. } => StatusCode::BAD_REQUEST,
@@ -78,7 +79,7 @@ impl IntoResponse for Error {
ErrorType::DuplicateNonce => StatusCode::CONFLICT,
ErrorType::VosoUnavailable => StatusCode::BAD_REQUEST,
ErrorType::NotFound => StatusCode::NOT_FOUND,
ErrorType::NoEffect => StatusCode::OK,
ErrorType::NoEffect => StatusCode::BAD_REQUEST,
ErrorType::FailedValidation { .. } => StatusCode::BAD_REQUEST,
ErrorType::LiveKitUnavailable => StatusCode::BAD_REQUEST,
ErrorType::NotConnected => StatusCode::BAD_REQUEST,

1
crates/core/result/src/lib.rs
View File

@@ -78,6 +78,7 @@ pub enum ErrorType {
UnknownChannel,
UnknownAttachment,
UnknownMessage,
CannotDeleteMessage,
CannotEditMessage,
CannotJoinCall,
TooManyAttachments {

3
crates/core/result/src/rocket.rs
View File

@@ -30,6 +30,7 @@ impl<'r> Responder<'r, 'static> for Error {
ErrorType::UnknownChannel => Status::NotFound,
ErrorType::UnknownMessage => Status::NotFound,
ErrorType::UnknownAttachment => Status::BadRequest,
ErrorType::CannotDeleteMessage => Status::Forbidden,
ErrorType::CannotEditMessage => Status::Forbidden,
ErrorType::CannotJoinCall => Status::BadRequest,
ErrorType::TooManyAttachments { .. } => Status::BadRequest,
@@ -84,7 +85,7 @@ impl<'r> Responder<'r, 'static> for Error {
ErrorType::NotAuthenticated => Status::Unauthorized,
ErrorType::DuplicateNonce => Status::Conflict,
ErrorType::NotFound => Status::NotFound,
ErrorType::NoEffect => Status::Ok,
ErrorType::NoEffect => Status::BadRequest,
ErrorType::FailedValidation { .. } => Status::BadRequest,
ErrorType::LiveKitUnavailable => Status::BadRequest,
ErrorType::NotAVoiceChannel => Status::BadRequest,

16
crates/daemons/crond/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-crond"
version = "0.12.1"
version = "0.13.7"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <me@insrt.uk>"]
edition = "2021"
@@ -16,8 +16,22 @@ log = { workspace = true }
# Async
tokio = { workspace = true }
# Redis
redis-kiss = { workspace = true }
# RabbitMQ
lapin = { workspace = true }
futures-lite = { workspace = true }
# Processing
serde_json = { workspace = true }
revolt_optional_struct = { workspace = true }
serde = { workspace = true }
iso8601-timestamp = { workspace = true, features = ["serde", "bson"] }
# Core
revolt-database = { workspace = true }
revolt-result = { workspace = true }
revolt-config = { workspace = true }
revolt-files = { workspace = true }
revolt-permissions = { workspace = true }

9
crates/daemons/crond/src/main.rs
View File

@@ -1,7 +1,7 @@
use revolt_config::configure;
use revolt_database::DatabaseInfo;
use revolt_database::{DatabaseInfo, AMQP};
use revolt_result::Result;
use tasks::{file_deletion, prune_dangling_files, prune_members};
use tasks::{acks, file_deletion, prune_dangling_files, prune_members};
use tokio::try_join;
pub mod tasks;
@@ -11,10 +11,13 @@ async fn main() -> Result<()> {
configure!(crond);
let db = DatabaseInfo::Auto.connect().await.expect("database");
let amqp = AMQP::new_auto().await;
try_join!(
file_deletion::task(db.clone()),
prune_dangling_files::task(db.clone()),
prune_members::task(db.clone())
prune_members::task(db.clone()),
acks::task(db.clone(), amqp.clone()),
)
.map(|_| ())
}

164
crates/daemons/crond/src/tasks/acks.rs Normal file
View File

@@ -0,0 +1,164 @@
use futures_lite::stream::StreamExt;
use lapin::{
options::*,
types::FieldTable,
uri::{AMQPAuthority, AMQPQueryString, AMQPUri, AMQPUserInfo},
ConnectionBuilder, ConnectionProperties, ExchangeKind,
};
use log::{debug, info};
use redis_kiss::{get_connection, AsyncCommands, Conn as RedisConnection};
use revolt_config::config;
use revolt_database::{events::rabbit::AckEventPayload, Database, AMQP};
use revolt_result::{Result, ToRevoltError};
use serde_json;
pub async fn task(db: Database, amqp: AMQP) -> Result<()> {
let config = config().await;
let mut redis = get_connection()
.await
.expect("Failed to get redis connection");
let uri = AMQPUri {
scheme: lapin::uri::AMQPScheme::AMQP,
authority: AMQPAuthority {
userinfo: AMQPUserInfo {
username: config.rabbit.username,
password: config.rabbit.password,
},
host: config.rabbit.host,
port: config.rabbit.port,
},
vhost: "/".to_string(),
query: AMQPQueryString::default(),
};
let connection = ConnectionBuilder::new()
.expect("Builder")
.with_uri(uri)
.with_properties(ConnectionProperties::default())
.connect()
.await
.expect("Failed to connect to rabbitmq");
let reader_channel = connection
.create_channel()
.await
.expect("Failed to create channel");
reader_channel
.exchange_declare(
config.rabbit.default_exchange.clone().into(),
ExchangeKind::Topic,
ExchangeDeclareOptions {
durable: true,
..Default::default()
},
FieldTable::default(),
)
.await
.expect("Failed to declare exchange");
reader_channel
.queue_declare(
config.rabbit.queues.acks.clone().into(),
QueueDeclareOptions {
durable: true,
..Default::default()
},
FieldTable::default(),
)
.await
.expect("Failed to bind queue");
reader_channel
.queue_bind(
config.rabbit.queues.acks.clone().into(),
config.rabbit.default_exchange.into(),
config.rabbit.queues.acks.clone().into(),
QueueBindOptions::default(),
FieldTable::default(),
)
.await
.expect("Failed to bind channel");
let mut consumer = reader_channel
.basic_consume(
config.rabbit.queues.acks.into(),
"crond-ack-consumer".into(),
BasicConsumeOptions::default(),
FieldTable::default(),
)
.await
.expect("Failed to create consumer");
while let Some(delivery) = consumer.next().await {
if let Ok(delivery) = delivery {
let payload = serde_json::from_slice::<AckEventPayload>(&delivery.data);
if let Ok(payload) = payload {
debug!("Received ack event: {payload:?}");
if let Err(e) = process_channel_ack(
&db,
&amqp,
payload.user_id,
payload.channel_id.unwrap(),
&mut redis,
)
.await
{
revolt_config::capture_error(&e);
_ = delivery.reject(BasicRejectOptions { requeue: false }).await;
} else {
_ = delivery.ack(BasicAckOptions { multiple: false }).await;
}
} else {
revolt_config::capture_message(
format!("Failed to decode ack data: {:?}", delivery.data).as_str(),
revolt_config::Level::Error,
);
}
}
}
Ok(())
}
#[allow(clippy::disallowed_methods)]
async fn process_channel_ack(
db: &Database,
amqp: &AMQP,
user: String,
channel: String,
redis: &mut RedisConnection,
) -> Result<()> {
let message_id: Option<String> = redis
.get_del(format!("acker:{user}+{channel}"))
.await
.to_internal_error()?;
if let Some(message_id) = message_id {
let unread = db.fetch_unread(&user, &channel).await?;
let updated = db.acknowledge_message(&channel, &user, &message_id).await?;
info!("Set new state for ack: {}:{}:{}", channel, user, message_id);
if let (Some(before), Some(after)) = (unread, updated) {
let before_mentions = before.mentions.unwrap_or_default().len();
let after_mentions = after.mentions.unwrap_or_default().len();
if after_mentions < before_mentions {
if let Err(err) = amqp
.ack_notification_message(user.to_string(), channel.to_string(), message_id)
.await
{
revolt_config::capture_error(&err);
}
};
}
Ok(())
} else {
Err(message_id.to_internal_error().expect_err("no err"))
}
}

28
crates/daemons/crond/src/tasks/file_deletion.rs
View File

@@ -11,22 +11,24 @@ pub async fn task(db: Database) -> Result<()> {
let files = db.fetch_deleted_attachments().await?;
for file in files {
let count = db
.count_file_hash_references(file.hash.as_ref().expect("no `hash` present"))
.await?;
// No other files reference this file on disk anymore
if count <= 1 {
let file_hash = db
.fetch_attachment_hash(file.hash.as_ref().expect("no `hash` present"))
if let Some(hash) = &file.hash {
let count = db
.count_file_hash_references(hash)
.await?;
// Delete from S3
delete_from_s3(&file_hash.bucket_id, &file_hash.path).await?;
// No other files reference this file on disk anymore
if count <= 1 {
let file_hash = db
.fetch_attachment_hash(hash)
.await?;
// Delete the hash
db.delete_attachment_hash(&file_hash.id).await?;
info!("Deleted file hash {}", file_hash.id);
// Delete from S3
delete_from_s3(&file_hash.bucket_id, &file_hash.path).await?;
// Delete the hash
db.delete_attachment_hash(&file_hash.id).await?;
info!("Deleted file hash {}", file_hash.id);
}
}
// Delete the file

1
crates/daemons/crond/src/tasks/mod.rs
View File

@@ -1,3 +1,4 @@
pub mod acks;
pub mod file_deletion;
pub mod prune_dangling_files;
pub mod prune_members;

4
crates/daemons/pushd/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-pushd"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "AGPL-3.0-or-later"
publish = false
@@ -15,7 +15,7 @@ revolt-parser = { workspace = true }
anyhow = { workspace = true }
amqprs = { workspace = true }
lapin = { workspace = true }
fcm_v1 = { workspace = true }
web-push = { workspace = true }
isahc = { workspace = true, features = ["json"], optional = true }

137
crates/daemons/pushd/src/consumers/inbound/ack.rs
View File

@@ -1,96 +1,69 @@
use crate::consumers::inbound::internal::*;
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use std::sync::Arc;
use crate::utils::Consumer;
use anyhow::Result;
use async_trait::async_trait;
use lapin::{message::Delivery, Channel, Connection};
use revolt_database::{events::rabbit::*, Database};
#[derive(Clone)]
#[allow(unused)]
pub struct AckConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
connection: Arc<Connection>,
channel: Arc<Channel>,
}
impl Channeled for AckConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl AckConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> AckConsumer {
AckConsumer {
#[async_trait]
impl Consumer for AckConsumer {
async fn create(
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<Channel>,
) -> Self {
Self {
db,
authifier_db,
conn: None,
channel: None,
connection,
channel,
}
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for AckConsumer {
fn channel(&self) -> &Arc<Channel> {
&self.channel
}
/// This consumer processes all acks the platform receives, and sends relevant badge updates to apple platforms.
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
let content = String::from_utf8(content).unwrap();
let payload: AckPayload = serde_json::from_str(content.as_str()).unwrap();
async fn consume(&self, delivery: Delivery) -> Result<()> {
let payload: AckPayload = serde_json::from_slice(&delivery.data)?;
// Step 1: fetch unreads and don't continue if there's no unreads
#[allow(clippy::disallowed_methods)]
let unreads = self.db.fetch_unread_mentions(&payload.user_id).await;
// #[allow(clippy::disallowed_methods)]
debug!("Processing unreads for {:}", &payload.user_id);
if let Ok(u) = &unreads {
let unreads = if let Ok(u) = self.db.fetch_unread_mentions(&payload.user_id).await {
if u.is_empty() {
debug!(
"Discarding unread task (no mentions found) for {:}",
&payload.user_id
);
return;
}
return Ok(());
};
u
} else {
return;
}
return Ok(());
};
if let Ok(sessions) = self.authifier_db.find_sessions(&payload.user_id).await {
let config = revolt_config::config().await;
// Step 2: find any apple sessions, since we don't need to calculate this for anything else.
// If there's no apple sessions, we can return early
let apple_sessions: Vec<&authifier::models::Session> = sessions
.iter()
let mut apple_sessions = sessions
.into_iter()
.filter(|session| {
if let Some(sub) = &session.subscription {
sub.endpoint == "apn"
@@ -98,19 +71,19 @@ impl AsyncConsumer for AckConsumer {
false
}
})
.collect();
.peekable();
if apple_sessions.is_empty() {
if apple_sessions.peek().is_none() {
debug!(
"Discarding unread task (no apn sessions found) for {:}",
&payload.user_id
);
return;
return Ok(());
}
// Step 3: calculate the actual mention count, since we have to send it out
let mut mention_count = 0;
for u in &unreads.unwrap() {
for u in &unreads {
mention_count += u.mentions.as_ref().unwrap().len()
}
@@ -123,26 +96,22 @@ impl AsyncConsumer for AckConsumer {
token: session.subscription.as_ref().unwrap().auth.clone(),
extras: Default::default(),
};
let raw_service_payload = serde_json::to_string(&service_payload);
let payload = serde_json::to_string(&service_payload)?;
if let Ok(p) = raw_service_payload {
let args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
log::debug!(
"Publishing ack to apn session {}",
session.subscription.as_ref().unwrap().auth
);
log::debug!(
"Publishing ack to apn session {}",
session.subscription.as_ref().unwrap().auth
);
publish_message(self, p.into(), args).await;
} else {
log::warn!("Failed to serialize ack badge update payload!");
revolt_config::capture_error(&raw_service_payload.unwrap_err());
}
self.publish_message(
payload.as_bytes(),
&config.pushd.exchange,
&config.pushd.apn.queue,
)
.await?;
}
}
Ok(())
}
}

140
crates/daemons/pushd/src/consumers/inbound/dm_call.rs
View File

@@ -1,70 +1,44 @@
use std::collections::HashMap;
use std::{collections::HashMap, sync::Arc};
use crate::consumers::inbound::internal::*;
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use crate::utils::Consumer;
use anyhow::Result;
use async_trait::async_trait;
use lapin::{message::Delivery, Channel, Connection};
use log::debug;
use revolt_database::{events::rabbit::*, Database};
#[derive(Clone)]
#[allow(unused)]
pub struct DmCallConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
connection: Arc<Connection>,
channel: Arc<Channel>,
}
impl Channeled for DmCallConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl DmCallConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> DmCallConsumer {
DmCallConsumer {
#[async_trait]
impl Consumer for DmCallConsumer {
async fn create(
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<Channel>,
) -> Self {
Self {
db,
authifier_db,
conn: None,
channel: None,
connection,
channel,
}
}
async fn consume_event(
&mut self,
_channel: &Channel,
_deliver: Deliver,
_basic_properties: BasicProperties,
content: Vec<u8>,
) -> Result<()> {
let content = String::from_utf8(content)?;
let _p: InternalDmCallPayload = serde_json::from_str(content.as_str())?;
fn channel(&self) -> &Arc<Channel> {
&self.channel
}
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(&self, delivery: Delivery) -> Result<()> {
let _p: InternalDmCallPayload = serde_json::from_slice(&delivery.data)?;
let payload = _p.payload;
debug!("Received dm call start/stop event");
@@ -107,36 +81,27 @@ impl DmCallConsumer {
extras: HashMap::new(),
};
let args: BasicPublishArguments;
let routing_key = match sub.endpoint.as_str() {
"apn" => &config.pushd.apn.queue,
"fcm" => &config.pushd.fcm.queue,
endpoint => {
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), endpoint.to_string());
if sub.endpoint == "apn" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
} else if sub.endpoint == "fcm" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.fcm.queue.as_str(),
)
.finish();
} else {
// web push (vapid)
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.vapid.queue.as_str(),
)
.finish();
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), sub.endpoint.clone());
}
&config.pushd.vapid.queue
}
};
let payload = serde_json::to_string(&sendable)?;
publish_message(self, payload.into(), args).await;
self.publish_message(
payload.as_bytes(),
&config.pushd.exchange,
routing_key,
)
.await?;
}
}
}
@@ -145,24 +110,3 @@ impl DmCallConsumer {
Ok(())
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for DmCallConsumer {
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
if let Err(err) = self
.consume_event(channel, deliver, basic_properties, content)
.await
{
revolt_config::capture_anyhow(&err);
warn!("Failed to process dm call start/stop event: {err:?}");
}
}
}

136
crates/daemons/pushd/src/consumers/inbound/fr_accepted.rs
View File

@@ -1,70 +1,44 @@
use std::collections::HashMap;
use std::{collections::HashMap, sync::Arc};
use crate::consumers::inbound::internal::*;
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use crate::utils::Consumer;
use anyhow::Result;
use async_trait::async_trait;
use lapin::{message::Delivery, Channel, Connection};
use log::debug;
use revolt_database::{events::rabbit::*, Database};
#[derive(Clone)]
#[allow(unused)]
pub struct FRAcceptedConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
connection: Arc<Connection>,
channel: Arc<Channel>,
}
impl Channeled for FRAcceptedConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl FRAcceptedConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> FRAcceptedConsumer {
FRAcceptedConsumer {
#[async_trait]
impl Consumer for FRAcceptedConsumer {
async fn create(
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<Channel>,
) -> Self {
Self {
db,
authifier_db,
conn: None,
channel: None,
connection,
channel,
}
}
async fn consume_event(
&mut self,
_channel: &Channel,
_deliver: Deliver,
_basic_properties: BasicProperties,
content: Vec<u8>,
) -> Result<()> {
let content = String::from_utf8(content)?;
let payload: FRAcceptedPayload = serde_json::from_str(content.as_str())?;
fn channel(&self) -> &Arc<Channel> {
&self.channel
}
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(&self, delivery: Delivery) -> Result<()> {
let payload: FRAcceptedPayload = serde_json::from_slice(&delivery.data)?;
debug!("Received FR accept event");
@@ -80,36 +54,23 @@ impl FRAcceptedConsumer {
extras: HashMap::new(),
};
let args: BasicPublishArguments;
let routing_key = match sub.endpoint.as_str() {
"apn" => &config.pushd.apn.queue,
"fcm" => &config.pushd.fcm.queue,
endpoint => {
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), endpoint.to_string());
if sub.endpoint == "apn" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
} else if sub.endpoint == "fcm" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.fcm.queue.as_str(),
)
.finish();
} else {
// web push (vapid)
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.vapid.queue.as_str(),
)
.finish();
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), sub.endpoint.clone());
}
&config.pushd.vapid.queue
}
};
let payload = serde_json::to_string(&sendable)?;
publish_message(self, payload.into(), args).await;
self.publish_message(payload.as_bytes(), &config.pushd.exchange, routing_key)
.await?;
}
}
}
@@ -117,24 +78,3 @@ impl FRAcceptedConsumer {
Ok(())
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for FRAcceptedConsumer {
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
if let Err(err) = self
.consume_event(channel, deliver, basic_properties, content)
.await
{
revolt_config::capture_anyhow(&err);
eprintln!("Failed to process friend request accepted event: {err:?}");
}
}
}

136
crates/daemons/pushd/src/consumers/inbound/fr_received.rs
View File

@@ -1,70 +1,44 @@
use std::collections::HashMap;
use std::{collections::HashMap, sync::Arc};
use crate::consumers::inbound::internal::*;
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use crate::utils::Consumer;
use anyhow::Result;
use async_trait::async_trait;
use lapin::{message::Delivery, Channel, Connection};
use log::debug;
use revolt_database::{events::rabbit::*, Database};
#[derive(Clone)]
#[allow(unused)]
pub struct FRReceivedConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
connection: Arc<Connection>,
channel: Arc<Channel>,
}
impl Channeled for FRReceivedConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl FRReceivedConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> FRReceivedConsumer {
FRReceivedConsumer {
#[async_trait]
impl Consumer for FRReceivedConsumer {
async fn create(
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<Channel>,
) -> Self {
Self {
db,
authifier_db,
conn: None,
channel: None,
connection,
channel,
}
}
async fn consume_event(
&mut self,
_channel: &Channel,
_deliver: Deliver,
_basic_properties: BasicProperties,
content: Vec<u8>,
) -> Result<()> {
let content = String::from_utf8(content)?;
let payload: FRReceivedPayload = serde_json::from_str(content.as_str())?;
fn channel(&self) -> &Arc<Channel> {
&self.channel
}
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(&self, delivery: Delivery) -> Result<()> {
let payload: FRReceivedPayload = serde_json::from_slice(&delivery.data)?;
debug!("Received FR received event");
@@ -80,36 +54,23 @@ impl FRReceivedConsumer {
extras: HashMap::new(),
};
let args: BasicPublishArguments;
let routing_key = match sub.endpoint.as_str() {
"apn" => &config.pushd.apn.queue,
"fcm" => &config.pushd.fcm.queue,
endpoint => {
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), endpoint.to_string());
if sub.endpoint == "apn" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
} else if sub.endpoint == "fcm" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.fcm.queue.as_str(),
)
.finish();
} else {
// web push (vapid)
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.vapid.queue.as_str(),
)
.finish();
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), sub.endpoint.clone());
}
&config.pushd.vapid.queue
}
};
let payload = serde_json::to_string(&sendable)?;
publish_message(self, payload.into(), args).await;
self.publish_message(payload.as_bytes(), &config.pushd.exchange, routing_key)
.await?;
}
}
}
@@ -117,24 +78,3 @@ impl FRReceivedConsumer {
Ok(())
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for FRReceivedConsumer {
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
if let Err(err) = self
.consume_event(channel, deliver, basic_properties, content)
.await
{
revolt_config::capture_anyhow(&err);
eprintln!("Failed to process friend request received event: {err:?}");
}
}
}

136
crates/daemons/pushd/src/consumers/inbound/generic.rs
View File

@@ -1,70 +1,44 @@
use std::collections::HashMap;
use std::{collections::HashMap, sync::Arc};
use crate::consumers::inbound::internal::*;
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use crate::utils::Consumer;
use anyhow::Result;
use async_trait::async_trait;
use lapin::{message::Delivery, Channel, Connection};
use log::debug;
use revolt_database::{events::rabbit::*, Database};
#[derive(Clone)]
#[allow(unused)]
pub struct GenericConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
connection: Arc<Connection>,
channel: Arc<Channel>,
}
impl Channeled for GenericConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl GenericConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> GenericConsumer {
GenericConsumer {
#[async_trait]
impl Consumer for GenericConsumer {
async fn create(
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<Channel>,
) -> Self {
Self {
db,
authifier_db,
conn: None,
channel: None,
connection,
channel,
}
}
async fn consume_event(
&mut self,
_channel: &Channel,
_deliver: Deliver,
_basic_properties: BasicProperties,
content: Vec<u8>,
) -> Result<()> {
let content = String::from_utf8(content)?;
let payload: MessageSentPayload = serde_json::from_str(content.as_str())?;
fn channel(&self) -> &Arc<Channel> {
&self.channel
}
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(&self, delivery: Delivery) -> Result<()> {
let payload: MessageSentPayload = serde_json::from_slice(&delivery.data)?;
debug!("Received message event on origin");
@@ -86,36 +60,23 @@ impl GenericConsumer {
extras: HashMap::new(),
};
let args: BasicPublishArguments;
let routing_key = match sub.endpoint.as_str() {
"apn" => &config.pushd.apn.queue,
"fcm" => &config.pushd.fcm.queue,
endpoint => {
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), endpoint.to_string());
if sub.endpoint == "apn" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
} else if sub.endpoint == "fcm" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.fcm.queue.as_str(),
)
.finish();
} else {
// web push (vapid)
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.vapid.queue.as_str(),
)
.finish();
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), sub.endpoint.clone());
}
&config.pushd.vapid.queue
}
};
let payload = serde_json::to_string(&sendable)?;
publish_message(self, payload.into(), args).await;
self.publish_message(payload.as_bytes(), &config.pushd.exchange, routing_key)
.await?;
}
}
}
@@ -123,24 +84,3 @@ impl GenericConsumer {
Ok(())
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for GenericConsumer {
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
if let Err(err) = self
.consume_event(channel, deliver, basic_properties, content)
.await
{
revolt_config::capture_anyhow(&err);
eprintln!("Failed to process generic event: {err:?}");
}
}
}

53
crates/daemons/pushd/src/consumers/inbound/internal.rs
View File

@@ -1,53 +0,0 @@
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::{Connection, OpenConnectionArguments},
BasicProperties,
};
use log::{debug, warn};
pub(crate) trait Channeled {
#[allow(unused)]
fn get_connection(&self) -> Option<&Connection>;
fn get_channel(&self) -> Option<&Channel>;
fn set_connection(&mut self, conn: Connection);
fn set_channel(&mut self, channel: Channel);
}
pub(crate) async fn make_channel<T: Channeled>(consumer: &mut T) {
let config = revolt_config::config().await;
let args = OpenConnectionArguments::new(
&config.rabbit.host,
config.rabbit.port,
&config.rabbit.username,
&config.rabbit.password,
);
let conn = amqprs::connection::Connection::open(&args).await.unwrap();
let channel = conn.open_channel(None).await.unwrap();
consumer.set_connection(conn);
consumer.set_channel(channel);
}
pub(crate) async fn publish_message<T: Channeled>(
consumer: &mut T,
payload: Vec<u8>,
args: BasicPublishArguments,
) {
let routing_key = &args.routing_key.clone();
let mut channel = consumer.get_channel();
if channel.is_none() {
make_channel(consumer).await;
channel = consumer.get_channel();
}
if let Some(chnl) = channel {
chnl.basic_publish(BasicProperties::default(), payload.clone(), args.clone())
.await
.unwrap();
debug!("Sent message to queue for target {}", routing_key);
} else {
warn!("Failed to unwrap channel (including attempt to make a channel)!")
}
}

149
crates/daemons/pushd/src/consumers/inbound/mass_mention.rs
View File

@@ -1,17 +1,13 @@
use std::{
collections::{HashMap, HashSet},
hash::RandomState,
sync::Arc,
};
use crate::{consumers::inbound::internal::*, utils};
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use crate::utils::{render_notification_content, Consumer};
use anyhow::Result;
use async_trait::async_trait;
use lapin::{message::Delivery, Channel, Connection};
use revolt_database::{
events::rabbit::*, util::bulk_permissions::BulkDatabasePermissionQuery, Database, Member,
MessageFlagsValue,
@@ -19,52 +15,18 @@ use revolt_database::{
use revolt_models::v0::{MessageFlags, PushNotification};
use revolt_result::ToRevoltError;
#[derive(Clone)]
#[allow(unused)]
pub struct MassMessageConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
}
impl Channeled for MassMessageConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
connection: Arc<Connection>,
channel: Arc<Channel>,
}
impl MassMessageConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> MassMessageConsumer {
MassMessageConsumer {
db,
authifier_db,
conn: None,
channel: None,
}
}
async fn fire_notification_for_users(
&mut self,
&self,
push: &PushNotification,
users: &[String],
) -> Result<()> {
@@ -84,56 +46,58 @@ impl MassMessageConsumer {
extras: HashMap::new(),
};
let args: BasicPublishArguments;
let routing_key = match sub.endpoint.as_str() {
"apn" => &config.pushd.apn.queue,
"fcm" => &config.pushd.fcm.queue,
endpoint => {
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), endpoint.to_string());
if sub.endpoint == "apn" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
} else if sub.endpoint == "fcm" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.fcm.queue.as_str(),
)
.finish();
} else {
// web push (vapid)
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.vapid.queue.as_str(),
)
.finish();
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), sub.endpoint.clone());
}
&config.pushd.vapid.queue
}
};
let payload = serde_json::to_string(&sendable)?;
publish_message(self, payload.into(), args).await;
self.publish_message(payload.as_bytes(), &config.pushd.exchange, routing_key)
.await?;
}
}
}
Ok(())
}
}
async fn consume_event(
&mut self,
_channel: &Channel,
_deliver: Deliver,
_basic_properties: BasicProperties,
content: Vec<u8>,
) -> Result<()> {
#[async_trait]
impl Consumer for MassMessageConsumer {
async fn create(
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<Channel>,
) -> Self {
Self {
db,
authifier_db,
connection,
channel,
}
}
fn channel(&self) -> &Arc<Channel> {
&self.channel
}
/// This consumer handles adding mentions for all the users affected by a mass mention ping, and then sends out push notifications.
async fn consume(&self, delivery: Delivery) -> Result<()> {
let mut payload: MassMessageSentPayload = serde_json::from_slice(&delivery.data)?;
let config = revolt_config::config().await;
let content = String::from_utf8(content)?;
let mut payload: MassMessageSentPayload = serde_json::from_str(content.as_str())?;
for push in payload.notifications.iter_mut() {
if let Ok(body) = utils::render_notification_content(push, &self.db)
if let Ok(body) = render_notification_content(push, &self.db)
.await
.to_internal_error()
{
@@ -280,24 +244,3 @@ impl MassMessageConsumer {
Ok(())
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for MassMessageConsumer {
/// This consumer handles adding mentions for all the users affected by a mass mention ping, and then sends out push notifications
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
if let Err(err) = self
.consume_event(channel, deliver, basic_properties, content)
.await
{
revolt_config::capture_anyhow(&err);
eprintln!("Failed to process mass message event: {err:?}");
}
}
}

143
crates/daemons/pushd/src/consumers/inbound/message.rs
View File

@@ -1,76 +1,46 @@
use std::collections::HashMap;
use std::{collections::HashMap, sync::Arc};
use crate::{consumers::inbound::internal::*, utils};
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use crate::utils::{render_notification_content, Consumer};
use anyhow::Result;
use async_trait::async_trait;
use lapin::{message::Delivery, Channel, Connection};
use log::debug;
use revolt_database::{events::rabbit::*, Database};
use revolt_result::ToRevoltError;
#[derive(Clone)]
#[allow(unused)]
pub struct MessageConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
connection: Arc<Connection>,
channel: Arc<Channel>,
}
impl Channeled for MessageConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl MessageConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> MessageConsumer {
MessageConsumer {
#[async_trait]
impl Consumer for MessageConsumer {
async fn create(
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<Channel>,
) -> Self {
Self {
db,
authifier_db,
conn: None,
channel: None,
connection,
channel,
}
}
async fn consume_event(
&mut self,
_channel: &Channel,
_deliver: Deliver,
_basic_properties: BasicProperties,
content: Vec<u8>,
) -> Result<()> {
let content = String::from_utf8(content)?;
let mut payload: MessageSentPayload = serde_json::from_str(content.as_str())?;
fn channel(&self) -> &Arc<Channel> {
&self.channel
}
if let Ok(body) = utils::render_notification_content(&payload.notification, &self.db)
.await
.to_internal_error()
{
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(&self, delivery: Delivery) -> Result<()> {
let mut payload: MessageSentPayload = serde_json::from_slice(&delivery.data)?;
if let Ok(body) = render_notification_content(&payload.notification, &self.db).await {
payload.notification.raw_body = Some(payload.notification.body);
payload.notification.body = body;
}
@@ -95,36 +65,22 @@ impl MessageConsumer {
extras: HashMap::new(),
};
let args: BasicPublishArguments;
let routing_key = match sub.endpoint.as_str() {
"apn" => &config.pushd.apn.queue,
"fcm" => &config.pushd.fcm.queue,
endpoint => {
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), endpoint.to_string());
if sub.endpoint == "apn" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
} else if sub.endpoint == "fcm" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.fcm.queue.as_str(),
)
.finish();
} else {
// web push (vapid)
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.vapid.queue.as_str(),
)
.finish();
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), sub.endpoint.clone());
}
&config.pushd.vapid.queue
}
};
let payload = serde_json::to_string(&sendable)?;
publish_message(self, payload.into(), args).await;
self.publish_message(payload.as_bytes(), &config.pushd.exchange, routing_key)
.await?;
}
}
}
@@ -132,24 +88,3 @@ impl MessageConsumer {
Ok(())
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for MessageConsumer {
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
if let Err(err) = self
.consume_event(channel, deliver, basic_properties, content)
.await
{
revolt_config::capture_anyhow(&err);
eprintln!("Failed to process message event: {err:?}");
}
}
}

1
crates/daemons/pushd/src/consumers/inbound/mod.rs
View File

@@ -3,6 +3,5 @@ pub mod dm_call;
pub mod fr_accepted;
pub mod fr_received;
pub mod generic;
mod internal;
pub mod mass_mention;
pub mod message;

159
crates/daemons/pushd/src/consumers/outbound/apn.rs
View File

@@ -1,12 +1,13 @@
use std::{borrow::Cow, collections::BTreeMap, io::Cursor};
use std::{borrow::Cow, collections::BTreeMap, io::Cursor, sync::Arc};
use amqprs::{channel::Channel as AmqpChannel, consumer::AsyncConsumer, BasicProperties, Deliver};
use anyhow::{anyhow, Result};
use crate::utils::Consumer;
use anyhow::Result;
use async_trait::async_trait;
use base64::{
engine::{self},
Engine as _,
};
use lapin::{message::Delivery, Channel as AMQPChannel, Connection};
use revolt_a2::{
request::{
notification::{DefaultAlert, NotificationOptions},
@@ -42,7 +43,7 @@ impl<'a> PayloadLike for MessagePayload<'a> {
fn get_device_token(&self) -> &'a str {
self.device_token
}
fn get_options(&self) -> &NotificationOptions {
fn get_options(&self) -> &NotificationOptions<'a> {
&self.options
}
}
@@ -68,16 +69,20 @@ impl<'a> PayloadLike for CallStartStopPayload<'a> {
fn get_device_token(&self) -> &'a str {
self.device_token
}
fn get_options(&self) -> &NotificationOptions {
fn get_options(&self) -> &NotificationOptions<'a> {
&self.options
}
}
// region: consumer
#[derive(Clone)]
#[allow(unused)]
pub struct ApnsOutboundConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<AMQPChannel>,
client: Client,
}
@@ -117,15 +122,21 @@ impl ApnsOutboundConsumer {
}
}
impl ApnsOutboundConsumer {
pub async fn new(db: Database) -> Result<ApnsOutboundConsumer, &'static str> {
#[async_trait]
impl Consumer for ApnsOutboundConsumer {
async fn create(
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<AMQPChannel>,
) -> Self {
let config = revolt_config::config().await;
if config.pushd.apn.pkcs8.is_empty()
|| config.pushd.apn.key_id.is_empty()
|| config.pushd.apn.team_id.is_empty()
{
return Err("Missing APN keys.");
panic!("Missing APN keys.");
}
let endpoint = if config.pushd.apn.sandbox {
@@ -148,18 +159,21 @@ impl ApnsOutboundConsumer {
)
.expect("could not create APN client");
Ok(ApnsOutboundConsumer { db, client })
Self {
db,
authifier_db,
connection,
channel,
client,
}
}
async fn consume_event(
&mut self,
_channel: &AmqpChannel,
_deliver: Deliver,
_basic_properties: BasicProperties,
content: Vec<u8>,
) -> Result<()> {
let content = String::from_utf8(content)?;
let payload: PayloadToService = serde_json::from_str(content.as_str())?;
fn channel(&self) -> &Arc<AMQPChannel> {
&self.channel
}
async fn consume(&self, delivery: Delivery) -> Result<()> {
let payload: PayloadToService = serde_json::from_slice(&delivery.data)?;
let payload_options = NotificationOptions {
apns_id: None,
@@ -170,20 +184,15 @@ impl ApnsOutboundConsumer {
apns_collapse_id: None,
};
let resp: Result<Response, Error>;
match payload.notification {
let resp = match payload.notification {
PayloadKind::FRReceived(alert) => {
let loc_args = vec![Cow::from(
alert
.from_user
.display_name
.or(Some(format!(
alert.from_user.display_name.clone().unwrap_or_else(|| {
format!(
"{}#{}",
alert.from_user.username, alert.from_user.discriminator
)))
.clone()
.ok_or_else(|| anyhow!("missing name"))?,
)
}),
)];
let apn_payload = Payload {
@@ -216,20 +225,17 @@ impl ApnsOutboundConsumer {
"Sending friend request received for user: {:}",
&payload.user_id
);
resp = self.client.send(apn_payload).await;
self.client.send(apn_payload).await
}
PayloadKind::FRAccepted(alert) => {
let loc_args = vec![Cow::from(
alert
.accepted_user
.display_name
.or(Some(format!(
alert.accepted_user.display_name.clone().unwrap_or_else(|| {
format!(
"{}#{}",
alert.accepted_user.username, alert.accepted_user.discriminator
)))
.clone()
.ok_or_else(|| anyhow!("missing name"))?,
)
}),
)];
let apn_payload = Payload {
@@ -262,7 +268,7 @@ impl ApnsOutboundConsumer {
"Sending friend request accept for user: {:}",
&payload.user_id
);
resp = self.client.send(apn_payload).await;
self.client.send(apn_payload).await
}
PayloadKind::Generic(alert) => {
let apn_payload = Payload {
@@ -295,7 +301,7 @@ impl ApnsOutboundConsumer {
"Sending generic notification for user: {:}",
&payload.user_id
);
resp = self.client.send(apn_payload).await;
self.client.send(apn_payload).await
}
PayloadKind::MessageNotification(alert) => {
@@ -334,7 +340,7 @@ impl ApnsOutboundConsumer {
"Sending message notification for user: {:}",
&payload.user_id
);
resp = self.client.send(apn_payload).await;
self.client.send(apn_payload).await
}
PayloadKind::BadgeUpdate(badge) => {
@@ -349,7 +355,7 @@ impl ApnsOutboundConsumer {
};
debug!("Sending badge update for user: {:}", &payload.user_id);
resp = self.client.send(apn_payload).await;
self.client.send(apn_payload).await
}
PayloadKind::DmCallStartEnd(alert) => {
@@ -378,58 +384,37 @@ impl ApnsOutboundConsumer {
"Sending call start/stop notification for user: {:}",
&payload.user_id
);
resp = self.client.send(apn_payload).await;
self.client.send(apn_payload).await
}
}
};
if let Err(err) = resp {
match err {
Error::ResponseError(Response {
error:
Some(ErrorBody {
reason: ErrorReason::BadDeviceToken | ErrorReason::Unregistered,
..
}),
..
}) => {
info!(
"Removing APNS subscription id {:} (user: {:}) due to invalid token",
&payload.session_id, &payload.user_id
);
if let Err(err) = self
.db
.remove_push_subscription_by_session_id(&payload.session_id)
.await
{
revolt_config::capture_error(&err);
}
}
err => {
match resp {
Err(Error::ResponseError(Response {
error:
Some(ErrorBody {
reason: ErrorReason::BadDeviceToken | ErrorReason::Unregistered,
..
}),
..
})) => {
info!(
"Removing APNS subscription id {:} (user: {:}) due to invalid token",
&payload.session_id, &payload.user_id
);
if let Err(err) = self
.db
.remove_push_subscription_by_session_id(&payload.session_id)
.await
{
revolt_config::capture_error(&err);
}
}
}
resp => {
resp?;
}
};
Ok(())
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for ApnsOutboundConsumer {
async fn consume(
&mut self,
channel: &AmqpChannel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
if let Err(err) = self
.consume_event(channel, deliver, basic_properties, content)
.await
{
revolt_config::capture_anyhow(&err);
eprintln!("Failed to process APN event: {err:?}");
}
}
}

156
crates/daemons/pushd/src/consumers/outbound/fcm.rs
View File

@@ -1,17 +1,16 @@
use std::{collections::HashMap, time::Duration};
use std::{collections::HashMap, sync::Arc, time::Duration};
use amqprs::{channel::Channel as AmqpChannel, consumer::AsyncConsumer, BasicProperties, Deliver};
use anyhow::{anyhow, bail, Result};
use crate::utils::Consumer;
use anyhow::{bail, Result};
use async_trait::async_trait;
use fcm_v1::{
auth::{Authenticator, ServiceAccountKey},
message::Message,
Client, Error as FcmError,
};
use lapin::{message::Delivery, Channel as AMQPChannel, Connection};
use revolt_config::config;
use revolt_database::{events::rabbit::*, Database};
use revolt_models::v0::{Channel, PushNotification};
use serde_json::Value;
/// Custom notification data
@@ -31,10 +30,12 @@ pub enum NotificationData {
image: Option<String>,
},
Message {
title: String,
message: String,
body: String,
image: String,
tag: String,
channel: String,
author: String,
author_name: String,
},
DmCallStartEnd {
initiator_id: String,
@@ -81,15 +82,19 @@ impl NotificationData {
}
}
NotificationData::Message {
title,
message,
body,
image,
tag,
channel,
author,
author_name,
} => {
data.insert("title".to_string(), Value::String(title));
data.insert("message".to_string(), Value::String(message));
data.insert("body".to_string(), Value::String(body));
data.insert("image".to_string(), Value::String(image));
data.insert("tag".to_string(), Value::String(tag));
data.insert("channel".to_string(), Value::String(channel));
data.insert("author".to_string(), Value::String(author));
data.insert("author_name".to_string(), Value::String(author_name));
}
NotificationData::DmCallStartEnd {
initiator_id,
@@ -110,37 +115,31 @@ impl NotificationData {
}
}
#[derive(Clone)]
#[allow(unused)]
pub struct FcmOutboundConsumer {
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<AMQPChannel>,
client: Client,
}
impl FcmOutboundConsumer {
fn format_title(&self, notification: &PushNotification) -> String {
// ideally this changes depending on context
// in a server, it would look like "Sendername, #channelname in servername"
// in a group, it would look like "Sendername in groupname"
// in a dm it should just be "Sendername".
// not sure how feasible all those are given the PushNotification object as it currently stands.
#[allow(deprecated)]
match &notification.channel {
Channel::DirectMessage { .. } => notification.author.clone(),
Channel::Group { name, .. } => format!("{}, #{}", notification.author, name),
Channel::TextChannel { name, .. } => {
format!("{} in #{}", notification.author, name)
}
_ => "Unknown".to_string(),
}
}
}
impl FcmOutboundConsumer {
pub async fn new(db: Database) -> Result<FcmOutboundConsumer, &'static str> {
#[async_trait]
impl Consumer for FcmOutboundConsumer {
async fn create(
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<AMQPChannel>,
) -> Self {
let config = revolt_config::config().await;
Ok(FcmOutboundConsumer {
Self {
db,
authifier_db,
connection,
channel,
client: Client::new(
Authenticator::service_account::<&str>(ServiceAccountKey {
key_type: Some(config.pushd.fcm.key_type),
@@ -160,33 +159,27 @@ impl FcmOutboundConsumer {
false,
Duration::from_secs(5),
),
})
}
}
async fn consume_event(
&mut self,
_channel: &AmqpChannel,
_deliver: Deliver,
_basic_properties: BasicProperties,
content: Vec<u8>,
) -> Result<()> {
let content = String::from_utf8(content)?;
let payload: PayloadToService = serde_json::from_str(content.as_str())?;
fn channel(&self) -> &Arc<AMQPChannel> {
&self.channel
}
async fn consume(&self, delivery: Delivery) -> Result<()> {
let payload: PayloadToService = serde_json::from_slice(&delivery.data)?;
#[allow(clippy::needless_late_init)]
let resp: Result<Message, FcmError>;
match payload.notification {
PayloadKind::FRReceived(alert) => {
let name = alert
.from_user
.display_name
.or(Some(format!(
let name = alert.from_user.display_name.clone().unwrap_or_else(|| {
format!(
"{}#{}",
alert.from_user.username, alert.from_user.discriminator
)))
.clone()
.ok_or_else(|| anyhow!("missing name"))?;
)
});
let data = NotificationData::FRReceived {
id: alert.from_user.id,
@@ -203,15 +196,12 @@ impl FcmOutboundConsumer {
}
PayloadKind::FRAccepted(alert) => {
let name = alert
.accepted_user
.display_name
.or(Some(format!(
let name = alert.accepted_user.display_name.clone().unwrap_or_else(|| {
format!(
"{}#{}",
alert.accepted_user.username, alert.accepted_user.discriminator
)))
.clone()
.ok_or_else(|| anyhow!("missing name"))?;
)
});
let data = NotificationData::FRAccepted {
id: alert.accepted_user.id,
@@ -244,10 +234,12 @@ impl FcmOutboundConsumer {
PayloadKind::MessageNotification(alert) => {
let data = NotificationData::Message {
title: self.format_title(&alert),
message: alert.message.id,
body: alert.body,
image: alert.icon,
tag: alert.tag,
channel: alert.message.channel,
author: alert.message.author,
author_name: alert.author,
};
let msg = Message {
@@ -282,43 +274,21 @@ impl FcmOutboundConsumer {
}
}
if let Err(err) = resp {
match err {
FcmError::Auth => {
if let Err(err) = self
.db
.remove_push_subscription_by_session_id(&payload.session_id)
.await
{
revolt_config::capture_error(&err);
}
}
err => {
match resp {
Err(FcmError::Auth) => {
if let Err(err) = self
.db
.remove_push_subscription_by_session_id(&payload.session_id)
.await
{
revolt_config::capture_error(&err);
}
}
}
res => {
res?;
}
};
Ok(())
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for FcmOutboundConsumer {
async fn consume(
&mut self,
channel: &AmqpChannel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
if let Err(err) = self
.consume_event(channel, deliver, basic_properties, content)
.await
{
revolt_config::capture_anyhow(&err);
eprintln!("Failed to process FCM event: {err:?}");
}
}
}

143
crates/daemons/pushd/src/consumers/outbound/vapid.rs
View File

@@ -1,6 +1,6 @@
use std::collections::HashMap;
use std::{collections::HashMap, sync::Arc};
use amqprs::{channel::Channel as AmqpChannel, consumer::AsyncConsumer, BasicProperties, Deliver};
use crate::utils::Consumer;
use anyhow::{anyhow, bail, Result};
use async_trait::async_trait;
@@ -8,46 +8,60 @@ use base64::{
engine::{self},
Engine as _,
};
use lapin::{message::Delivery, Channel as AMQPChannel, Connection};
use revolt_database::{events::rabbit::*, util::format_display_name, Database};
use web_push::{
ContentEncoding, IsahcWebPushClient, SubscriptionInfo, SubscriptionKeys, VapidSignatureBuilder,
WebPushClient, WebPushError, WebPushMessageBuilder,
};
#[derive(Clone)]
#[allow(unused)]
pub struct VapidOutboundConsumer {
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<AMQPChannel>,
client: IsahcWebPushClient,
pkey: Vec<u8>,
pkey: Arc<Vec<u8>>,
}
impl VapidOutboundConsumer {
pub async fn new(db: Database) -> Result<VapidOutboundConsumer> {
#[async_trait]
impl Consumer for VapidOutboundConsumer {
async fn create(
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<AMQPChannel>,
) -> Self {
let config = revolt_config::config().await;
if config.pushd.vapid.private_key.is_empty() | config.pushd.vapid.public_key.is_empty() {
bail!("no Vapid keys present");
if config.pushd.vapid.private_key.is_empty() || config.pushd.vapid.public_key.is_empty() {
panic!("no Vapid keys present");
}
let web_push_private_key = engine::general_purpose::URL_SAFE_NO_PAD
.decode(config.pushd.vapid.private_key)
.expect("valid `VAPID_PRIVATE_KEY`");
let web_push_private_key = Arc::new(
engine::general_purpose::URL_SAFE_NO_PAD
.decode(config.pushd.vapid.private_key)
.expect("valid `VAPID_PRIVATE_KEY`"),
);
Ok(VapidOutboundConsumer {
Self {
db,
authifier_db,
connection,
channel,
client: IsahcWebPushClient::new().unwrap(),
pkey: web_push_private_key,
})
}
}
async fn consume_event(
&mut self,
_channel: &AmqpChannel,
_deliver: Deliver,
_basic_properties: BasicProperties,
content: Vec<u8>,
) -> Result<()> {
let content = String::from_utf8(content)?;
let payload: PayloadToService = serde_json::from_str(content.as_str())?;
fn channel(&self) -> &Arc<AMQPChannel> {
&self.channel
}
async fn consume(&self, delivery: Delivery) -> Result<()> {
let payload: PayloadToService = serde_json::from_slice(&delivery.data)?;
let subscription = SubscriptionInfo {
endpoint: payload
@@ -65,10 +79,7 @@ impl VapidOutboundConsumer {
},
};
#[allow(clippy::needless_late_init)]
let payload_body: String;
match payload.notification {
let payload_body = match payload.notification {
PayloadKind::FRReceived(alert) => {
let name = alert
.from_user
@@ -83,7 +94,7 @@ impl VapidOutboundConsumer {
let mut body = HashMap::new();
body.insert("body", format!("{} sent you a friend request", name));
payload_body = serde_json::to_string(&body)?;
serde_json::to_string(&body)?
}
PayloadKind::FRAccepted(alert) => {
let name = alert
@@ -99,14 +110,10 @@ impl VapidOutboundConsumer {
let mut body = HashMap::new();
body.insert("body", format!("{} accepted your friend request", name));
payload_body = serde_json::to_string(&body)?;
}
PayloadKind::Generic(alert) => {
payload_body = serde_json::to_string(&alert)?;
}
PayloadKind::MessageNotification(alert) => {
payload_body = serde_json::to_string(&alert)?;
serde_json::to_string(&body)?
}
PayloadKind::Generic(alert) => serde_json::to_string(&alert)?,
PayloadKind::MessageNotification(alert) => serde_json::to_string(&alert)?,
PayloadKind::DmCallStartEnd(alert) => {
let initiator_name = if let Some(server_id) =
self.db.fetch_channel(&alert.channel_id).await?.server()
@@ -132,59 +139,41 @@ impl VapidOutboundConsumer {
_ => bail!("Invalid DmCallStart/End channel type"),
}
payload_body = serde_json::to_string(&body)?;
serde_json::to_string(&body)?
}
PayloadKind::BadgeUpdate(_) => {
bail!("Vapid cannot handle badge updates and they should not be sent here.");
}
}
};
match VapidSignatureBuilder::from_pem(std::io::Cursor::new(&self.pkey), &subscription) {
Ok(sig_builder) => match sig_builder.build() {
Ok(signature) => {
let mut builder = WebPushMessageBuilder::new(&subscription);
builder.set_vapid_signature(signature);
let signature = VapidSignatureBuilder::from_pem(
std::io::Cursor::new(self.pkey.as_ref()),
&subscription,
)?
.build()?;
builder.set_payload(ContentEncoding::AesGcm, payload_body.as_bytes());
let mut builder = WebPushMessageBuilder::new(&subscription);
builder.set_vapid_signature(signature);
match builder.build() {
Ok(msg) => {
if let Err(err) = self.client.send(msg).await {
if err == WebPushError::Unauthorized {
self.db
.remove_push_subscription_by_session_id(&payload.session_id)
.await?;
}
}
builder.set_payload(ContentEncoding::AesGcm, payload_body.as_bytes());
Ok(())
}
Err(err) => Err(err.into()),
}
let msg = builder.build()?;
match self.client.send(msg).await {
Err(WebPushError::Unauthorized) => {
if let Err(err) = self
.db
.remove_push_subscription_by_session_id(&payload.session_id)
.await
{
revolt_config::capture_error(&err);
}
Err(err) => Err(err.into()),
},
Err(err) => Err(err.into()),
}
}
}
}
res => {
res?;
}
};
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for VapidOutboundConsumer {
async fn consume(
&mut self,
channel: &AmqpChannel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
if let Err(err) = self
.consume_event(channel, deliver, basic_properties, content)
.await
{
revolt_config::capture_anyhow(&err);
eprintln!("Failed to process Vapid event: {err:?}");
}
Ok(())
}
}

251
crates/daemons/pushd/src/main.rs
View File

@@ -1,17 +1,16 @@
#[macro_use]
extern crate log;
use amqprs::{
channel::{
BasicConsumeArguments, Channel, ExchangeDeclareArguments, QueueBindArguments,
QueueDeclareArguments,
},
connection::{Connection, OpenConnectionArguments},
consumer::AsyncConsumer,
FieldTable,
use std::sync::Arc;
use lapin::{
options::{BasicConsumeOptions, ExchangeDeclareOptions, QueueBindOptions, QueueDeclareOptions},
types::{AMQPValue, FieldTable},
Channel, Connection, ConnectionProperties,
};
use revolt_config::{config, Settings};
use tokio::sync::Notify;
use revolt_database::Database;
use tokio::signal::ctrl_c;
mod consumers;
mod utils;
@@ -24,6 +23,8 @@ use consumers::{
outbound::{apn::ApnsOutboundConsumer, fcm::FcmOutboundConsumer, vapid::VapidOutboundConsumer},
};
use crate::utils::{Consumer, Delegate};
#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
async fn main() {
// Configure logging and environment
@@ -43,7 +44,24 @@ async fn main() {
panic!("Mongo is not in use, can't connect via authifier!")
}
let mut connections: Vec<(Channel, Connection)> = Vec::new();
let config = config().await;
let connection = Arc::new(
Connection::connect(
&format!(
"amqp://{}:{}@{}:{}",
&config.rabbit.username,
&config.rabbit.password,
&config.rabbit.host,
&config.rabbit.port,
),
ConnectionProperties::default(),
)
.await
.expect("Failed to connect to RabbitMQ"),
);
let mut channels = Vec::new();
// An explainer of how this works:
// The inbound connections are on separate routing keys, such that they only receive the proper payload
@@ -54,171 +72,178 @@ async fn main() {
// This'll require some interesting shimming if we need to add more events once this is in prod (different payloads between prod and test),
// but that sounds like a problem for future us.
let config = config().await;
// inbound: generic
connections.push(
make_queue_and_consume(
channels.push(
make_queue_and_consume::<GenericConsumer>(
&db,
&authifier,
&connection,
&config,
&config.pushd.generic_queue,
config.pushd.get_generic_routing_key().as_str(),
&config.pushd.get_generic_routing_key(),
None,
GenericConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
// inbound: messages
connections.push(
make_queue_and_consume(
channels.push(
make_queue_and_consume::<MessageConsumer>(
&db,
&authifier,
&connection,
&config,
&config.pushd.message_queue,
config.pushd.get_message_routing_key().as_str(),
&config.pushd.get_message_routing_key(),
None,
MessageConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
// inbound: FR received
connections.push(
make_queue_and_consume(
channels.push(
make_queue_and_consume::<FRReceivedConsumer>(
&db,
&authifier,
&connection,
&config,
&config.pushd.fr_received_queue,
config.pushd.get_fr_received_routing_key().as_str(),
&config.pushd.get_fr_received_routing_key(),
None,
FRReceivedConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
// inbound: FR accepted
connections.push(
make_queue_and_consume(
channels.push(
make_queue_and_consume::<FRAcceptedConsumer>(
&db,
&authifier,
&connection,
&config,
&config.pushd.fr_accepted_queue,
config.pushd.get_fr_accepted_routing_key().as_str(),
&config.pushd.get_fr_accepted_routing_key(),
None,
FRAcceptedConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
// inbound: Mass Mentions
connections.push(
make_queue_and_consume(
channels.push(
make_queue_and_consume::<MassMessageConsumer>(
&db,
&authifier,
&connection,
&config,
&config.pushd.mass_mention_queue,
config.pushd.get_mass_mention_routing_key().as_str(),
&config.pushd.get_mass_mention_routing_key(),
None,
MassMessageConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
// inbound: Dm Calls
connections.push(
make_queue_and_consume(
channels.push(
make_queue_and_consume::<DmCallConsumer>(
&db,
&authifier,
&connection,
&config,
&config.pushd.dm_call_queue,
config.pushd.get_dm_call_routing_key().as_str(),
&config.pushd.get_dm_call_routing_key(),
None,
DmCallConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
if !config.pushd.apn.pkcs8.is_empty() {
connections.push(
make_queue_and_consume(
channels.push(
make_queue_and_consume::<ApnsOutboundConsumer>(
&db,
&authifier,
&connection,
&config,
&config.pushd.apn.queue,
&config.pushd.apn.queue,
None,
ApnsOutboundConsumer::new(db.clone()).await.unwrap(),
)
.await,
);
let mut table = FieldTable::new();
table.insert("x-message-deduplication".try_into().unwrap(), "true".into());
let mut table = FieldTable::default();
table.insert("x-message-deduplication".into(), AMQPValue::Boolean(true));
connections.push(
make_queue_and_consume(
channels.push(
make_queue_and_consume::<AckConsumer>(
&db,
&authifier,
&connection,
&config,
&config.pushd.ack_queue,
&config.pushd.ack_queue,
Some(table),
AckConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
}
if !config.pushd.fcm.auth_uri.is_empty() {
connections.push(
make_queue_and_consume(
channels.push(
make_queue_and_consume::<FcmOutboundConsumer>(
&db,
&authifier,
&connection,
&config,
&config.pushd.fcm.queue,
&config.pushd.fcm.queue,
None,
FcmOutboundConsumer::new(db.clone()).await.unwrap(),
)
.await,
)
);
}
if !config.pushd.vapid.public_key.is_empty() {
connections.push(
make_queue_and_consume(
channels.push(
make_queue_and_consume::<VapidOutboundConsumer>(
&db,
&authifier,
&connection,
&config,
&config.pushd.vapid.queue,
&config.pushd.vapid.queue,
None,
VapidOutboundConsumer::new(db.clone()).await.unwrap(),
)
.await,
)
);
}
let guard = Notify::new();
guard.notified().await;
ctrl_c().await.unwrap();
for (channel, conn) in connections {
channel.close().await.expect("Unable to close channel");
conn.close().await.expect("Unable to close connection");
for channel in channels {
let _ = channel.close(0, "close".into()).await;
}
}
async fn make_queue_and_consume<F>(
db: &Database,
authifier_db: &authifier::Database,
connection: &Arc<Connection>,
config: &Settings,
queue_name: &str,
routing_key: &str,
queue_args: Option<FieldTable>,
consumer: F,
) -> (Channel, Connection)
) -> Arc<Channel>
where
F: AsyncConsumer + Send + 'static,
F: Consumer,
{
let connection = Connection::open(&OpenConnectionArguments::new(
&config.rabbit.host,
config.rabbit.port,
&config.rabbit.username,
&config.rabbit.password,
))
.await
.unwrap();
let channel = connection.open_channel(None).await.unwrap();
let channel = Arc::new(connection.create_channel().await.unwrap());
channel
.exchange_declare(
ExchangeDeclareArguments::new(&config.pushd.exchange, "direct")
.durable(true)
.finish(),
config.pushd.exchange.clone().into(),
lapin::ExchangeKind::Direct,
ExchangeDeclareOptions {
durable: true,
..Default::default()
},
FieldTable::default(),
)
.await
.expect("Failed to declare pushd exchange");
.expect("Failed to declare exchange");
let mut queue_name = queue_name.to_string();
@@ -230,35 +255,59 @@ where
let queue_name = queue_name.as_str();
let mut args = QueueDeclareArguments::new(queue_name);
args.durable(true);
if let Some(arg) = queue_args {
args.arguments(arg);
}
let args = args.finish();
_ = channel.queue_declare(args).await.unwrap().unwrap();
let args = QueueDeclareOptions {
durable: true,
..Default::default()
};
channel
.queue_bind(QueueBindArguments::new(
queue_name,
&config.pushd.exchange,
routing_key,
))
.queue_declare(queue_name.into(), args, queue_args.unwrap_or_default())
.await
.unwrap();
channel
.queue_bind(
queue_name.into(),
config.pushd.exchange.clone().into(),
routing_key.into(),
QueueBindOptions::default(),
FieldTable::default(),
)
.await
.expect(
"This probably means the revolt.notifications exchange does not exist in rabbitmq!",
);
let args = BasicConsumeArguments::new(queue_name, "")
.manual_ack(false)
.finish();
let routing_key = channel.basic_consume(consumer, args).await.unwrap();
let consumer = channel
.basic_consume(
queue_name.into(),
"".into(),
BasicConsumeOptions {
no_ack: true,
..Default::default()
},
FieldTable::default(),
)
.await
.unwrap();
info!(
"Consuming routing key {} as queue {}, tag {}",
routing_key, queue_name, routing_key
routing_key,
queue_name,
consumer.tag()
);
(channel, connection)
let delegate = Delegate(
F::create(
db.clone(),
authifier_db.clone(),
connection.clone(),
channel.clone(),
)
.await,
);
consumer.set_delegate(delegate);
channel
}

91
crates/daemons/pushd/src/utils/consumer.rs Normal file
View File

@@ -0,0 +1,91 @@
use std::{
future::{ready, Future},
pin::Pin,
sync::Arc,
};
use anyhow::Result;
use async_trait::async_trait;
use lapin::{
message::{Delivery, DeliveryResult},
options::BasicPublishOptions,
BasicProperties, Channel, Connection, ConsumerDelegate, Error as AMQPError,
};
use log::debug;
use revolt_database::Database;
#[async_trait]
pub trait Consumer: Clone + Send + Sync + 'static {
async fn create(
db: Database,
authifier_db: authifier::Database,
connection: Arc<Connection>,
channel: Arc<Channel>,
) -> Self;
fn channel(&self) -> &Arc<Channel>;
async fn consume(&self, delivery: Delivery) -> Result<()>;
async fn publish_message_with_options(
&self,
payload: &[u8],
exchange: &str,
routing_key: &str,
options: BasicPublishOptions,
properties: BasicProperties,
) -> Result<(), AMQPError> {
let channel = self.channel();
channel
.basic_publish(
exchange.into(),
routing_key.into(),
options,
payload,
properties,
)
.await?;
debug!("Sent message to queue for target {}", routing_key);
Ok(())
}
async fn publish_message(
&self,
payload: &[u8],
exchange: &str,
routing_key: &str,
) -> Result<(), AMQPError> {
self.publish_message_with_options(
payload,
exchange,
routing_key,
BasicPublishOptions::default(),
BasicProperties::default(),
)
.await
}
}
pub struct Delegate<C: Consumer>(pub C);
impl<C: Consumer> ConsumerDelegate for Delegate<C> {
fn on_new_delivery(
&self,
delivery: DeliveryResult,
) -> Pin<Box<dyn Future<Output = ()> + Send>> {
match delivery {
Ok(Some(delivery)) => {
let consumer = self.0.clone();
Box::pin(async move {
if let Err(e) = consumer.consume(delivery).await {
revolt_config::capture_anyhow(&e);
log::error!("{e:?}");
};
})
}
Ok(None) => Box::pin(ready(())),
Err(e) => Box::pin(async move { log::error!("Received bad delivery: {e:?}") }),
}
}
}

3
crates/daemons/pushd/src/utils/mod.rs
View File

@@ -1,2 +1,5 @@
mod renderer;
mod consumer;
pub use renderer::render_notification_content;
pub use consumer::{Consumer, Delegate};

5
crates/daemons/voice-ingress/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-voice-ingress"
version = "0.12.1"
version = "0.13.7"
license = "AGPL-3.0-or-later"
edition = "2021"
publish = false
@@ -44,6 +44,3 @@ revolt-permissions = { workspace = true }
livekit-api = { workspace = true }
livekit-protocol = { workspace = true }
livekit-runtime = { workspace = true, features = ["tokio"] }
# RabbitMQ
amqprs = { workspace = true }

4
crates/delta/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-delta"
version = "0.12.1"
version = "0.13.7"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <paulmakles@gmail.com>"]
edition = "2018"
@@ -64,7 +64,7 @@ schemars = { workspace = true }
revolt_rocket_okapi = { workspace = true, features = ["swagger"] }
# rabbit
amqprs = { workspace = true }
lapin = { workspace = true, features = ["tokio"] }
# core
authifier = { workspace = true }

52
crates/delta/src/main.rs
View File

@@ -9,8 +9,7 @@ pub mod routes;
pub mod util;
use revolt_config::config;
use revolt_database::events::client::EventV1;
use revolt_database::AMQP;
use revolt_database::{AMQP, events::client::EventV1};
use revolt_ratelimits::rocket as ratelimiter;
use rocket::{Build, Rocket};
use rocket_cors::{AllowedOrigins, CorsOptions};
@@ -18,14 +17,10 @@ use rocket_prometheus::PrometheusMetrics;
use std::net::Ipv4Addr;
use std::str::FromStr;
use amqprs::{
channel::ExchangeDeclareArguments,
connection::{Connection, OpenConnectionArguments},
};
use async_std::channel::unbounded;
use authifier::AuthifierEvent;
use rocket::data::ToByteUnit;
use revolt_database::voice::VoiceClient;
use rocket::data::ToByteUnit;
pub async fn web() -> Rocket<Build> {
// Get settings
@@ -36,7 +31,6 @@ pub async fn web() -> Rocket<Build> {
// Setup database
let db = revolt_database::DatabaseInfo::Auto.connect().await.unwrap();
log::info!("database_here {db:?}");
db.migrate_database().await.unwrap();
// Setup Authifier event channel
@@ -93,49 +87,11 @@ pub async fn web() -> Rocket<Build> {
)
.into();
let swagger_0_8 = revolt_rocket_okapi::swagger_ui::make_swagger_ui(
&revolt_rocket_okapi::swagger_ui::SwaggerUIConfig {
url: "/0.8/openapi.json".to_owned(),
..Default::default()
},
)
.into();
let swagger_0_8 = revolt_rocket_okapi::swagger_ui::make_swagger_ui(
&revolt_rocket_okapi::swagger_ui::SwaggerUIConfig {
url: "/0.8/openapi.json".to_owned(),
..Default::default()
},
)
.into();
// Voice handler
let voice_client = VoiceClient::new(config.api.livekit.nodes.clone());
// Configure Rabbit
let connection = Connection::open(&OpenConnectionArguments::new(
&config.rabbit.host,
config.rabbit.port,
&config.rabbit.username,
&config.rabbit.password,
))
.await
.expect("Failed to connect to RabbitMQ");
let channel = connection
.open_channel(None)
.await
.expect("Failed to open RabbitMQ channel");
channel
.exchange_declare(
ExchangeDeclareArguments::new(&config.pushd.exchange, "direct")
.durable(true)
.finish(),
)
.await
.expect("Failed to declare exchange");
let amqp = AMQP::new(connection, channel);
let amqp = AMQP::new_auto().await;
// Launch background task workers
revolt_database::tasks::start_workers(db.clone(), amqp.clone());
@@ -153,7 +109,6 @@ pub async fn web() -> Rocket<Build> {
.mount("/", rocket_cors::catch_all_options_routes())
.mount("/", ratelimiter::routes())
.mount("/swagger/", swagger)
.mount("/0.8/swagger/", swagger_0_8)
.manage(authifier)
.manage(db)
.manage(amqp)
@@ -166,6 +121,7 @@ pub async fn web() -> Rocket<Build> {
limits: rocket::data::Limits::default().limit("string", 5.megabytes()),
address: Ipv4Addr::new(0, 0, 0, 0).into(),
port: 14702,
ip_header: Some("X-Forwarded-For".into()),
..Default::default()
})
}

5
crates/delta/src/routes/channels/channel_ack.rs
View File

@@ -1,6 +1,6 @@
use revolt_database::{
util::{permissions::DatabasePermissionQuery, reference::Reference},
Database, User,
Database, User, AMQP,
};
use revolt_permissions::{calculate_channel_permissions, ChannelPermission};
use revolt_result::{create_error, Result};
@@ -14,6 +14,7 @@ use rocket_empty::EmptyResponse;
#[put("/<target>/ack/<message>")]
pub async fn ack(
db: &State<Database>,
amqp: &State<AMQP>,
user: User,
target: Reference<'_>,
message: Reference<'_>,
@@ -29,7 +30,7 @@ pub async fn ack(
.throw_if_lacking_channel_permission(ChannelPermission::ViewChannel)?;
channel
.ack(&user.id, message.id)
.ack(&user.id, message.id, amqp)
.await
.map(|_| EmptyResponse)
}

31
crates/delta/src/routes/channels/message_send.rs
View File

@@ -1,12 +1,14 @@
use std::time::Duration;
use redis_kiss::{get_connection, redis, AsyncCommands};
use revolt_database::events::client::EventV1;
use revolt_database::util::permissions::DatabasePermissionQuery;
use revolt_database::{
util::idempotency::IdempotencyKey, util::reference::Reference, Database, User,
};
use revolt_database::{Channel, Interactions, Message, AMQP};
use revolt_models::v0;
use revolt_models::v0::ChannelSlowmode;
use revolt_permissions::PermissionQuery;
use revolt_permissions::{calculate_channel_permissions, ChannelPermission};
use revolt_result::{create_error, Result};
@@ -84,6 +86,16 @@ pub async fn message_send(
.await
.unwrap_or(None);
if set_result.is_some() {
let idx_key = format!("slowmode_idx:{}", user.id);
conn.sadd::<_, _, ()>(&idx_key, channel_id.as_str())
.await
.ok();
conn.expire::<_, ()>(&idx_key, *channel_slowmode as usize)
.await
.ok();
}
// If `set_result` is None, the `NX` condition failed because the key already exists.
// This means the user is currently in slowmode.
if set_result.is_none() {
@@ -92,10 +104,29 @@ pub async fn message_send(
// Redis returns positive integers for valid TTLs
if ttl > 0 {
EventV1::UserSlowmodes {
slowmodes: vec![ChannelSlowmode {
channel_id: channel_id.to_string(),
duration: *channel_slowmode,
retry_after: ttl as u64,
}],
}
.private(user.id.clone())
.await;
return Err(create_error!(InSlowmode {
retry_after: ttl as u64
}));
}
} else {
EventV1::UserSlowmodes {
slowmodes: vec![ChannelSlowmode {
channel_id: channel_id.to_string(),
duration: *channel_slowmode,
retry_after: *channel_slowmode,
}],
}
.private(user.id.clone())
.await;
}
}
// If Redis connection fails, just skip the slowmode check

212
crates/delta/src/routes/customisation/emoji_edit.rs Normal file
View File

@@ -0,0 +1,212 @@
use revolt_database::{
util::{permissions::DatabasePermissionQuery, reference::Reference},
Database, EmojiParent, PartialEmoji, User,
};
use revolt_models::v0;
use revolt_permissions::{calculate_server_permissions, ChannelPermission};
use revolt_result::{create_error, Result};
use rocket::{serde::json::Json, State};
use validator::Validate;
/// # Edit Emoji
///
/// Edit an emoji by its id.
#[openapi(tag = "Emojis")]
#[patch("/emoji/<emoji_id>", data = "<data>")]
pub async fn edit_emoji(
db: &State<Database>,
user: User,
emoji_id: Reference<'_>,
data: Json<v0::DataEditEmoji>,
) -> Result<Json<v0::Emoji>> {
let data = data.into_inner();
data.validate().map_err(|error| {
create_error!(FailedValidation {
error: error.to_string()
})
})?;
let mut emoji = emoji_id.as_emoji(db).await?;
match &emoji.parent {
EmojiParent::Server { id } => {
let server = db.fetch_server(id.as_str()).await?;
let mut query = DatabasePermissionQuery::new(db, &user).server(&server);
calculate_server_permissions(&mut query)
.await
.throw_if_lacking_channel_permission(ChannelPermission::ManageCustomisation)?;
}
EmojiParent::Detached => return Err(create_error!(NotAuthenticated)),
}
if data.name.is_none() {
return Ok(Json(emoji.into()));
}
let partial = PartialEmoji { name: data.name };
emoji.update(db, partial).await?;
Ok(Json(emoji.into()))
}
#[cfg(test)]
mod test {
use crate::util::test::TestHarness;
use revolt_database::{Emoji, EmojiParent, Member};
use revolt_models::v0;
use rocket::http::{ContentType, Header, Status};
use ulid::Ulid;
#[rocket::async_test]
async fn edit_emoji_name_as_creator() {
let harness = TestHarness::new().await;
let (_, session, user) = harness.new_user().await;
let (server, _) = harness.new_server(&user).await;
let emoji_id = Ulid::new().to_string();
let emoji = Emoji {
id: emoji_id.clone(),
parent: EmojiParent::Server {
id: server.id.clone(),
},
creator_id: user.id.clone(),
name: "initial_name".to_string(),
animated: false,
nsfw: false,
};
emoji.create(&harness.db).await.expect("`Emoji` created");
let response = harness
.client
.patch(format!("/custom/emoji/{emoji_id}"))
.header(Header::new("x-session-token", session.token.to_string()))
.header(ContentType::JSON)
.body(
json!(v0::DataEditEmoji {
name: Some("renamed_emoji".to_string()),
})
.to_string(),
)
.dispatch()
.await;
assert_eq!(response.status(), Status::Ok);
let edited: v0::Emoji = response.into_json().await.expect("`Emoji`");
assert_eq!(edited.name, "renamed_emoji");
}
#[rocket::async_test]
async fn reject_invalid_emoji_name() {
let harness = TestHarness::new().await;
let (_, session, user) = harness.new_user().await;
let (server, _) = harness.new_server(&user).await;
let emoji_id = Ulid::new().to_string();
let emoji = Emoji {
id: emoji_id.clone(),
parent: EmojiParent::Server {
id: server.id.clone(),
},
creator_id: user.id.clone(),
name: "valid_name".to_string(),
animated: false,
nsfw: false,
};
emoji.create(&harness.db).await.expect("`Emoji` created");
let response = harness
.client
.patch(format!("/custom/emoji/{emoji_id}"))
.header(Header::new("x-session-token", session.token.to_string()))
.header(ContentType::JSON)
.body(
json!(v0::DataEditEmoji {
name: Some("Invalid Name".to_string()),
})
.to_string(),
)
.dispatch()
.await;
assert_eq!(response.status(), Status::BadRequest);
}
#[rocket::async_test]
async fn reject_edit_for_detached_emoji() {
let harness = TestHarness::new().await;
let (_, session, user) = harness.new_user().await;
let emoji_id = Ulid::new().to_string();
let emoji = Emoji {
id: emoji_id.clone(),
parent: EmojiParent::Detached,
creator_id: user.id.clone(),
name: "detached_name".to_string(),
animated: false,
nsfw: false,
};
emoji.create(&harness.db).await.expect("`Emoji` created");
let response = harness
.client
.patch(format!("/custom/emoji/{emoji_id}"))
.header(Header::new("x-session-token", session.token.to_string()))
.header(ContentType::JSON)
.body(
json!(v0::DataEditEmoji {
name: Some("should_not_apply".to_string()),
})
.to_string(),
)
.dispatch()
.await;
assert_eq!(response.status(), Status::Unauthorized);
}
#[rocket::async_test]
async fn reject_edit_for_creator_without_manage_customisation() {
let harness = TestHarness::new().await;
let (_, _, owner) = harness.new_user().await;
let (_, creator_session, creator) = harness.new_user().await;
let (server, _) = harness.new_server(&owner).await;
Member::create(&harness.db, &server, &creator, None)
.await
.expect("`Member` created");
let emoji_id = Ulid::new().to_string();
let emoji = Emoji {
id: emoji_id.clone(),
parent: EmojiParent::Server {
id: server.id.clone(),
},
creator_id: creator.id.clone(),
name: "member_uploaded_name".to_string(),
animated: false,
nsfw: false,
};
emoji.create(&harness.db).await.expect("`Emoji` created");
let response = harness
.client
.patch(format!("/custom/emoji/{emoji_id}"))
.header(Header::new(
"x-session-token",
creator_session.token.to_string(),
))
.header(ContentType::JSON)
.body(
json!(v0::DataEditEmoji {
name: Some("renamed_without_permission".to_string()),
})
.to_string(),
)
.dispatch()
.await;
assert_eq!(response.status(), Status::Forbidden);
}
}

2
crates/delta/src/routes/customisation/mod.rs
View File

@@ -3,12 +3,14 @@ use rocket::Route;
mod emoji_create;
mod emoji_delete;
mod emoji_edit;
mod emoji_fetch;
pub fn routes() -> (Vec<Route>, OpenApi) {
openapi_get_routes_spec![
emoji_create::create_emoji,
emoji_delete::delete_emoji,
emoji_edit::edit_emoji,
emoji_fetch::fetch_emoji
]
}

18
crates/delta/src/routes/root.rs
View File

@@ -57,6 +57,8 @@ pub struct RevoltFeatures {
pub livekit: VoiceFeature,
/// Limits
pub limits: LimitsConfig,
/// Legal links
pub legal_links: LegalLinks,
}
/// # Limits For Users
@@ -70,6 +72,17 @@ pub struct LimitsConfig {
pub default: UserLimits,
}
/// # Legal links
#[derive(Serialize, JsonSchema, Debug)]
pub struct LegalLinks {
/// Terms of Service URL
pub terms_of_service: String,
/// Privacy Policy URL
pub privacy_policy: String,
/// Guidelines URL
pub guidelines: String,
}
/// # Global limits
#[derive(Serialize, JsonSchema, Debug)]
pub struct GlobalLimits {
@@ -238,6 +251,11 @@ pub async fn root() -> Result<Json<RevoltConfig>> {
new_user: UserLimits::from_feature_limits(config.features.limits.new_user),
default: UserLimits::from_feature_limits(config.features.limits.default),
},
legal_links: LegalLinks {
terms_of_service: config.features.legal_links.terms_of_service,
privacy_policy: config.features.legal_links.privacy_policy,
guidelines: config.features.legal_links.guidelines,
},
},
ws: config.hosts.events,
app: config.hosts.app,

20
crates/delta/src/routes/servers/roles_edit.rs
View File

@@ -1,7 +1,7 @@
use revolt_database::{
util::{permissions::DatabasePermissionQuery, reference::Reference},
voice::{sync_voice_permissions, VoiceClient},
Database, PartialRole, User
Database, File, PartialRole, User,
};
use revolt_models::v0;
use revolt_permissions::{calculate_server_permissions, ChannelPermission};
@@ -47,14 +47,27 @@ pub async fn edit(
name,
colour,
hoist,
icon,
remove,
..
} = data;
if remove.contains(&v0::FieldsRole::Icon) {
if let Some(existing_icon) = &role.icon {
db.mark_attachment_as_deleted(&existing_icon.id).await?;
}
}
let mut final_icon = None;
if let Some(icon_id) = icon {
final_icon = Some(File::use_role_icon(db, &icon_id, &role_id, &user.id).await?);
}
let partial = PartialRole {
name,
colour,
hoist,
icon: final_icon,
..Default::default()
};
@@ -69,8 +82,9 @@ pub async fn edit(
for channel_id in &server.channels {
let channel = Reference::from_unchecked(channel_id).as_channel(db).await?;
sync_voice_permissions(db, voice_client, &channel, Some(&server), Some(&role_id)).await?;
};
sync_voice_permissions(db, voice_client, &channel, Some(&server), Some(&role_id))
.await?;
}
Ok(Json(role.into()))
} else {

16
crates/delta/src/routes/servers/server_ack.rs
View File

@@ -1,6 +1,6 @@
use revolt_database::{
util::{permissions::DatabasePermissionQuery, reference::Reference},
Database, User,
util::{acker, permissions::DatabasePermissionQuery, reference::Reference},
Database, User, AMQP,
};
use revolt_permissions::PermissionQuery;
use revolt_result::{create_error, Result};
@@ -12,7 +12,12 @@ use rocket_empty::EmptyResponse;
/// Mark all channels in a server as read.
#[openapi(tag = "Server Information")]
#[put("/<target>/ack")]
pub async fn ack(db: &State<Database>, user: User, target: Reference<'_>) -> Result<EmptyResponse> {
pub async fn ack(
db: &State<Database>,
amqp: &State<AMQP>,
user: User,
target: Reference<'_>,
) -> Result<EmptyResponse> {
if user.bot.is_some() {
return Err(create_error!(IsBot));
}
@@ -23,7 +28,6 @@ pub async fn ack(db: &State<Database>, user: User, target: Reference<'_>) -> Res
return Err(create_error!(NotFound));
}
db.acknowledge_channels(&user.id, &server.channels)
.await
.map(|_| EmptyResponse)
acker::ack_server(&user, &server, db, amqp).await?;
Ok(EmptyResponse)
}

10
crates/delta/src/routes/webhooks/mod.rs
View File

@@ -1,19 +1,23 @@
use rocket::Route;
use revolt_rocket_okapi::revolt_okapi::openapi3::OpenApi;
use rocket::Route;
mod webhook_delete;
mod webhook_delete_message;
mod webhook_delete_token;
mod webhook_edit;
mod webhook_edit_message;
mod webhook_edit_token;
mod webhook_execute;
mod webhook_fetch_token;
mod webhook_fetch;
mod webhook_execute_github;
mod webhook_fetch;
mod webhook_fetch_token;
pub fn routes() -> (Vec<Route>, OpenApi) {
openapi_get_routes_spec![
webhook_delete_message::webhook_delete_message,
webhook_delete_token::webhook_delete_token,
webhook_delete::webhook_delete,
webhook_edit_message::webhook_edit_message,
webhook_edit_token::webhook_edit_token,
webhook_edit::webhook_edit,
webhook_execute_github::webhook_execute_github,

27
crates/delta/src/routes/webhooks/webhook_delete_message.rs Normal file
View File

@@ -0,0 +1,27 @@
use revolt_database::{util::reference::Reference, Database};
use revolt_result::{create_error, Result};
use rocket::State;
use rocket_empty::EmptyResponse;
/// # Deletes a webhook message
///
/// Deletes a message sent by a webhook
#[openapi(tag = "Webhooks")]
#[delete("/<webhook_id>/<token>/<message_id>")]
pub async fn webhook_delete_message(
db: &State<Database>,
webhook_id: Reference<'_>,
token: String,
message_id: Reference<'_>,
) -> Result<EmptyResponse> {
let webhook = webhook_id.as_webhook(db).await?;
webhook.assert_token(&token)?;
let message = message_id.as_message(db).await?;
if message.author != webhook.id {
return Err(create_error!(CannotDeleteMessage));
}
message.delete(db).await.map(|_| EmptyResponse)
}

84
crates/delta/src/routes/webhooks/webhook_edit_message.rs Normal file
View File

@@ -0,0 +1,84 @@
use iso8601_timestamp::Timestamp;
use revolt_config::config;
use revolt_database::{
tasks::process_embeds::queue, util::reference::Reference, Database, Message, PartialMessage,
};
use revolt_models::v0::{self, DataEditMessage, Embed};
use revolt_models::validator::Validate;
use revolt_result::{create_error, Result};
use rocket::{serde::json::Json, State};
/// # Edits a webhook message
///
/// Edits a message sent by a webhook
#[openapi(tag = "Webhooks")]
#[patch("/<webhook_id>/<token>/<message_id>", data = "<data>")]
pub async fn webhook_edit_message(
db: &State<Database>,
webhook_id: Reference<'_>,
token: String,
message_id: Reference<'_>,
data: Json<DataEditMessage>,
) -> Result<Json<v0::Message>> {
let edit = data.into_inner();
edit.validate().map_err(|error| {
create_error!(FailedValidation {
error: error.to_string()
})
})?;
Message::validate_sum(
&edit.content,
edit.embeds.as_deref().unwrap_or_default(),
config().await.features.limits.default.message_length,
)?;
let webhook = webhook_id.as_webhook(db).await?;
webhook.assert_token(&token)?;
let mut message = message_id.as_message(db).await?;
if message.author != webhook.id {
return Err(create_error!(CannotEditMessage));
}
message.edited = Some(Timestamp::now_utc());
let mut partial = PartialMessage {
edited: message.edited,
..Default::default()
};
// 1. Handle content update
if let Some(content) = &edit.content {
partial.content = Some(content.clone());
}
// 2. Clear any auto generated embeds
let mut new_embeds = vec![];
if let Some(embeds) = &message.embeds {
for embed in embeds {
if let Embed::Text(embed) = embed {
new_embeds.push(Embed::Text(embed.clone()))
}
}
}
// 3. Replace if we are given new embeds
if let Some(embeds) = edit.embeds {
new_embeds.clear();
for embed in embeds {
new_embeds.push(message.create_embed(db, embed).await?);
}
}
partial.embeds = Some(new_embeds);
message.update(db, partial, vec![]).await?;
// Queue up a task for processing embeds
if let Some(content) = edit.content {
queue(message.channel.to_string(), message.id.to_string(), content).await;
}
Ok(Json(message.into_model(None, None)))
}

18
crates/delta/src/util/test.rs
View File

@@ -4,7 +4,7 @@ use authifier::{
};
use futures::StreamExt;
use rand::Rng;
use redis_kiss::redis::aio::PubSub;
use redis_kiss::{redis::aio::PubSub};
use revolt_database::{
events::client::EventV1, Channel, Database, Member, Message, PartialRole, Server, User, AMQP,
};
@@ -25,8 +25,6 @@ pub struct TestHarness {
impl TestHarness {
pub async fn new() -> TestHarness {
let config = revolt_config::config().await;
let client = Client::tracked(crate::web().await)
.await
.expect("valid rocket instance");
@@ -49,19 +47,7 @@ impl TestHarness {
.expect("`Authifier`")
.clone();
let connection = amqprs::connection::Connection::open(
&amqprs::connection::OpenConnectionArguments::new(
&config.rabbit.host,
config.rabbit.port,
&config.rabbit.username,
&config.rabbit.password,
),
)
.await
.unwrap();
let channel = connection.open_channel(None).await.unwrap();
let amqp = AMQP::new(connection, channel);
let amqp = AMQP::new_auto().await;
TestHarness {
client,

5
crates/services/autumn/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-autumn"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "AGPL-3.0-or-later"
publish = false
@@ -18,6 +18,7 @@ kamadak-exif = { workspace = true }
# revolt_little_exif = "0.5.1"
image = { workspace = true }
thumbhash = { workspace = true }
lcms2 = { workspace = true }
# File processing
revolt_clamav-client = { workspace = true }
@@ -31,7 +32,7 @@ imagesize = { workspace = true }
# Utility
lazy_static = { workspace = true }
moka = { workspace = true, features = ["future"] }
url-escape = { workspace = true }
# Serialisation
strum_macros = { workspace = true }
serde_json = { workspace = true }

5
crates/services/autumn/src/api.rs
View File

@@ -24,6 +24,7 @@ use sha2::Digest;
use tempfile::NamedTempFile;
use tokio::time::Instant;
use tower_http::cors::{AllowHeaders, Any, CorsLayer};
use url_escape::encode_component;
use utoipa::ToSchema;
use crate::{
@@ -479,8 +480,10 @@ async fn fetch_file(
// Ensure filename is correct
if file_name != file.filename {
if file_name == "original" {
let safe_filename = encode_component(&file.filename);
return Ok(
Redirect::permanent(&format!("/{tag}/{file_id}/{}", file.filename)).into_response(),
Redirect::permanent(&format!("/{tag}/{file_id}/{}", safe_filename)).into_response(),
);
}

112
crates/services/autumn/src/exif.rs
View File

@@ -1,13 +1,24 @@
use std::io::{Cursor, Read};
use crate::utils::apply_icc_profile;
use exif::Reader;
use image::{ImageFormat, ImageReader};
use image::{ImageEncoder, ImageReader};
use revolt_config::report_internal_error;
use revolt_database::Metadata;
use revolt_result::{create_error, Result};
use tempfile::NamedTempFile;
use tokio::process::Command;
macro_rules! encode_with_icc {
($encoder:expr, $icc:expr, $image:expr, $width:expr, $height:expr, $color:expr) => {{
let mut encoder = $encoder;
if let Some(icc) = $icc {
let _ = encoder.set_icc_profile(icc.clone());
}
encoder.write_image($image, $width, $height, $color)
}};
}
/// Strip EXIF data from given file and produce new file and metadata
pub async fn strip_metadata(
file: NamedTempFile,
@@ -17,8 +28,8 @@ pub async fn strip_metadata(
) -> Result<(Vec<u8>, Metadata)> {
match &metadata {
Metadata::Image {
width,
height,
width: _,
height: _,
thumbhash,
animated,
} => match mime {
@@ -46,11 +57,12 @@ pub async fn strip_metadata(
let mut cursor = Cursor::new(buf);
// Decode the image
let image = report_internal_error!(report_internal_error!(ImageReader::new(
&mut cursor
)
.with_guessed_format())?
.decode());
let reader =
report_internal_error!(ImageReader::new(&mut cursor).with_guessed_format())?;
let mut decoder = report_internal_error!(reader.into_decoder())?;
let mut icc_profile =
report_internal_error!(image::ImageDecoder::icc_profile(&mut decoder))?;
let mut image = report_internal_error!(image::DynamicImage::from_decoder(decoder))?;
// Reset read position
cursor.set_position(0);
@@ -71,38 +83,68 @@ pub async fn strip_metadata(
// Apply the EXIF rotation
// See https://jdhao.github.io/2019/07/31/image_rotation_exif_info/
report_internal_error!(match &rotation {
2 => image?.fliph(),
3 => image?.rotate180(),
4 => image?.rotate180().fliph(),
5 => image?.rotate90().fliph(),
6 => image?.rotate90(),
7 => image?.rotate270().fliph(),
8 => image?.rotate270(),
_ => image?,
}
.write_to(
&mut writer,
match mime {
"image/jpeg" => ImageFormat::Jpeg,
"image/png" => ImageFormat::Png,
"image/avif" => ImageFormat::Avif,
"image/tiff" => ImageFormat::Tiff,
_ => todo!(),
},
))?;
// Calculate dimensions after rotation.
let (width, height) = match &rotation {
2 | 4 | 5 | 7 => (*height, *width),
_ => (*width, *height),
image = match &rotation {
2 => image.fliph(),
3 => image.rotate180(),
4 => image.rotate180().fliph(),
5 => image.rotate90().fliph(),
6 => image.rotate90(),
7 => image.rotate270().fliph(),
8 => image.rotate270(),
_ => image,
};
if let Some(icc) = &icc_profile {
image = apply_icc_profile(image, icc);
icc_profile = None;
}
let color_type = image.color();
let width = image.width();
let height = image.height();
report_internal_error!(match mime {
"image/jpeg" => encode_with_icc!(
image::codecs::jpeg::JpegEncoder::new(&mut writer),
&icc_profile,
image.as_bytes(),
width,
height,
color_type.into()
),
"image/png" => encode_with_icc!(
image::codecs::png::PngEncoder::new(&mut writer),
&icc_profile,
image.as_bytes(),
width,
height,
color_type.into()
),
"image/avif" => {
// avif encoder doesn't implement set_icc_profile currently
image::codecs::avif::AvifEncoder::new(&mut writer).write_image(
image.as_bytes(),
width,
height,
color_type.into(),
)
}
"image/tiff" => encode_with_icc!(
image::codecs::tiff::TiffEncoder::new(&mut writer),
&icc_profile,
image.as_bytes(),
width,
height,
color_type.into()
),
_ => unreachable!(),
})?;
Ok((
bytes,
Metadata::Image {
width,
height,
width: width as isize,
height: height as isize,
thumbhash: thumbhash.clone(),
animated: *animated,
},

1
crates/services/autumn/src/main.rs
View File

@@ -18,6 +18,7 @@ pub mod exif;
pub mod metadata;
pub mod mime_type;
mod ratelimits;
mod utils;
#[derive(FromRef, Clone)]
struct AppState {

31
crates/services/autumn/src/metadata.rs
View File

@@ -1,5 +1,6 @@
use std::io::Cursor;
use crate::utils::apply_icc_profile;
use image::{GenericImageView, ImageError, ImageReader};
use revolt_database::Metadata;
use revolt_files::{image_size, is_animated, video_size};
@@ -27,16 +28,26 @@ pub fn generate_metadata(f: &NamedTempFile, mime_type: &str) -> Metadata {
.map(|(width, height)| Metadata::Image {
width: width as isize,
height: height as isize,
thumbhash: ImageReader::open(f)
.and_then(|r| r.with_guessed_format())
.map_err(ImageError::from)
.and_then(|r| r.decode())
.map(|img| img.thumbnail(100, 100))
.map(|img| (img.dimensions(), img.to_rgba8().into_raw()))
.map(|((width, height), rgba)| {
thumbhash::rgba_to_thumb_hash(width as usize, height as usize, &rgba)
})
.ok(),
thumbhash: (|| {
let reader = ImageReader::open(f).ok()?.with_guessed_format().ok()?;
let mut decoder = reader.into_decoder().ok()?;
let icc_profile = image::ImageDecoder::icc_profile(&mut decoder)
.ok()
.flatten();
let mut img = image::DynamicImage::from_decoder(decoder).ok()?;
if let Some(icc) = icc_profile {
img = apply_icc_profile(img, &icc);
}
let img = img.thumbnail(100, 100);
let (width, height) = img.dimensions();
Some(thumbhash::rgba_to_thumb_hash(
width as usize,
height as usize,
&img.into_rgba8().into_raw(),
))
})(),
animated: is_animated(f, mime_type).or(Some(false)),
})
.unwrap_or_default()

31
crates/services/autumn/src/utils.rs Normal file
View File

@@ -0,0 +1,31 @@
/// Convert image to sRGB using the provided ICC profile.
/// Returns the converted image, or the original if conversion fails.
pub fn apply_icc_profile(image: image::DynamicImage, icc: &[u8]) -> image::DynamicImage {
let Ok(src_profile) = lcms2::Profile::new_icc(icc) else {
return image;
};
let dst_profile = lcms2::Profile::new_srgb();
let format = if image.color().has_alpha() {
lcms2::PixelFormat::RGBA_8
} else {
lcms2::PixelFormat::RGB_8
};
let Ok(t) = lcms2::Transform::new(
&src_profile,
format,
&dst_profile,
format,
lcms2::Intent::Perceptual,
) else {
return image;
};
if image.color().has_alpha() {
let mut rgba_image = image.into_rgba8();
t.transform_in_place(rgba_image.as_mut());
image::DynamicImage::ImageRgba8(rgba_image)
} else {
let mut rgb_image = image.into_rgb8();
t.transform_in_place(rgb_image.as_mut());
image::DynamicImage::ImageRgb8(rgb_image)
}
}

2
crates/services/gifbox/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-gifbox"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "AGPL-3.0-or-later"
publish = false

4
crates/services/january/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-january"
version = "0.12.1"
version = "0.13.7"
edition = "2021"
license = "AGPL-3.0-or-later"
publish = false
@@ -27,6 +27,8 @@ tokio = { workspace = true, features = [] }
# Web requests
reqwest = { workspace = true, features = ["json"] }
pdk-ip-filter-lib = "1.8.0"
url = { workspace = true }
# Logging
tracing = { workspace = true }

259
crates/services/january/src/requests.rs
View File

@@ -1,42 +1,43 @@
use encoding_rs::{Encoding, UTF_8_INIT};
use lazy_static::lazy_static;
use mime::Mime;
use pdk_ip_filter_lib::IpFilter;
use regex::Regex;
use reqwest::{
dns::{Addrs, Name, Resolve},
header::{self, CONTENT_TYPE},
redirect, Client, Response,
};
use revolt_config::report_internal_error;
use revolt_config::{config, report_internal_error};
use revolt_files::{create_thumbnail, decode_image, image_size_vec, is_valid_image, video_size};
use revolt_models::v0::{Embed, Image, ImageSize, Video};
use revolt_result::{create_error, Error, Result};
use revolt_result::{create_error, Error, Result, ToRevoltError};
use std::net::{IpAddr, SocketAddr};
use std::{
io::{Cursor, Write},
str::FromStr,
time::Duration,
};
use url::{Host, Url};
lazy_static! {
/// Request client
static ref CLIENT: Client = reqwest::Client::builder()
.dns_resolver(CachedDnsResolver {})
.timeout(Duration::from_secs(10)) // TODO config
.connect_timeout(Duration::from_secs(5)) // TODO config
.redirect(redirect::Policy::custom(|attempt| {
if attempt.previous().len() > 5 { // TODO config
attempt.error("too many redirects")
} else if attempt.url().host_str() == Some("jan.revolt.chat") { // TODO config
attempt.stop()
} else {
attempt.follow()
}
}))
.redirect(redirect::Policy::none())
.build()
.expect("reqwest Client");
/// Spoof User Agent as Discord
static ref RE_USER_AGENT_SPOOFING_AS_DISCORD: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:vx|fx)?twitter|(?:fixv|fixup)?x|(?:old\\.|new\\.|www\\.)reddit).com").expect("valid regex");
static ref RE_USER_AGENT_SPOOFING_AS_DISCORD: Regex = Regex::new("^(?:(?:vx|fx)?twitter|(?:fixv|fixup)?x|(?:old\\.|new\\.|www\\.)reddit).com").expect("valid regex");
/// Regex for matching new Reddit URLs
static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex");
static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex");
/// Regex for matching YouTube Shorts URLs
static ref RE_URL_YOUTUBE_SHORTS: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www\\.)?youtube\\.com)/shorts/([a-zA-Z0-9_-]+)").expect("valid regex");
/// Cache for proxy results
static ref PROXY_CACHE: moka::future::Cache<String, Result<(String, Vec<u8>)>> = moka::future::Cache::builder()
@@ -59,6 +60,72 @@ lazy_static! {
.max_capacity(10_000) // Cache up to 10k embeds
.time_to_live(Duration::from_secs(60)) // For up to 1 minute
.build();
static ref DNS_CACHE: moka::future::Cache<String, Vec<SocketAddr>> = moka::future::Cache::builder()
.max_capacity(10_000)
.time_to_idle(Duration::from_secs(30))
.build();
static ref IP_BLOCKLIST: IpFilter = IpFilter::block(&[
"0.0.0.0/8",
"10.0.0.0/8",
"192.168.0.0/16",
"127.0.0.0/8",
"172.16.0.0/12",
"169.254.0.0/16",
"::1",
"fc00::/7",
]
).unwrap();
}
#[derive(Clone)]
pub struct IPRequest {
url: Url,
ip: IpAddr,
pub blocked: bool,
}
impl From<IPRequest> for Url {
fn from(value: IPRequest) -> Self {
let mut url = value.url.clone();
url.set_host(Some(&value.ip.to_string()))
.map(|_| url)
.unwrap_or(value.url)
}
}
struct CachedDnsResolver {}
impl reqwest::dns::Resolve for CachedDnsResolver {
fn resolve(&self, name: Name) -> reqwest::dns::Resolving {
Box::pin(async move {
{
if let Some(addrs) = DNS_CACHE.get(&name.as_str().to_string()).await {
let resp: Addrs = Box::new(addrs.clone().into_iter());
return Ok(resp);
}
}
let mut lookup = name.as_str().to_string();
if !lookup.contains(":") {
lookup += ":0";
}
let fallback: Vec<SocketAddr> = tokio::net::lookup_host(&lookup)
.await
.map_err(|e| -> Box<dyn std::error::Error + Send + Sync> { Box::new(e) })?
.collect();
{
DNS_CACHE
.insert(name.as_str().to_string().clone(), fallback.clone())
.await;
let addrs: Addrs = Box::new(fallback.clone().into_iter());
Ok(addrs)
}
})
}
}
/// Information about a successful request
@@ -73,7 +140,7 @@ impl Request {
if let Some(hit) = PROXY_CACHE.get(url).await {
hit
} else {
let Request { response, mime } = Request::new(url).await?;
let Request { response, mime } = Request::new_from_str(url).await?;
if matches!(mime.type_(), mime::IMAGE | mime::VIDEO) {
let bytes = report_internal_error!(response.bytes().await);
@@ -135,7 +202,7 @@ impl Request {
let request = if let Some(request) = request {
request
} else {
let request = Request::new(url).await?;
let request = Request::new_from_str(url).await?;
if matches!(request.mime.type_(), mime::IMAGE) {
request
} else {
@@ -173,7 +240,7 @@ impl Request {
let response = if let Some(Request { response, .. }) = request {
response
} else {
let Request { response, mime } = Request::new(url).await?;
let Request { response, mime } = Request::new_from_str(url).await?;
if matches!(mime.type_(), mime::VIDEO) {
response
} else {
@@ -208,11 +275,18 @@ impl Request {
.to_string();
}
// Re-map Youtube Shorts to regular Youtube links
if let Some(captures) = RE_URL_YOUTUBE_SHORTS.captures(&url) {
if let Some(video_id) = captures.get(1) {
url = format!("https://youtube.com/watch?v={}", video_id.as_str());
}
}
// Generate the actual embed
if let Some(hit) = EMBED_CACHE.get(&url).await {
Ok(hit)
} else {
let request = Request::new(&url).await?;
let request = Request::new_from_str(&url).await?;
let embed = match (request.mime.type_(), request.mime.subtype()) {
(_, mime::HTML) => {
let content_type = request
@@ -255,15 +329,27 @@ impl Request {
}
/// Send a new request to a service
pub async fn new(url: &str) -> Result<Request> {
let response = CLIENT
pub async fn new(url: Url) -> Result<Request> {
let mut url = url;
let url_host_str = url.host_str().ok_or(create_error!(ProxyError))?.to_string();
let mut blocker = Request::url_is_blacklisted(&url).await?;
if blocker.blocked {
return Err(create_error!(InvalidOperation));
}
let mut redirect_count = 0;
loop {
let response = CLIENT
.get(url)
.header(
"User-Agent",
if RE_USER_AGENT_SPOOFING_AS_DISCORD.is_match(url) {
if RE_USER_AGENT_SPOOFING_AS_DISCORD.is_match(&url_host_str) {
"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"
} else {
"Mozilla/5.0 (compatible; January/2.0; +https://github.com/revoltchat/backend)"
"Mozilla/5.0 (compatible; January/2.0; +https://github.com/stoatchat/stoatchat)"
},
)
.header("Accept-Language", "en-US,en;q=0.5")
@@ -271,31 +357,130 @@ impl Request {
.await
.map_err(|_| create_error!(ProxyError))?;
if !response.status().is_success() {
tracing::error!("{:?}", response);
return Err(create_error!(ProxyError));
if response.status().is_redirection() {
redirect_count += 1;
if redirect_count > 5 {
return Err(create_error!(ProxyError));
}
if let Some(location) = response.headers().get("location") {
let location = location.to_str().map_err(|_| create_error!(ProxyError))?;
url = Url::from_str(location).to_internal_error()?;
blocker = Request::url_is_blacklisted(&url).await?;
if blocker.blocked {
return Err(create_error!(InvalidOperation));
}
continue;
} else {
return Err(create_error!(ProxyError));
}
}
if !response.status().is_success() {
tracing::error!("{:?}", response);
return Err(create_error!(ProxyError));
}
let content_type = response
.headers()
.get(CONTENT_TYPE)
.ok_or(create_error!(ProxyError))?
.to_str()
.map_err(|_| create_error!(ProxyError))?;
let mime: mime::Mime = content_type
.parse()
.map_err(|_| create_error!(ProxyError))?;
return Ok(Request { response, mime });
}
}
let content_type = response
.headers()
.get(CONTENT_TYPE)
.ok_or(create_error!(ProxyError))?
.to_str()
.map_err(|_| create_error!(ProxyError))?;
let mime: mime::Mime = content_type
.parse()
.map_err(|_| create_error!(ProxyError))?;
Ok(Request { response, mime })
pub async fn new_from_str(url: &str) -> Result<Request> {
let proper_url = Url::parse(url).map_err(|_| create_error!(ProxyError))?;
Request::new(proper_url).await
}
/// Check if something exists
pub async fn exists(url: &str) -> bool {
pub async fn exists(url: Url) -> bool {
if let Ok(response) = CLIENT.head(url).send().await {
response.status().is_success()
} else {
false
}
}
pub async fn exists_from_str(url: &str) -> Result<bool> {
let proper_url = Url::parse(url).map_err(|_| create_error!(ProxyError))?;
Ok(Request::exists(proper_url).await)
}
pub async fn url_is_blacklisted(url: &Url) -> Result<IPRequest> {
let resolved_address: IpAddr;
if let Some(host) = url.host() {
match host {
Host::Ipv4(ipv4) => {
resolved_address = ipv4.into();
if !IP_BLOCKLIST.is_allowed(&ipv4.to_string()) {
return Err(create_error!(InvalidOperation));
}
}
Host::Ipv6(ipv6) => {
resolved_address = ipv6.into();
if !IP_BLOCKLIST.is_allowed(&ipv6.to_string()) {
return Err(create_error!(InvalidOperation));
}
}
Host::Domain(domain) => {
let domain = domain.to_string();
let config = config().await;
// First step: TLDs and blocked domains
if !domain.contains(".") // lazily block TLDs
|| config.january.blocked_domains.iter().any(|x| x == &domain)
{
return Err(create_error!(InvalidOperation));
}
// Second step: resolve the IP and check the blocklist
let resolver = CachedDnsResolver {};
if let Ok(mut resolved_ip) = resolver
.resolve(
Name::from_str(&domain)
.map_err(|_| create_error!(ProxyError))
.unwrap(),
)
.await
{
if let Some(resolved_ip) = resolved_ip.next() {
resolved_address = resolved_ip.ip();
let resolved_string = resolved_address.to_string();
if !IP_BLOCKLIST.is_allowed(&resolved_string)
|| resolved_string.contains("::ffff:")
{
return Err(create_error!(InvalidOperation));
}
} else {
return Err(create_error!(InvalidOperation));
}
} else {
return Err(create_error!(ProxyError));
}
}
}
} else {
return Err(create_error!(ProxyError));
};
Ok(IPRequest {
url: url.clone(),
ip: resolved_address,
blocked: false,
})
}
}

5
crates/services/january/src/website_embed.rs
View File

@@ -190,7 +190,7 @@ pub async fn create_website_embed(original_url: &str, document: &str) -> Option<
pub async fn populate_special(original_url: String, metadata: &mut WebsiteMetadata) {
lazy_static! {
static ref RE_YOUTUBE: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www|m)\\.)?(?:(?:youtube\\.com|youtu.be))(?:/(?:[\\w\\-]+\\?v=|embed/|v/)?)([\\w\\-]+)(?:\\S+)?$").unwrap();
static ref RE_YOUTUBE: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www|m)\\.)?(?:(?:youtube\\.com|youtu.be))(?:/(?:[\\w\\-]+\\?v=|embed/|v/|shorts/)?)([\\w\\-]+)(?:\\S+)?$").unwrap();
static ref RE_LIGHTSPEED: Regex = Regex::new("^(?:https?://)?(?:[\\w]+\\.)?lightspeed\\.tv/([a-z0-9_]{4,25})").unwrap();
@@ -236,11 +236,12 @@ pub async fn populate_special(original_url: String, metadata: &mut WebsiteMetada
metadata.site_name.take();
// Verify the video exists
if !crate::requests::Request::exists(&format!(
if !crate::requests::Request::exists_from_str(&format!(
"http://img.youtube.com/vi/{}/sddefault.jpg",
id
))
.await
.unwrap_or(false)
{
return;
}

16
docs/docs/developers/events/protocol.md
View File

@@ -539,6 +539,22 @@ Emoji created, the event object has the same schema as the Emoji object in the A
}
```
### EmojiUpdate
Emoji has been updated.
```json
{
"type": "EmojiUpdate",
"id": "{emoji_id}",
"data": {
"name"?: "{emoji_name}"
}
}
```
- `data` field contains a partial Emoji object.
### EmojiDelete
Emoji has been deleted.

2
version.txt
View File

@@ -1 +1 @@
0.12.1
0.13.7