Compare commits

...

94 Commits

Author SHA1 Message Date
jmug
35cfb9a489 chore(images): Build all images.
Some checks failed
Docker / base (push) Successful in 32m54s
Rust build, test, and generate specification / Rust project (push) Failing after 28m20s
Docker / Build autumn image (push) Has been skipped
Docker / Build bonfire image (push) Has been skipped
Docker / Build crond image (push) Has been skipped
Docker / Build january image (push) Has been skipped
Docker / Build pushd image (push) Has been skipped
Docker / Build delta image (push) Has been skipped
Docker / base (release) Successful in 21m18s
Docker / Build bonfire image (release) Successful in 2m5s
Docker / Build autumn image (release) Successful in 2m10s
Docker / Build crond image (release) Successful in 1m59s
Docker / Build january image (release) Successful in 2m10s
Docker / Build delta image (release) Successful in 1m51s
Docker / Build pushd image (release) Successful in 2m12s
2026-06-27 11:31:14 -07:00
jmug
7297a64856 chore(images): s/handmade-revolt/handmade-revolt-backend/
Some checks failed
Docker / Build pushd image (push) Has been cancelled
Docker / base (push) Has been cancelled
2026-06-27 11:28:34 -07:00
jmug
720cd46879 chore(pushd/ci): Build image pushd image in-fork.
Some checks failed
Docker / Build pushd image (push) Has been cancelled
Docker / base (push) Has been cancelled
Rust build, test, and generate specification / Rust project (push) Has been cancelled
Docker / base (release) Successful in 21m35s
Docker / Build pushd image (release) Successful in 1m32s
2026-06-27 00:34:53 -07:00
jmug
7d099d5ae7 feat(pushd): Guard against malformed vapid web pushes. 2026-06-27 00:12:11 -07:00
jmug
8847b3c5ba feat(pushd): Fix typo in inbound notification handling. 2026-06-27 00:06:05 -07:00
Paul Makles
eb5f5f91cd feat: add crond container definitions 2025-02-10 20:28:48 +00:00
Paul Makles
52e1f0ddde merge: branch 'insert/feat/crond' 2025-02-10 19:54:40 +00:00
Paul Makles
defc9ec79b fix: match new error type for status code 2025-02-10 19:05:21 +00:00
Paul Makles
7026961df4 chore: fix version references 2025-02-10 19:03:48 +00:00
Paul Makles
078380d305 merge: branch 'main' into insert/feat/crond 2025-02-10 19:02:19 +00:00
Paul Makles
add3f40b23 fix: don't allow users to time themselves out
closes #376
2025-02-10 18:54:15 +00:00
Paul Makles
7216e9909b chore: disable LTO because it likely overloads GitHub worker 2025-02-10 18:50:19 +00:00
Paul Makles
a1e6a19210 chore: bump version to 0.8.2 2025-02-10 18:37:21 +00:00
Paul Makles
5f39403ce7 fix: ensure ratelimiter adjusts for version prefix 2025-02-10 18:34:37 +00:00
Paul Makles
e525ffe5e4 chore: mount API at /0.8/ as well 2025-02-10 18:19:34 +00:00
Paul Makles
e3723d647e fix: change permission check for fetching channel webhooks 2025-02-10 18:04:46 +00:00
Paul Makles
5f84daa9db fix: ensure limit is always at least 1 when sent to database 2025-02-10 17:51:22 +00:00
Paul Makles
e957af4ca3 docs: add some notes on overrides; fix defaults 2025-02-10 17:21:23 +00:00
Paul Makles
439bacf067 refactor: add error messages for Rabbit issues 2025-02-10 17:20:29 +00:00
Paul Makles
fa55e88dd9 chore: add pushd to debug image script 2025-02-10 17:19:54 +00:00
IAmTomahawkx
71e7fe3086 chore: bump release for patch 2024-12-27 16:35:02 -08:00
IAmTomahawkx
03f2e3b1bf fix: last message in a channel could not be acked, resulting in ghost dms and channel messages 2024-12-27 16:29:56 -08:00
Paul Makles
8bbb579d23 ci: revert back to GH runner 2024-12-26 14:59:03 +00:00
Paul Makles
6448af071b ci: remove purge command 2024-12-22 16:10:09 +00:00
Paul Makles
c2a0ab71df ci: bring our own runner 2024-12-22 16:03:49 +00:00
Paul Makles
7a17165c24 ci: try pin to 20.04 2024-12-22 15:24:36 +00:00
Paul Makles
d7213fa409 fix(ci): try to work-around runner being killed 2024-12-22 14:13:30 +00:00
Paul Makles
479f0402ca ci: switch to ubuntu 24.04 pre-emptively
trying to fix build cancellations
2024-12-22 13:30:27 +00:00
Paul Makles
b62eeef80c chore(autumn): conditional animation stripping
feat(autumn): add /original redirect for files
fix(autumn): block non-images for non-attachment tags
fix(ci): use correct port for Rust test
2024-12-22 12:35:06 +00:00
Paul Makles
7b15006a50 fix: don't create "previews" of GIFs
chore: change hardcoded cache limit (should be configed!)
2024-12-20 16:42:32 +00:00
Paul Makles
ac731e547d chore: log database errors to Sentry 2024-12-17 21:09:44 +00:00
TheBobBobs
443f374f23 fix(core): fix _id typo (#384) 2024-12-17 17:52:59 +00:00
Kirill Mironov
42367f477c perf(lto): enable link-time optimization (#371)
Co-authored-by: Paul Makles <me@insrt.uk>
2024-12-17 17:23:15 +00:00
IAmTomahawkx
3cb91151ca fix exposed port in delta dockerfile 2024-12-15 19:39:57 -08:00
IAmTomahawkx
4c46054bff add uname to other docker images 2024-12-13 15:40:04 -08:00
IAmTomahawkx
f01794af93 fix: bonfire needs uname
fixes revoltchat/self-hosted#104
2024-12-12 01:58:55 -08:00
IAmTomahawkx
49f7f9549c feat: env variable to not clear presences on bonfire boot 2024-12-12 01:47:06 -08:00
IAmTomahawkx
72e3129557 fix: make docker the default connection for config 2024-11-28 15:44:00 -08:00
IAmTomahawkx
75a07a84c0 feat: version bump everything 2024-11-28 13:52:28 -08:00
Tom
b55765d7c7 Feat: Push notification server (#387)
* feat: create base of push daemon

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* Add outbound senders

* Make web_push send to rabbit instead (temp stuff)

* feat: stability and friend requests

* make vapid fr stuff not suck

* swap naming of queue

* move pushd into daemons folder

* fix cargo file for move into daemons folder

* feat: probably working fcm push notifs

* comment out fcm webpush stuff since the config keys dont exist

* fix fcm, name queues according to their prod status and configure routing keys

* add pushd to docker

* mix: Remove old code, add stuff to pushd

* fix: lockfile

* feat: update rocket to 5.0.1

* fix: fix queues and ack bugs

* Move rabbit messsage processing into ack queue

* chore: update readme

* chore: optimizations for ack database hits

* pushd flowchart

* misc: update flowchart

* exit dependancy hell

* add rocket_impl flag to authifier

* make the tests file of delta actually compile

* fix: don't silence every push message

* fix: don't silence all messages

* add debug logging for sending data to rabbit from message events

* validate mentions at a server membership level

* put back that import that was actually important

* minor fix to lockfile

* update delta authifier

* feat: proper permissions for push notifications

* add unit test for mention sanitization

* remove local file dependancy on authifier

* update ports to proper defaults

* fixTM the node bindings

* Theoretically configure docker releases for pushd

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* declare exchange in pushd and delta

* fix createbuckets script

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: reference db implementation

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: remove finally redundant code

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: changes

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: other changes

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* fix: make channel name return result

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

---------

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>
2024-11-28 13:34:17 -08:00
Paul Makles
ed5ded5e45 feat(autumn): use real memory size for s3 cache eviction
feat(january): use real memory size for proxy cache eviction
2024-11-28 11:58:36 +00:00
Paul Makles
249749e14d feat: file deletion implementation 2024-11-27 21:54:39 +00:00
Paul Makles
4c00a7dfb7 chore: strip dotenv (unmaintained) from project
It is no longer needed as configuration is loaded via. TOML or direct env if appropriate.
2024-11-27 17:05:11 +00:00
Paul Makles
acc4317246 feat: init crond crate 2024-11-27 16:56:30 +00:00
Paul Makles
b9ae333b02 fix(bonfire): make error handling consistent 2024-11-11 12:31:34 +00:00
Paul Makles
bf0fc504a9 chore(vscode): select nix env 2024-11-11 12:31:19 +00:00
Paul Makles
705e517871 fix: db migration for webhooks does not consider missing channels
fixes #378
2024-10-31 20:31:09 +00:00
Paul Makles
6f99ac2160 chore: bump little_exif debug code 2024-10-28 15:38:07 +00:00
Paul Makles
2fcc714546 chore: add helper script for running everything at once 2024-10-28 14:15:48 +00:00
Paul Makles
1e72f7bc77 chore: copy createbuckets config from self-hosted 2024-10-28 13:45:37 +00:00
Paul Makles
af0d24c7c6 fix: don't use failed file hashes
fixes #377
2024-10-28 13:40:40 +00:00
Paul Makles
ab58177dfa feat: add support for path style buckets 2024-10-27 23:28:28 +00:00
Paul Makles
df07426019 chore: bump version to 0.7.19 2024-10-24 17:48:11 +01:00
Paul Makles
c30d9a2620 chore(bonfire): fire error event with more information 2024-10-24 17:47:17 +01:00
Paul Makles
7b44317705 docs: add new features page 2024-10-24 17:45:10 +01:00
Paul Makles
876068a37e fix: populate empty vector if clear field is missing
fixes #367
2024-10-24 17:14:30 +01:00
IAmTomahawkx
397b9878e1 fix: temp fix for spam attack via notification bug abuse, but git actually commits the changes this time 2024-10-06 13:22:13 -07:00
Paul Makles
d9deadc65a fix: temp fix for spam attack via notification bug abuse 2024-10-06 02:34:47 -07:00
Paul Makles
58d3c5cc2e fix(services/january): remove image if video present and hence fix logic error
refactor(services/january): throw an error if embed fails to generate
2024-10-02 16:08:24 +01:00
Paul Makles
520fb02fb6 fix(services/january): support svg for embed generation 2024-10-02 15:20:35 +01:00
Paul Makles
2cb12a3d59 fix: do not handle error early 2024-10-02 15:05:57 +01:00
Paul Makles
18888eae5f fix: mangled symbol 2024-10-02 14:13:32 +01:00
Paul Makles
f31020fb6e refactor(core): add ImageProcessingFailed error 2024-10-02 14:12:26 +01:00
Paul Makles
bb202079e0 fix(services/january): put html parsing into block to prevent issues with futures 2024-10-02 14:09:41 +01:00
Paul Makles
bb6bcda8bd fix(services/january): reddit embeds by spoofing as discord / mapping to old reddit
closes #360
2024-10-02 12:56:32 +01:00
Paul Makles
68099bd2b7 chore: bump version & close issues
closes #366
closes #364
closes #359
closes #356
closes #354
closes #351
closes #348
closes #345
2024-10-02 12:39:32 +01:00
Paul Makles
a8db1cb40d feat(core/files): SVG rendering for thumbnails 2024-10-02 12:36:56 +01:00
Paul Makles
efa7ba78ed feat(core/files): support for jixel decoding
closes #342
2024-10-02 12:20:14 +01:00
Paul Makles
25fc692dc1 chore(core/models): truncate the rest of the embed fields 2024-10-02 11:35:37 +01:00
Paul Makles
530d68fe89 feat(services/autumn): add accept-language header
closes #358
2024-10-02 11:32:55 +01:00
Paul Makles
afd8c906ba fix(services/january): YouTube fallback 2024-10-02 11:31:55 +01:00
Paul Makles
c596dd5458 chore: bump all versions 2024-10-01 21:18:55 +01:00
Paul Makles
ed78b253ff feat(services/january): website embed generation 2024-10-01 21:11:08 +01:00
Paul Makles
66c84e0ad9 feat(services/january): image/video embeds 2024-10-01 19:13:48 +01:00
Paul Makles
21335b3297 feat(services/january): proxy images/videos 2024-10-01 19:02:39 +01:00
Paul Makles
8fc791f81a ci: remove .env file entry 2024-09-29 17:18:41 +01:00
Paul Makles
1689ee5ddc ci: use compose.yml file instead of deleted file 2024-09-29 17:08:42 +01:00
Paul Makles
7a061bb3c6 ci: add january to build list 2024-09-29 17:05:20 +01:00
Paul Makles
6c3b8eaa92 chore(core/files): do not decrypt file if nonce unavailable 2024-09-29 17:03:02 +01:00
Paul Makles
5e1b2e165f ci: add autumn to build list 2024-09-29 15:10:23 +01:00
Paul Makles
e270b5df6a chore: correct ports in Dockerfile 2024-09-29 15:10:12 +01:00
Paul Makles
cbf9e81256 fix: delete attachments by used_for.id and add corresponding index 2024-09-29 15:05:56 +01:00
Paul Makles
3080ec1f5a refactor: capture errors with line numbers
refactor: update file api
2024-09-29 14:37:08 +01:00
Paul Makles
4fd66b2719 refactor(core/result): add NotAuthenticated error which is distinct from InvalidCredentials 2024-09-29 12:02:10 +01:00
Paul Makles
916f47e2f5 feat(services/autumn): cors support 2024-09-29 12:01:32 +01:00
Paul Makles
a3c5b1bf87 chore: releasing beta build of autumn 2024-09-11 14:42:00 +01:00
Paul Makles
ace6c30ba5 feat(services/autumn): file uploads
feat(services/autumn): deduplicate uploads
feat(services/autumn): ClamAV support
2024-09-11 14:29:52 +01:00
Paul Makles
f4104612b2 chore: strip all the duplicate stuff from dev Revolt.toml 2024-09-10 16:54:07 +01:00
Paul Makles
6209bc7152 feat(services/autumn): authenticate the user on upload
feat(services/autumn): file size limits on upload
chore: add database migrations for new autumn things
2024-09-10 16:53:58 +01:00
Paul Makles
1a6a8a809b feat(core/database): axum implementation for User 2024-09-10 16:53:02 +01:00
Paul Makles
5485781e7c docs: update README with more badges 2024-09-10 15:15:53 +01:00
Paul Makles
8b21825ffe docs: update README 2024-09-09 14:24:22 +01:00
Paul Makles
530ff4f9c7 docs: this will be the developer experience ever (deprecate .env file )
fix: port is wrong for maildev
2024-09-09 14:19:31 +01:00
Paul Makles
2eb596fe99 chore: pin crate versions to "MaxSRV" 2024-09-09 13:05:43 +01:00
145 changed files with 6836 additions and 2086 deletions

View File

@@ -1,107 +0,0 @@
# MongoDB URI
MONGODB=mongodb://localhost
AUTUMN_MONGO_URI=mongodb://database
REDIS_URI=redis://localhost/
# URL to where the Revolt app is publicly accessible
REVOLT_APP_URL=http://local.revolt.chat:5000
# URL to where the API is publicly accessible
REVOLT_PUBLIC_URL=http://local.revolt.chat:8000
VITE_API_URL=http://local.revolt.chat:8000
# URL to where the WebSocket server is publicly accessible
REVOLT_EXTERNAL_WS_URL=ws://local.revolt.chat:9000
# URL to where Autumn is publicly available
AUTUMN_PUBLIC_URL=http://local.revolt.chat:3000
# URL to where January is publicly available
JANUARY_PUBLIC_URL=http://local.revolt.chat:7000
# URL to where Vortex is publicly available
# VOSO_PUBLIC_URL=https://voso.revolt.chat
##
## hCaptcha Settings
##
# If you are sure that you don't want to use hCaptcha, set to 1.
REVOLT_UNSAFE_NO_CAPTCHA=1
# hCaptcha API key
# REVOLT_HCAPTCHA_KEY=0x0000000000000000000000000000000000000000
# hCaptcha site key
# REVOLT_HCAPTCHA_SITEKEY=10000000-ffff-ffff-ffff-000000000001
##
## Email Settings
##
# If you are sure that you don't want to use email verification, set to 1.
REVOLT_UNSAFE_NO_EMAIL=1
# SMTP host
# REVOLT_SMTP_HOST=smtp.example.com
# SMTP username
# REVOLT_SMTP_USERNAME=noreply@example.com
# SMTP password
# REVOLT_SMTP_PASSWORD=CHANGEME
# SMTP From header
# REVOLT_SMTP_FROM=Revolt <noreply@example.com>
##
## Application Settings
##
# Whether to enable staging only features
REVOLT_IS_STAGING=1
# Whether to only allow users to sign up if they have an invite code
REVOLT_INVITE_ONLY=0
# Maximum number of people that can be in a group chat
REVOLT_MAX_GROUP_SIZE=150
# VAPID keys for push notifications
# Generate using this guide: https://gitlab.insrt.uk/revolt/delta/-/wikis/vapid
# --> Please replace these keys before going into production! <--
REVOLT_VAPID_PRIVATE_KEY=LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJSUWpyTWxLRnBiVWhsUHpUbERvcEliYk1yeVNrNXpKYzVYVzIxSjJDS3hvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFWnkrQkg2TGJQZ2hEa3pEempXOG0rUXVPM3pCajRXT1phdkR6ZU00c0pqbmFwd1psTFE0WAp1ZDh2TzVodU94QWhMQlU3WWRldVovWHlBdFpWZmNyQi9BPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=
REVOLT_VAPID_PUBLIC_KEY=BGcvgR-i2z4IQ5Mw841vJvkLjt8wY-FjmWrw83jOLCY52qcGZS0OF7nfLzuYbjsQISwVO2HXrmf18gLWVX3Kwfw=
##
## Vortex configuration
##
# VOSO_MANAGE_TOKEN=CHANGEME
##
## Autumn configuration
##
# S3 Region
AUTUMN_S3_REGION=minio
# S3 Endpoint
AUTUMN_S3_ENDPOINT=http://minio:9000
# MinIO Root User
MINIO_ROOT_USER=minioautumn
# MinIO Root Password
MINIO_ROOT_PASSWORD=minioautumn
# AWS Access Key ID
AWS_ACCESS_KEY_ID=minioautumn
# AWS Secret Key
AWS_SECRET_ACCESS_KEY=minioautumn

View File

@@ -1,125 +1,113 @@
name: Docker Test & Publish
name: Docker
on:
push:
# branches:
# - "main"
tags:
- "*"
paths-ignore:
- ".github/**"
- "!.github/workflows/docker.yml"
- ".vscode/**"
- "doc/**"
- ".gitignore"
- "LICENSE"
- "README"
branches:
- "handmade"
pull_request:
branches:
- "main"
paths:
- "Dockerfile"
workflow_dispatch:
- "handmade"
release:
types:
- published
permissions:
contents: read
packages: write
contents: read
jobs:
base:
runs-on: ubuntu-latest
name: Build base image
steps:
# Configure build environment
- name: Checkout
uses: actions/checkout@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
# Builds the shared base image: the whole Rust workspace compiled once.
# Every service image (currently only pushd) is derived from this so the
# workspace is built a single time per release. On non-release events this
# still runs as a compile check, but nothing is pushed.
base:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Resolve registry host and owner
id: registry
# Owner is lowercased: Docker repository names must be lowercase.
run: |
echo "host=${GITHUB_SERVER_URL#*://}" >> "$GITHUB_OUTPUT"
echo "owner=$(echo "$GITHUB_REPOSITORY_OWNER" | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT"
- name: Base image ref
id: base
run: echo "image=${{ steps.registry.outputs.host }}/${{ steps.registry.outputs.owner }}/handmade-revolt-backend-base" >> "$GITHUB_OUTPUT"
- name: Login to registry
uses: docker/login-action@v3
if: github.event_name == 'release'
with:
registry: ${{ steps.registry.outputs.host }}
username: ${{ secrets.PACKAGE_PUBLISH_USERNAME }}
password: ${{ secrets.PACKAGE_PUBLISH_TOKEN }}
- name: Build base image
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile
push: ${{ github.event_name == 'release' }}
platforms: linux/amd64
# Pinned to the commit so the publish job below consumes this
# exact base.
tags: |
${{ steps.base.outputs.image }}:latest
${{ steps.base.outputs.image }}:${{ github.sha }}
# Authenticate with GHCR
- name: Login to Github Container Registry
if: ${{ github.event_name != 'pull_request' }}
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build base image
uses: docker/build-push-action@v4
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
platforms: linux/amd64,linux/arm64
tags: ghcr.io/${{ github.repository_owner }}/base:latest
cache-from: type=gha,scope=buildx-base-multi-arch
cache-to: type=gha,scope=buildx-base-multi-arch,mode=max
publish:
needs: [base]
runs-on: ubuntu-latest
if: github.event_name != 'pull_request'
strategy:
matrix:
project: [delta, bonfire]
name: Build ${{ matrix.project }} image
steps:
# Configure build environment
- name: Checkout
uses: actions/checkout@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
# Authenticate with Docker Hub and GHCR
- name: Login to DockerHub
uses: docker/login-action@v2
with:
registry: docker.io
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Github Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Resolve the correct project
- uses: kanga333/variable-mapper@master
id: export
with:
key: "${{ matrix.project }}"
map: |
{
"delta": {
"path": "crates/delta",
"tag": "${{ github.repository_owner }}/server"
},
"bonfire": {
"path": "crates/bonfire",
"tag": "${{ github.repository_owner }}/bonfire"
}
}
export_to: output
# Configure metadata
- name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
images: |
docker.io/${{ steps.export.outputs.tag }}
ghcr.io/${{ steps.export.outputs.tag }}
# Build crate image
- name: Publish
uses: docker/build-push-action@v4
with:
context: .
push: true
platforms: linux/amd64,linux/arm64
file: ${{ steps.export.outputs.path }}/Dockerfile
tags: ${{ steps.meta.outputs.tags }}
build-args: |
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest
labels: ${{ steps.meta.outputs.labels }}
# Builds individual service images from the base. Only runs on release.
publish:
needs: [base]
runs-on: ubuntu-latest
if: github.event_name == 'release'
strategy:
matrix:
include:
- { project: pushd, path: crates/daemons/pushd, image: handmade-revolt-backend-pushd }
- { project: delta, path: crates/delta, image: handmade-revolt-backend-server }
- { project: bonfire, path: crates/bonfire, image: handmade-revolt-backend-bonfire }
- { project: autumn, path: crates/services/autumn, image: handmade-revolt-backend-autumn }
- { project: january, path: crates/services/january, image: handmade-revolt-backend-january }
- { project: crond, path: crates/daemons/crond, image: handmade-revolt-backend-crond }
name: Build ${{ matrix.project }} image
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Resolve registry host and owner
id: registry
# Owner is lowercased: Docker repository names must be lowercase,
# but the org (e.g. HMC) may be uppercase.
run: |
echo "host=${GITHUB_SERVER_URL#*://}" >> "$GITHUB_OUTPUT"
echo "owner=$(echo "$GITHUB_REPOSITORY_OWNER" | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT"
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ steps.registry.outputs.host }}/${{ steps.registry.outputs.owner }}/${{ matrix.image }}
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=raw,value=latest
env:
DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
- name: Login to registry
uses: docker/login-action@v3
with:
registry: ${{ steps.registry.outputs.host }}
username: ${{ secrets.PACKAGE_PUBLISH_USERNAME }}
password: ${{ secrets.PACKAGE_PUBLISH_TOKEN }}
- name: Build and publish
uses: docker/build-push-action@v6
with:
context: .
file: ${{ matrix.path }}/Dockerfile
push: true
platforms: linux/amd64
build-args: |
BASE_IMAGE=${{ steps.registry.outputs.host }}/${{ steps.registry.outputs.owner }}/handmade-revolt-backend-base:${{ github.sha }}
tags: ${{ steps.meta.outputs.tags }}
annotations: ${{ steps.meta.outputs.annotations }}
labels: ${{ steps.meta.outputs.labels }}

View File

@@ -41,11 +41,7 @@ jobs:
- name: Run services in background
run: |
docker compose -f docker-compose.db.yml up -d
- name: Copy .env.example
run: |
cp .env.example .env
docker compose -f compose.yml up -d
- name: Run cargo test
env:
@@ -71,7 +67,7 @@ jobs:
if: github.event_name != 'pull_request' && github.ref_name == 'main'
uses: nev7n/wait_for_response@v1
with:
url: "http://localhost:8000/"
url: "http://localhost:14702/"
- name: Checkout API repository
if: github.event_name != 'pull_request' && github.ref_name == 'main'
@@ -83,7 +79,7 @@ jobs:
- name: Download OpenAPI specification
if: github.event_name != 'pull_request' && github.ref_name == 'main'
run: curl http://localhost:8000/openapi.json -o api/OpenAPI.json
run: curl http://localhost:14702/openapi.json -o api/OpenAPI.json
- name: Commit changes
if: github.event_name != 'pull_request' && github.ref_name == 'main'

3
.gitignore vendored
View File

@@ -1,5 +1,6 @@
Rocket.toml
Revolt.*.toml
compose.override.yml
target
.data
@@ -7,3 +8,5 @@ target
.vercel
.DS_Store
.idea

View File

@@ -1,5 +1,6 @@
{
"editor.formatOnSave": true,
"rust-analyzer.checkOnSave.command": "clippy",
"nixEnvSelector.suggestion": false
}
"nixEnvSelector.suggestion": false,
"nixEnvSelector.nixFile": "${workspaceFolder}/default.nix"
}

1226
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,16 +1,24 @@
[workspace]
resolver = "2"
members = [
"crates/delta",
"crates/bonfire",
"crates/core/*",
"crates/services/*",
"crates/bindings/*",
"crates/daemons/*",
]
[patch.crates-io]
# mobc-redis = { git = "https://github.com/insertish/mobc", rev = "8b880bb59f2ba80b4c7bc40c649c113d8857a186" }
redis22 = { package = "redis", version = "0.22.3", git = "https://github.com/revoltchat/redis-rs", rev = "1a41faf356fd21aebba71cea7eb7eb2653e5f0ef" }
redis23 = { package = "redis", version = "0.23.1", git = "https://github.com/revoltchat/redis-rs", rev = "f8ca28ab85da59d2ccde526b4d2fb390eff5a5f9" }
# authifier = { package = "authifier", version = "1.0.8", path = "../authifier/crates/authifier" }
# rocket_authifier = { package = "rocket_authifier", version = "1.0.8", path = "../authifier/crates/rocket_authifier" }
# I'm 99% sure this is overloading the GitHub worker
# hence builds have been failing since, let's just
# disable it for now. In the future, we could use this
# if we were rolling our own CI.
# [profile.release]
# lto = true

View File

@@ -29,6 +29,8 @@ COPY crates/core/presence/Cargo.toml ./crates/core/presence/
COPY crates/core/result/Cargo.toml ./crates/core/result/
COPY crates/services/autumn/Cargo.toml ./crates/services/autumn/
COPY crates/services/january/Cargo.toml ./crates/services/january/
COPY crates/daemons/crond/Cargo.toml ./crates/daemons/crond/
COPY crates/daemons/pushd/Cargo.toml ./crates/daemons/pushd/
RUN sh /tmp/build-image-layer.sh deps
# Build all apps

View File

@@ -25,6 +25,8 @@ COPY crates/core/presence/Cargo.toml ./crates/core/presence/
COPY crates/core/result/Cargo.toml ./crates/core/result/
COPY crates/services/autumn/Cargo.toml ./crates/services/autumn/
COPY crates/services/january/Cargo.toml ./crates/services/january/
COPY crates/daemons/crond/Cargo.toml ./crates/daemons/crond/
COPY crates/daemons/pushd/Cargo.toml ./crates/daemons/pushd/
RUN sh /tmp/build-image-layer.sh deps
# Build all apps

111
README.md
View File

@@ -1,19 +1,36 @@
# Revolt Backend
<div align="center">
<h1>
Revolt Backend
[![Stars](https://img.shields.io/github/stars/revoltchat/backend?style=flat-square&logoColor=white)](https://github.com/revoltchat/backend/stargazers)
[![Forks](https://img.shields.io/github/forks/revoltchat/backend?style=flat-square&logoColor=white)](https://github.com/revoltchat/backend/network/members)
[![Pull Requests](https://img.shields.io/github/issues-pr/revoltchat/backend?style=flat-square&logoColor=white)](https://github.com/revoltchat/backend/pulls)
[![Issues](https://img.shields.io/github/issues/revoltchat/backend?style=flat-square&logoColor=white)](https://github.com/revoltchat/backend/issues)
[![Contributors](https://img.shields.io/github/contributors/revoltchat/backend?style=flat-square&logoColor=white)](https://github.com/revoltchat/backend/graphs/contributors)
[![License](https://img.shields.io/github/license/revoltchat/backend?style=flat-square&logoColor=white)](https://github.com/revoltchat/backend/blob/main/LICENSE)
</h1>
The services and libraries that power the Revolt service.<br/>
<br/>
This is a monorepo for the Revolt backend.
| Crate | Path | Description | |
| ------------------ | -------------------------------------------------- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `core/config` | [crates/core/config](crates/core/config) | Core: Configuration | ![Crates.io Version](https://img.shields.io/crates/v/revolt-config) ![Crates.io Version](https://img.shields.io/crates/msrv/revolt-config) ![Crates.io Version](https://img.shields.io/crates/size/revolt-config) ![Crates.io License](https://img.shields.io/crates/l/revolt-config) |
| `core/database` | [crates/core/database](crates/core/database) | Core: Database Implementation | ![Crates.io Version](https://img.shields.io/crates/v/revolt-database) ![Crates.io Version](https://img.shields.io/crates/msrv/revolt-database) ![Crates.io Version](https://img.shields.io/crates/size/revolt-database) ![Crates.io License](https://img.shields.io/crates/l/revolt-database) |
| `core/files` | [crates/core/files](crates/core/files) | Core: S3 and encryption subroutines | ![Crates.io Version](https://img.shields.io/crates/v/revolt-files) ![Crates.io Version](https://img.shields.io/crates/msrv/revolt-files) ![Crates.io Version](https://img.shields.io/crates/size/revolt-files) ![Crates.io License](https://img.shields.io/crates/l/revolt-files) |
| `core/models` | [crates/core/models](crates/core/models) | Core: API Models | ![Crates.io Version](https://img.shields.io/crates/v/revolt-models) ![Crates.io Version](https://img.shields.io/crates/msrv/revolt-models) ![Crates.io Version](https://img.shields.io/crates/size/revolt-models) ![Crates.io License](https://img.shields.io/crates/l/revolt-models) |
| `core/permissions` | [crates/core/permissions](crates/core/permissions) | Core: Permission Logic | ![Crates.io Version](https://img.shields.io/crates/v/revolt-permissions) ![Crates.io Version](https://img.shields.io/crates/msrv/revolt-permissions) ![Crates.io Version](https://img.shields.io/crates/size/revolt-permissions) ![Crates.io License](https://img.shields.io/crates/l/revolt-permissions) |
| `core/presence` | [crates/core/presence](crates/core/presence) | Core: User Presence | ![Crates.io Version](https://img.shields.io/crates/v/revolt-presence) ![Crates.io Version](https://img.shields.io/crates/msrv/revolt-presence) ![Crates.io Version](https://img.shields.io/crates/size/revolt-presence) ![Crates.io License](https://img.shields.io/crates/l/revolt-presence) |
| `core/result` | [crates/core/result](crates/core/result) | Core: Result and Error types | ![Crates.io Version](https://img.shields.io/crates/v/revolt-result) ![Crates.io Version](https://img.shields.io/crates/msrv/revolt-result) ![Crates.io Version](https://img.shields.io/crates/size/revolt-result) ![Crates.io License](https://img.shields.io/crates/l/revolt-result) |
| `delta` | [crates/delta](crates/delta) | REST API server | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue) |
| `bonfire` | [crates/bonfire](crates/bonfire) | WebSocket events server | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue) |
| `services/january` | [crates/services/january](crates/services/january) | Proxy server | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue) |
| `services/autumn` | [crates/services/autumn](crates/services/autumn) | File server | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue) |
| `bindings/node` | [crates/bindings/node](crates/bindings/node) | Node.js bindings for the Revolt software | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue) |
| `daemons/crond` | [crates/daemons/crond](crates/daemons/crond) | Timed data clean up daemon server | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue) |
| `daemons/pushd` | [crates/daemons/pushd](crates/daemons/pushd) | Push notification daemon server | ![License](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue) |
| Crate | Path | Description |
| ------------------ | -------------------------------------------------- | ----------------------------- |
| `core/config` | [crates/core/config](crates/core/config) | Core: Configuration |
| `core/database` | [crates/core/database](crates/core/database) | Core: Database Implementation |
| `core/models` | [crates/core/models](crates/core/models) | Core: API Models |
| `core/permissions` | [crates/core/permissions](crates/core/permissions) | Core: Permission Logic |
| `core/presence` | [crates/core/presence](crates/core/presence) | Core: User Presence |
| `core/result` | [crates/core/result](crates/core/result) | Core: Result and Error types |
| `delta` | [crates/delta](crates/delta) | REST API server |
| `bonfire` | [crates/bonfire](crates/bonfire) | WebSocket events server |
Note: `january`, `autumn`, and `vortex` are yet to be moved into this monorepo.
</div>
<br/>
## Minimum Supported Rust Version
@@ -40,11 +57,12 @@ As a heads-up, the development environment uses the following ports:
| Service | Port |
| ------------------------- | :------------: |
| MongoDB | 14017 |
| Redis | 14079 |
| MongoDB | 27017 |
| Redis | 6379 |
| MinIO | 14009 |
| Maildev | 14025<br>14080 |
| Revolt Web App | 14701 |
| RabbitMQ | 5672<br>15672 |
| `crates/delta` | 14702 |
| `crates/bonfire` | 14703 |
| `crates/services/autumn` | 14704 |
@@ -60,13 +78,51 @@ cargo build
A default configuration `Revolt.toml` is present in this project that is suited for development.
If you'd like to change anything, create a `Revolt.overrides.toml` file to overwrite it.
If you'd like to change anything, create a `Revolt.overrides.toml` file and specify relevant variables.
You may need to configure the legacy environment options:
> [!TIP]
> Use Sentry to catch unexpected service errors:
>
> ```toml
> # Revolt.overrides.toml
> [sentry]
> api = "https://abc@your.sentry/1"
> events = "https://abc@your.sentry/1"
> files = "https://abc@your.sentry/1"
> proxy = "https://abc@your.sentry/1"
> ```
```bash
cp .env.example .env
```
> [!TIP]
> If you have port conflicts on common services, you can try the following:
>
> ```yaml
> # compose.override.yml
> services:
> redis:
> ports: !override
> - "14079:6379"
>
> database:
> ports: !override
> - "14017:27017"
>
> rabbit:
> ports: !override
> - "14072:5672"
> - "14672:15672"
> ```
>
> And corresponding Revolt configuration:
>
> ```toml
> # Revolt.overrides.toml
> [database]
> mongodb = "mongodb://127.0.0.1:14017"
> redis = "redis://127.0.0.1:14079/"
>
> [rabbit]
> port = 14072
> ```
Then continue:
@@ -74,17 +130,23 @@ Then continue:
# start other necessary services
docker compose up -d
# run everything together
./scripts/start.sh
# .. or individually
# run the API server
cargo run --bin revolt-delta
# run the events server
cargo run --bin revolt-bonfire
# run the file server
cargo run --bin revolt-autumn
# run th proxy server
# run the proxy server
cargo run --bin revolt-january
# run the push daemon (not usually needed in regular development)
cargo run --bin revolt-pushd
# hint:
# mold -run <cargo build, cargo run, etc...>
# mold -run ./scripts/start.sh
```
You can start a web client by doing the following:
@@ -98,10 +160,13 @@ git clone --recursive https://github.com/revoltchat/revite
cd revite
yarn
yarn build:deps
echo "VITE_API_URL=http://local.revolt.chat:14702" > .env.local
yarn dev --port 14701
```
Then go to https://local.revolt.chat:14701
Then go to http://local.revolt.chat:14701 to create an account/login.
When signing up, go to http://localhost:14080 to find confirmation/password reset emails.
## Deployment Guide

View File

@@ -4,10 +4,13 @@
[database]
# MongoDB connection URL
# Defaults to the container name specified in self-hosted
mongodb = "mongodb://localhost:14017"
mongodb = "mongodb://127.0.0.1:27017"
# Redis connection URL
# Defaults to the container name specified in self-hosted
redis = "redis://localhost:14079/"
redis = "redis://127.0.0.1:6379/"
[rabbit]
host = "127.0.0.1"
[hosts]
# Web locations of various services
@@ -25,11 +28,6 @@ voso_legacy_ws = ""
[api]
[api.registration]
# Whether an invite should be required for registration
# See https://github.com/revoltchat/self-hosted#making-your-instance-invite-only
invite_only = false
[api.smtp]
# Email server configuration for verification
# Defaults to no email verification (host field is empty)
@@ -41,97 +39,9 @@ reply_to = "support@revolt.chat"
port = 14025
use_tls = false
[api.vapid]
# Generate your own keys:
# 1. Run `openssl ecparam -name prime256v1 -genkey -noout -out vapid_private.pem`
# 2. Find `private_key` using `base64 vapid_private.pem`
# 3. Find `public_key` using `openssl ec -in vapid_private.pem -outform DER|tail -c 65|base64|tr '/+' '_-'|tr -d '\n'`
private_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJSUWpyTWxLRnBiVWhsUHpUbERvcEliYk1yeVNrNXpKYzVYVzIxSjJDS3hvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFWnkrQkg2TGJQZ2hEa3pEempXOG0rUXVPM3pCajRXT1phdkR6ZU00c0pqbmFwd1psTFE0WAp1ZDh2TzVodU94QWhMQlU3WWRldVovWHlBdFpWZmNyQi9BPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo"
public_key = "BGcvgR-i2z4IQ5Mw841vJvkLjt8wY-FjmWrw83jOLCY52qcGZS0OF7nfLzuYbjsQISwVO2HXrmf18gLWVX3Kwfw="
[api.fcm]
# Google Firebase Cloud Messaging Service Account Key
# Obtained from the cloud messaging console
key_type = ""
project_id = ""
private_key_id = ""
private_key = ""
client_email = ""
client_id = ""
auth_uri = ""
token_uri = ""
auth_provider_x509_cert_url = ""
client_x509_cert_url = ""
[api.apn]
# Apple Push Notifications keys for sending notifications
sandbox = false
pkcs8 = ""
key_id = ""
team_id = ""
[api.security]
# Authifier Shield API key
authifier_shield_key = ""
# Legacy voice server management token
voso_legacy_token = ""
# Whether services are behind the Cloudflare network
trust_cloudflare = false
[api.security.captcha]
# hCaptcha configuration
hcaptcha_key = ""
hcaptcha_sitekey = ""
[api.workers]
# Maximum concurrent connections (to proxy server)
max_concurrent_connections = 50
[files]
# Encryption key for stored files
# Generate your own key using `openssl rand -base64 32`
encryption_key = "qcuMA+ssxhMyKaNAKBGFfryfFtUH8NDlamQyDwGW6fU="
# Quality used for lossy WebP previews (set to 100 for lossless)
webp_quality = 80.0
# Mime types that cannot be uploaded or served
#
# Example for Windows executables and Android installation files:
# ["application/vnd.microsoft.portable-executable", "application/vnd.android.package-archive"]
blocked_mime_types = []
# ClamAV service
# hostname:port
clamd_host = ""
[files.limit]
# Minimum image resolution
min_resolution = [1, 1]
# Maximum MP of images
max_mega_pixels = 40
# Maximum pixel side of an image
max_pixel_side = 10_000
[files.preview]
# Maximum image resolution
attachments = [1280, 1280]
avatars = [128, 128]
backgrounds = [1280, 720]
icons = [128, 128]
banners = [480, 480]
emojis = [128, 128]
[files.s3]
# Configuration for S3
# Defaults included for MinIO + self-hosted setup
#
# Backblaze B2:
# - endpoint is listed on the "Buckets" page
# - region is `eu-central-003` string from endpoint URL
# - access_key_id is keyID generated on the "Application Keys" page
# - secret_access_key is token generated on the "Application Keys" page
# - default_bucket matches the name of the bucket you've created
# S3 protocol endpoint
endpoint = "http://localhost:14009"
endpoint = "http://127.0.0.1:14009"
# S3 region name
region = "minio"
# S3 protocol key ID
@@ -140,60 +50,3 @@ access_key_id = "minioautumn"
secret_access_key = "minioautumn"
# Bucket to upload to by default
default_bucket = "revolt-uploads"
[features]
# Feature gate options
webhooks_enabled = false
[features.limits]
[features.limits.global]
group_size = 100
message_embeds = 5
message_replies = 5
message_reactions = 20
server_emoji = 100
server_roles = 200
server_channels = 200
# How many days since creation a user is considered new
new_user_days = 3
# Maximum permissible body size in bytes for uploads
# (should be greater than any one file upload limit)
body_limit_size = 20_000_000
[features.limits.new_user]
outgoing_friend_requests = 5
bots = 2
message_length = 2000
message_attachments = 5
servers = 100
attachment_size = 20_000_000
avatar_size = 4_000_000
background_size = 6_000_000
icon_size = 2_500_000
banner_size = 6_000_000
emoji_size = 500_000
[features.limits.default]
outgoing_friend_requests = 10
bots = 5
message_length = 2000
message_attachments = 5
servers = 100
attachment_size = 20_000_000
avatar_size = 4_000_000
background_size = 6_000_000
icon_size = 2_500_000
banner_size = 6_000_000
emoji_size = 500_000
[sentry]
# Configuration for Sentry error reporting
api = ""
events = ""

View File

@@ -3,13 +3,13 @@ services:
redis:
image: eqalpha/keydb
ports:
- "14079:6379"
- "6379:6379"
# MongoDB
database:
image: mongo
ports:
- "14017:27017"
- "27017:27017"
volumes:
- ./.data/db:/data/db
@@ -24,6 +24,7 @@ services:
- ./.data/minio:/data
ports:
- "14009:9000"
- "14010:9001"
restart: always
# Create buckets for minio.
@@ -31,22 +32,32 @@ services:
image: minio/mc
depends_on:
- minio
environment:
MINIO_ROOT_USER: minioautumn
MINIO_ROOT_PASSWORD: minioautumn
entrypoint: >
/bin/sh -c "
/usr/bin/mc config host add minio http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD;
while ! /usr/bin/mc ready minio; do echo 'Waiting minio...' && sleep 1; done;
/usr/bin/mc mb minio/revolt-uploads;
exit 0;
"
/bin/sh -c "while ! /usr/bin/mc ready minio; do
/usr/bin/mc config host add minio http://minio:9000 minioautumn minioautumn;
echo 'Waiting minio...' && sleep 1;
done; /usr/bin/mc mb minio/revolt-uploads; exit 0;"
# Rabbit
rabbit:
image: rabbitmq:3-management
environment:
RABBITMQ_DEFAULT_USER: rabbituser
RABBITMQ_DEFAULT_PASS: rabbitpass
volumes:
- ./.data/rabbit:/var/lib/rabbitmq
#- ./rabbit_plugins:/opt/rabbitmq/plugins/
#- ./rabbit_enabled_plugins:/etc/rabbitmq/enabled_plugins
# uncomment this if you need to enable other plugins
ports:
- "5672:5672"
- "15672:15672" # management UI, for development
# Mock SMTP server
maildev:
image: soulteary/maildev
ports:
- "14025:8080"
- "14025:25"
- "14080:8080"
environment:
MAILDEV_SMTP_PORT: 25

View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-nodejs-bindings"
version = "0.7.16"
version = "0.8.2"
description = "Node.js bindings for the Revolt software"
authors = ["Paul Makles <me@insrt.uk>"]
license = "MIT"
@@ -20,6 +20,6 @@ serde = { version = "1", features = ["derive"] }
async-std = "1.12.0"
revolt-config = { version = "0.7.16", path = "../../core/config" }
revolt-result = { version = "0.7.16", path = "../../core/result" }
revolt-database = { version = "0.7.16", path = "../../core/database" }
revolt-config = { version = "0.8.2", path = "../../core/config" }
revolt-result = { version = "0.8.2", path = "../../core/result" }
revolt-database = { version = "0.8.2", path = "../../core/database" }

View File

@@ -7,25 +7,25 @@ use neon::prelude::*;
use revolt_database::{Database, DatabaseInfo};
fn js_init(mut cx: FunctionContext) -> JsResult<JsUndefined> {
static INIT: OnceLock<()> = OnceLock::new();
if INIT.get().is_none() {
INIT.get_or_init(|| {
async_std::task::block_on(async {
revolt_config::configure!(api);
// static INIT: OnceLock<()> = OnceLock::new();
// if INIT.get().is_none() {
// INIT.get_or_init(|| {
// async_std::task::block_on(async {
// revolt_config::configure!(api);
match DatabaseInfo::Auto.connect().await {
Ok(db) => {
let authifier_db = db.clone().to_authifier().await.database;
revolt_database::tasks::start_workers(db, authifier_db);
Ok(())
}
Err(err) => Err(err),
}
})
.or_else(|err| cx.throw_error(err))
.unwrap();
});
}
// match DatabaseInfo::Auto.connect().await {
// Ok(db) => {
// let authifier_db = db.clone().to_authifier().await.database;
// revolt_database::tasks::start_workers(db, authifier_db);
// Ok(())
// }
// Err(err) => Err(err),
// }
// })
// .or_else(|err| cx.throw_error(err))
// .unwrap();
// });
// }
Ok(cx.undefined())
}

View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-bonfire"
version = "0.7.16"
version = "0.8.2"
license = "AGPL-3.0-or-later"
edition = "2021"
@@ -36,12 +36,12 @@ async-std = { version = "1.8.0", features = [
] }
# core
authifier = { version = "1.0.8" }
authifier = { version = "1.0.9" }
revolt-result = { path = "../core/result" }
revolt-models = { path = "../core/models" }
revolt-config = { path = "../core/config" }
revolt-database = { path = "../core/database" }
revolt-permissions = { version = "0.7.16", path = "../core/permissions" }
revolt-permissions = { version = "0.8.2", path = "../core/permissions" }
revolt-presence = { path = "../core/presence", features = ["redis-is-patched"] }
# redis

View File

@@ -1,11 +1,12 @@
# Build Stage
FROM ghcr.io/revoltchat/base:latest AS builder
FROM debian:12 AS debian
# Bundle Stage
FROM gcr.io/distroless/cc-debian12:nonroot
COPY --from=builder /home/rust/src/target/release/revolt-bonfire ./
COPY --from=debian /usr/bin/uname /usr/bin/uname
EXPOSE 9000
EXPOSE 14703
USER nonroot
ENV HOST=0.0.0.0:9000
CMD ["./revolt-bonfire"]

View File

@@ -19,7 +19,10 @@ async fn main() {
database::connect().await;
// Clean up the current region information.
clear_region(None).await;
let no_clear_region = env::var("NO_CLEAR_PRESENCE").unwrap_or_else(|_| "0".into()) == "1";
if !no_clear_region {
clear_region(None).await;
}
// Setup a TCP listener to accept WebSocket connections on.
// By default, we bind to port 14703 on all interfaces.

View File

@@ -14,6 +14,7 @@ use futures::{
FutureExt, SinkExt, StreamExt, TryStreamExt,
};
use redis_kiss::{PayloadType, REDIS_PAYLOAD_TYPE, REDIS_URI};
use revolt_config::report_internal_error;
use revolt_database::{
events::{client::EventV1, server::ClientMessage},
Database, User, UserHint,
@@ -78,7 +79,9 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr)
// Try to authenticate the user.
let Some(token) = config.get_session_token().as_ref() else {
write
.send(config.encode(&create_error!(InvalidSession)))
.send(config.encode(&EventV1::Error {
data: create_error!(InvalidSession),
}))
.await
.ok();
return;
@@ -87,7 +90,10 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr)
let (user, session_id) = match User::from_token(db, token, UserHint::Any).await {
Ok(user) => user,
Err(err) => {
write.send(config.encode(&err)).await.ok();
write
.send(config.encode(&EventV1::Error { data: err }))
.await
.ok();
return;
}
};
@@ -99,27 +105,21 @@ pub async fn client(db: &'static Database, stream: TcpStream, addr: SocketAddr)
let user_id = state.cache.user_id.clone();
// Notify socket we have authenticated.
if let Err(err) = write.send(config.encode(&EventV1::Authenticated)).await {
error!("Failed to write: {err:?}");
sentry::capture_error(&err);
if report_internal_error!(write.send(config.encode(&EventV1::Authenticated)).await).is_err() {
return;
}
// Download required data to local cache and send Ready payload.
let ready_payload = match state
.generate_ready_payload(db, config.get_ready_payload_fields())
.await
{
let ready_payload = match report_internal_error!(
state
.generate_ready_payload(db, config.get_ready_payload_fields())
.await
) {
Ok(ready_payload) => ready_payload,
Err(err) => {
sentry::capture_error(&err);
return;
}
Err(_) => return,
};
if let Err(err) = write.send(config.encode(&ready_payload)).await {
error!("Failed to write: {err:?}");
sentry::capture_error(&err);
if report_internal_error!(write.send(config.encode(&ready_payload)).await).is_err() {
return;
}
@@ -214,21 +214,16 @@ async fn listener(
write: &Mutex<WsWriter>,
) {
let redis_config = RedisConfig::from_url(&REDIS_URI).unwrap();
let subscriber = match fred::types::Builder::from_config(redis_config).build_subscriber_client()
{
let subscriber = match report_internal_error!(
fred::types::Builder::from_config(redis_config).build_subscriber_client()
) {
Ok(subscriber) => subscriber,
Err(err) => {
error!("Failed to build a subscriber: {err:?}");
sentry::capture_error(&err);
return;
}
Err(_) => return,
};
if let Err(err) = subscriber.init().await {
error!("Failed to init subscriber: {err:?}");
sentry::capture_error(&err);
if report_internal_error!(subscriber.init().await).is_err() {
return;
};
}
// Handle Redis connection dropping
let (clean_up_s, clean_up_r) = async_channel::bounded(1);
@@ -249,17 +244,13 @@ async fn listener(
// Check for state changes for subscriptions.
match state.apply_state().await {
SubscriptionStateChange::Reset => {
if let Err(err) = subscriber.unsubscribe_all().await {
error!("Unsubscribe all failed: {err:?}");
sentry::capture_error(&err);
if report_internal_error!(subscriber.unsubscribe_all().await).is_err() {
break 'out;
}
let subscribed = state.subscribed.read().await;
for id in subscribed.iter() {
if let Err(err) = subscriber.subscribe(id).await {
error!("Subscribe failed: {err:?}");
sentry::capture_error(&err);
if report_internal_error!(subscriber.subscribe(id).await).is_err() {
break 'out;
}
}
@@ -272,9 +263,7 @@ async fn listener(
#[cfg(debug_assertions)]
info!("{addr:?} unsubscribing from {id}");
if let Err(err) = subscriber.unsubscribe(id).await {
error!("Unsubscribe failed: {err:?}");
sentry::capture_error(&err);
if report_internal_error!(subscriber.unsubscribe(id).await).is_err() {
break 'out;
}
}
@@ -283,9 +272,7 @@ async fn listener(
#[cfg(debug_assertions)]
info!("{addr:?} subscribing to {id}");
if let Err(err) = subscriber.subscribe(id).await {
error!("Subscribe failed: {err:?}");
sentry::capture_error(&err);
if report_internal_error!(subscriber.subscribe(id).await).is_err() {
break 'out;
}
}
@@ -310,38 +297,32 @@ async fn listener(
_ = t2 => {},
message = t1 => {
// Handle incoming events.
let message = match message {
let message = match report_internal_error!(message) {
Ok(message) => message,
Err(e) => {
error!("Error while consuming pub/sub messages: {e:?}");
sentry::capture_error(&e);
break 'out;
}
Err(_) => break 'out
};
let event = match *REDIS_PAYLOAD_TYPE {
PayloadType::Json => message
.value
.as_str()
.and_then(|s| serde_json::from_str::<EventV1>(s.as_ref()).ok()),
.and_then(|s| report_internal_error!(serde_json::from_str::<EventV1>(s.as_ref())).ok()),
PayloadType::Msgpack => message
.value
.as_bytes()
.and_then(|b| rmp_serde::from_slice::<EventV1>(b).ok()),
.and_then(|b| report_internal_error!(rmp_serde::from_slice::<EventV1>(b)).ok()),
PayloadType::Bincode => message
.value
.as_bytes()
.and_then(|b| bincode::deserialize::<EventV1>(b).ok()),
.and_then(|b| report_internal_error!(bincode::deserialize::<EventV1>(b)).ok()),
};
let Some(mut event) = event else {
let err = format!(
"Failed to deserialise an event for {}! Introspection: `{:?}`",
"Failed to deserialise event for {}: `{:?}`",
message.channel,
message
.value
.as_string()
.map(|x| x.chars().take(32).collect::<String>())
);
error!("{}", err);
@@ -393,10 +374,7 @@ async fn listener(
}
}
if let Err(err) = subscriber.quit().await {
error!("{}", err);
sentry::capture_error(&err);
}
report_internal_error!(subscriber.quit().await).ok();
}
#[allow(clippy::too_many_arguments)]

View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-config"
version = "0.7.16"
version = "0.8.2"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -9,12 +9,12 @@ description = "Revolt Backend: Configuration"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
report-macros = ["revolt-result"]
test = ["async-std"]
default = ["test"]
[dependencies]
# Utility
dotenv = "0.15.0"
config = "0.13.3"
cached = "0.44.0"
once_cell = "1.18.0"
@@ -32,3 +32,6 @@ pretty_env_logger = "0.4.0"
# Sentry
sentry = "0.31.5"
# Core
revolt-result = { version = "0.8.2", path = "../result", optional = true }

View File

@@ -1,3 +1,9 @@
[database]
mongodb = "mongodb://localhost"
redis = "redis://localhost/"
[rabbit]
host = "127.0.0.1"
port = 5672
username = "rabbituser"
password = "rabbitpass"

View File

@@ -20,6 +20,12 @@ january = "http://local.revolt.chat/january"
voso_legacy = ""
voso_legacy_ws = ""
[rabbit]
host = "rabbit"
port = 5672
username = "rabbituser"
password = "rabbitpass"
[api]
[api.registration]
@@ -38,34 +44,6 @@ from_address = "noreply@example.com"
# port = 587
# use_tls = true
[api.vapid]
# Generate your own keys:
# 1. Run `openssl ecparam -name prime256v1 -genkey -noout -out vapid_private.pem`
# 2. Find `private_key` using `base64 vapid_private.pem`
# 3. Find `public_key` using `openssl ec -in vapid_private.pem -outform DER|tail -c 65|base64|tr '/+' '_-'|tr -d '\n'`
private_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJSUWpyTWxLRnBiVWhsUHpUbERvcEliYk1yeVNrNXpKYzVYVzIxSjJDS3hvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFWnkrQkg2TGJQZ2hEa3pEempXOG0rUXVPM3pCajRXT1phdkR6ZU00c0pqbmFwd1psTFE0WAp1ZDh2TzVodU94QWhMQlU3WWRldVovWHlBdFpWZmNyQi9BPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo"
public_key = "BGcvgR-i2z4IQ5Mw841vJvkLjt8wY-FjmWrw83jOLCY52qcGZS0OF7nfLzuYbjsQISwVO2HXrmf18gLWVX3Kwfw="
[api.fcm]
# Google Firebase Cloud Messaging Service Account Key
# Obtained from the cloud messaging console
key_type = ""
project_id = ""
private_key_id = ""
private_key = ""
client_email = ""
client_id = ""
auth_uri = ""
token_uri = ""
auth_provider_x509_cert_url = ""
client_x509_cert_url = ""
[api.apn]
# Apple Push Notifications keys for sending notifications
sandbox = false
pkcs8 = ""
key_id = ""
team_id = ""
[api.security]
# Authifier Shield API key
@@ -84,6 +62,46 @@ hcaptcha_sitekey = ""
# Maximum concurrent connections (to proxy server)
max_concurrent_connections = 50
[pushd]
# this changes the names of the queues to not overlap
# prod/beta if they happen to be on the same exchange/instance.
# Usually they have to be, so that messages sent from one or the other get sent to everyone
production = true
# none of these should need changing
exchange = "revolt.notifications"
message_queue = "notifications.origin.message"
fr_accepted_queue = "notifications.ingest.fr_accepted" # friend request accepted
fr_received_queue = "notifications.ingest.fr_received" # friend request received
generic_queue = "notifications.ingest.generic" # generic messages (title + body)
ack_queue = "notifications.process.ack" # updates badges for apple devices
[pushd.vapid]
queue = "notifications.outbound.vapid"
private_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJSUWpyTWxLRnBiVWhsUHpUbERvcEliYk1yeVNrNXpKYzVYVzIxSjJDS3hvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFWnkrQkg2TGJQZ2hEa3pEempXOG0rUXVPM3pCajRXT1phdkR6ZU00c0pqbmFwd1psTFE0WAp1ZDh2TzVodU94QWhMQlU3WWRldVovWHlBdFpWZmNyQi9BPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo"
public_key = "BGcvgR-i2z4IQ5Mw841vJvkLjt8wY-FjmWrw83jOLCY52qcGZS0OF7nfLzuYbjsQISwVO2HXrmf18gLWVX3Kwfw="
[pushd.fcm]
queue = "notifications.outbound.fcm"
key_type = ""
project_id = ""
private_key_id = ""
private_key = ""
client_email = ""
client_id = ""
auth_uri = ""
token_uri = ""
auth_provider_x509_cert_url = ""
client_x509_cert_url = ""
[pushd.apn]
sandbox = false
queue = "notifications.outbound.apn"
pkcs8 = ""
key_id = ""
team_id = ""
[files]
# Encryption key for stored files
# Generate your own key using `openssl rand -base64 32`
@@ -98,8 +116,18 @@ blocked_mime_types = []
# ClamAV service
# hostname:port
clamd_host = ""
# Mime types that should be virus scanned
#
# Leave empty to scan all file types
scan_mime_types = [
"application/vnd.microsoft.portable-executable",
"application/vnd.android.package-archive",
"application/zip",
]
[files.limit]
# Minimum file size (in bytes)
min_file_size = 1
# Minimum image resolution
min_resolution = [1, 1]
# Maximum MP of images
@@ -122,6 +150,7 @@ emojis = [128, 128]
#
# Backblaze B2:
# - endpoint is listed on the "Buckets" page
# - path_style_buckets is set to true
# - region is `eu-central-003` string from endpoint URL
# - access_key_id is keyID generated on the "Application Keys" page
# - secret_access_key is token generated on the "Application Keys" page
@@ -129,16 +158,20 @@ emojis = [128, 128]
# S3 protocol endpoint
endpoint = "http://minio:9000"
# Whether to use path-style buckets
# Generally true, except for MinIO
path_style_buckets = false
# S3 region name
region = "minio"
# S3 protocol key ID
access_key_id = "minioautumn"
# S3 protocol access key
secret_access_key = "minioautumn"
# Bucket to upload to by default
default_bucket = "revolt-uploads"
[features]
# Bucket to upload to by default
# Feature gate options
webhooks_enabled = false
@@ -153,44 +186,76 @@ server_emoji = 100
server_roles = 200
server_channels = 200
# How many days since creation a user is considered new
new_user_days = 3
# How many hours since creation a user is considered new
new_user_hours = 72
# Maximum permissible body size in bytes for uploads
# (should be greater than any one file upload limit)
body_limit_size = 20_000_000
[features.limits.new_user]
# Limits imposed on new users
# Number of outgoing friend requests permitted at any time
outgoing_friend_requests = 5
# Maximum number of owned bots
bots = 2
message_length = 2000
message_attachments = 5
servers = 100
attachment_size = 20_000_000
avatar_size = 4_000_000
background_size = 6_000_000
icon_size = 2_500_000
banner_size = 6_000_000
emoji_size = 500_000
# Message content length
message_length = 2000
# Number of attachments that can be included
message_attachments = 5
# Maximum number of servers the user can create/join
servers = 50
[features.limits.new_user.file_upload_size_limit]
# Maximum file size limits (in bytes)
attachments = 20_000_000
avatars = 4_000_000
backgrounds = 6_000_000
icons = 2_500_000
banners = 6_000_000
emojis = 500_000
[features.limits.default]
# Limits imposed on users by default
# Number of outgoing friend requests permitted at any time
outgoing_friend_requests = 10
# Maximum number of owned bots
bots = 5
# Message content length
message_length = 2000
# Number of attachments that can be included
message_attachments = 5
# Maximum number of servers the user can create/join
servers = 100
attachment_size = 20_000_000
avatar_size = 4_000_000
background_size = 6_000_000
icon_size = 2_500_000
banner_size = 6_000_000
emoji_size = 500_000
[features.limits.default.file_upload_size_limit]
# Maximum file size limits (in bytes)
attachments = 20_000_000
avatars = 4_000_000
backgrounds = 6_000_000
icons = 2_500_000
banners = 6_000_000
emojis = 500_000
[features.advanced]
# The max amount of messages the rabbitmq provider/db mention adder job will delay for before forcing handling of a channel.
# default: 5
process_message_delay_limit = 5
[sentry]
# Configuration for Sentry error reporting
api = ""
events = ""
files = ""
proxy = ""
crond = ""

View File

@@ -6,11 +6,60 @@ use futures_locks::RwLock;
use once_cell::sync::Lazy;
use serde::Deserialize;
pub use sentry::capture_error;
pub use sentry::{capture_error, capture_message, Level};
#[cfg(not(debug_assertions))]
use std::env;
#[cfg(feature = "report-macros")]
#[macro_export]
macro_rules! report_error {
( $expr: expr, $error: ident $( $tt:tt )? ) => {
$expr
.inspect_err(|err| {
$crate::capture_message(
&format!("{err:?} ({}:{}:{})", file!(), line!(), column!()),
$crate::Level::Error,
);
})
.map_err(|_| ::revolt_result::create_error!($error))
};
}
#[cfg(feature = "report-macros")]
#[macro_export]
macro_rules! capture_internal_error {
( $expr: expr ) => {
$crate::capture_message(
&format!("{:?} ({}:{}:{})", $expr, file!(), line!(), column!()),
$crate::Level::Error,
);
};
}
#[cfg(feature = "report-macros")]
#[macro_export]
macro_rules! report_internal_error {
( $expr: expr ) => {
$expr
.inspect_err(|err| {
$crate::capture_message(
&format!("{err:?} ({}:{}:{})", file!(), line!(), column!()),
$crate::Level::Error,
);
})
.map_err(|_| ::revolt_result::create_error!(InternalError))
};
}
/// Paths to search for configuration
static CONFIG_SEARCH_PATHS: [&str; 3] = [
// current working directory
"Revolt.toml",
// current working directory - overrides file
"Revolt.overrides.toml",
// root directory, for Docker containers
"/Revolt.toml",
];
/// Configuration builder
static CONFIG_BUILDER: Lazy<RwLock<Config>> = Lazy::new(|| {
RwLock::new({
let mut builder = Config::builder().add_source(File::from_str(
@@ -23,24 +72,32 @@ static CONFIG_BUILDER: Lazy<RwLock<Config>> = Lazy::new(|| {
include_str!("../Revolt.test.toml"),
FileFormat::Toml,
));
} else if std::path::Path::new("Revolt.toml").exists() {
builder = builder.add_source(File::new("Revolt.toml", FileFormat::Toml));
} else if std::path::Path::new("/Revolt.toml").exists() {
builder = builder.add_source(File::new("/Revolt.toml", FileFormat::Toml));
}
for path in CONFIG_SEARCH_PATHS {
if std::path::Path::new(path).exists() {
builder = builder.add_source(File::new(path, FileFormat::Toml));
}
}
builder.build().unwrap()
})
});
// https://gifbox.me/view/gT5mqxYKCZv-twilight-meow
#[derive(Deserialize, Debug, Clone)]
pub struct Database {
pub mongodb: String,
pub redis: String,
}
#[derive(Deserialize, Debug, Clone)]
pub struct Rabbit {
pub host: String,
pub port: u16,
pub username: String,
pub password: String,
}
#[derive(Deserialize, Debug, Clone)]
pub struct Hosts {
pub app: String,
@@ -69,13 +126,15 @@ pub struct ApiSmtp {
}
#[derive(Deserialize, Debug, Clone)]
pub struct ApiVapid {
pub struct PushVapid {
pub queue: String,
pub private_key: String,
pub public_key: String,
}
#[derive(Deserialize, Debug, Clone)]
pub struct ApiFcm {
pub struct PushFcm {
pub queue: String,
pub key_type: String,
pub project_id: String,
pub private_key_id: String,
@@ -89,7 +148,8 @@ pub struct ApiFcm {
}
#[derive(Deserialize, Debug, Clone)]
pub struct ApiApn {
pub struct PushApn {
pub queue: String,
pub sandbox: bool,
pub pkcs8: String,
pub key_id: String,
@@ -119,15 +179,57 @@ pub struct ApiWorkers {
pub struct Api {
pub registration: ApiRegistration,
pub smtp: ApiSmtp,
pub vapid: ApiVapid,
pub fcm: ApiFcm,
pub apn: ApiApn,
pub security: ApiSecurity,
pub workers: ApiWorkers,
}
#[derive(Deserialize, Debug, Clone)]
pub struct Pushd {
pub production: bool,
pub exchange: String,
pub message_queue: String,
pub fr_accepted_queue: String,
pub fr_received_queue: String,
pub generic_queue: String,
pub ack_queue: String,
pub vapid: PushVapid,
pub fcm: PushFcm,
pub apn: PushApn,
}
impl Pushd {
fn get_routing_key(&self, key: String) -> String {
match self.production {
true => key + "-prd",
false => key + "-tst",
}
}
pub fn get_ack_routing_key(&self) -> String {
self.get_routing_key(self.ack_queue.clone())
}
pub fn get_message_routing_key(&self) -> String {
self.get_routing_key(self.message_queue.clone())
}
pub fn get_fr_accepted_routing_key(&self) -> String {
self.get_routing_key(self.fr_accepted_queue.clone())
}
pub fn get_fr_received_routing_key(&self) -> String {
self.get_routing_key(self.fr_received_queue.clone())
}
pub fn get_generic_routing_key(&self) -> String {
self.get_routing_key(self.generic_queue.clone())
}
}
#[derive(Deserialize, Debug, Clone)]
pub struct FilesLimit {
pub min_file_size: usize,
pub min_resolution: [usize; 2],
pub max_mega_pixels: usize,
pub max_pixel_side: usize,
@@ -136,6 +238,7 @@ pub struct FilesLimit {
#[derive(Deserialize, Debug, Clone)]
pub struct FilesS3 {
pub endpoint: String,
pub path_style_buckets: bool,
pub region: String,
pub access_key_id: String,
pub secret_access_key: String,
@@ -148,6 +251,7 @@ pub struct Files {
pub webp_quality: f32,
pub blocked_mime_types: Vec<String>,
pub clamd_host: String,
pub scan_mime_types: Vec<String>,
pub limit: FilesLimit,
pub preview: HashMap<String, [usize; 2]>,
@@ -164,7 +268,7 @@ pub struct GlobalLimits {
pub server_roles: usize,
pub server_channels: usize,
pub new_user_days: usize,
pub new_user_hours: usize,
pub body_limit_size: usize,
}
@@ -178,12 +282,7 @@ pub struct FeaturesLimits {
pub message_attachments: usize,
pub servers: usize,
pub attachment_size: usize,
pub avatar_size: usize,
pub background_size: usize,
pub icon_size: usize,
pub banner_size: usize,
pub emoji_size: usize,
pub file_upload_size_limit: HashMap<String, usize>,
}
#[derive(Deserialize, Debug, Clone)]
@@ -197,23 +296,44 @@ pub struct FeaturesLimitsCollection {
pub roles: HashMap<String, FeaturesLimits>,
}
#[derive(Deserialize, Debug, Clone)]
pub struct FeaturesAdvanced {
#[serde(default)]
pub process_message_delay_limit: u16,
}
impl Default for FeaturesAdvanced {
fn default() -> Self {
Self {
process_message_delay_limit: 5,
}
}
}
#[derive(Deserialize, Debug, Clone)]
pub struct Features {
pub limits: FeaturesLimitsCollection,
pub webhooks_enabled: bool,
#[serde(default)]
pub advanced: FeaturesAdvanced,
}
#[derive(Deserialize, Debug, Clone)]
pub struct Sentry {
pub api: String,
pub events: String,
pub files: String,
pub proxy: String,
pub crond: String,
}
#[derive(Deserialize, Debug, Clone)]
pub struct Settings {
pub database: Database,
pub rabbit: Rabbit,
pub hosts: Hosts,
pub api: Api,
pub pushd: Pushd,
pub files: Files,
pub features: Features,
pub sentry: Sentry,
@@ -222,22 +342,10 @@ pub struct Settings {
impl Settings {
pub fn preflight_checks(&self) {
if self.api.smtp.host.is_empty() {
#[cfg(not(debug_assertions))]
if !env::var("REVOLT_UNSAFE_NO_EMAIL").map_or(false, |v| v == *"1") {
panic!("Running in production without email is not recommended, set REVOLT_UNSAFE_NO_EMAIL=1 to override.");
}
#[cfg(debug_assertions)]
log::warn!("No SMTP settings specified! Remember to configure email.");
}
if self.api.security.captcha.hcaptcha_key.is_empty() {
#[cfg(not(debug_assertions))]
if !env::var("REVOLT_UNSAFE_NO_CAPTCHA").map_or(false, |v| v == *"1") {
panic!("Running in production without CAPTCHA is not recommended, set REVOLT_UNSAFE_NO_CAPTCHA=1 to override.");
}
#[cfg(debug_assertions)]
log::warn!("No Captcha key specified! Remember to add hCaptcha key.");
}
}
@@ -261,8 +369,6 @@ pub async fn config() -> Settings {
/// Configure logging and common Rust variables
pub async fn setup_logging(release: &'static str, dsn: String) -> Option<sentry::ClientInitGuard> {
dotenv::dotenv().ok();
if std::env::var("RUST_LOG").is_err() {
std::env::set_var("RUST_LOG", "info");
}
@@ -271,6 +377,12 @@ pub async fn setup_logging(release: &'static str, dsn: String) -> Option<sentry:
std::env::set_var("ROCKET_ADDRESS", "0.0.0.0");
}
if std::env::var("REDIS_URL").is_err() {
// Configure redis-kiss library
let config = config().await;
std::env::set_var("REDIS_URI", config.database.redis);
}
pretty_env_logger::init();
log::info!("Starting {release}");

View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-database"
version = "0.7.16"
version = "0.8.2"
edition = "2021"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -16,6 +16,7 @@ mongodb = ["dep:mongodb", "bson"]
tasks = ["isahc", "linkify", "url-escape"]
async-std-runtime = ["async-std"]
rocket-impl = ["rocket", "schemars", "revolt_okapi", "revolt_rocket_okapi"]
axum-impl = ["axum"]
redis-is-patched = ["revolt-presence/redis-is-patched"]
# Default Features
@@ -23,13 +24,15 @@ default = ["mongodb", "async-std-runtime", "tasks"]
[dependencies]
# Core
revolt-config = { version = "0.7.16", path = "../config" }
revolt-result = { version = "0.7.16", path = "../result" }
revolt-models = { version = "0.7.16", path = "../models", features = [
revolt-config = { version = "0.8.2", path = "../config", features = [
"report-macros",
] }
revolt-result = { version = "0.8.2", path = "../result" }
revolt-models = { version = "0.8.2", path = "../models", features = [
"validator",
] }
revolt-presence = { version = "0.7.16", path = "../presence" }
revolt-permissions = { version = "0.7.16", path = "../permissions", features = [
revolt-presence = { version = "0.8.2", path = "../presence" }
revolt-permissions = { version = "0.8.2", path = "../permissions", features = [
"serde",
"bson",
] }
@@ -76,13 +79,16 @@ async-recursion = "1.0.4"
# Async
async-std = { version = "1.8.0", features = ["attributes"], optional = true }
# Axum Impl
axum = { version = "0.7.5", optional = true }
# Rocket Impl
schemars = { version = "0.8.8", optional = true }
rocket = { version = "0.5.0-rc.2", default-features = false, features = [
rocket = { version = "0.5.1", default-features = false, features = [
"json",
], optional = true }
revolt_okapi = { version = "0.9.1", optional = true }
revolt_rocket_okapi = { version = "0.9.1", optional = true }
revolt_rocket_okapi = { version = "0.10.0", optional = true }
# Notifications
fcm_v1 = "0.3.0"
@@ -90,4 +96,7 @@ web-push = "0.10.0"
revolt_a2 = { version = "0.10", default-features = false, features = ["ring"] }
# Authifier
authifier = { version = "1.0.8" }
authifier = { version = "1.0.9", features = ["rocket_impl"] }
# RabbitMQ
amqprs = { version = "1.7.0" }

View File

@@ -0,0 +1,211 @@
use std::collections::HashSet;
use crate::events::rabbit::*;
use crate::User;
use amqprs::channel::BasicPublishArguments;
use amqprs::{channel::Channel, connection::Connection, error::Error as AMQPError};
use amqprs::{BasicProperties, FieldTable};
use revolt_models::v0::PushNotification;
use revolt_presence::filter_online;
use serde_json::to_string;
#[derive(Clone)]
pub struct AMQP {
#[allow(unused)]
connection: Connection,
channel: Channel,
}
impl AMQP {
pub fn new(connection: Connection, channel: Channel) -> AMQP {
AMQP {
connection,
channel,
}
}
pub async fn friend_request_accepted(
&self,
accepted_request_user: &User,
sent_request_user: &User,
) -> Result<(), AMQPError> {
let config = revolt_config::config().await;
let payload = FRAcceptedPayload {
accepted_user: accepted_request_user.to_owned(),
user: sent_request_user.id.clone(),
};
let payload = to_string(&payload).unwrap();
debug!(
"Sending friend request accept payload on channel {}: {}",
config.pushd.get_fr_accepted_routing_key(),
payload
);
self.channel
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_fr_accepted_routing_key(),
),
)
.await
}
pub async fn friend_request_received(
&self,
received_request_user: &User,
sent_request_user: &User,
) -> Result<(), AMQPError> {
let config = revolt_config::config().await;
let payload = FRReceivedPayload {
from_user: sent_request_user.to_owned(),
user: received_request_user.id.clone(),
};
let payload = to_string(&payload).unwrap();
debug!(
"Sending friend request received payload on channel {}: {}",
config.pushd.get_fr_received_routing_key(),
payload
);
self.channel
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_fr_received_routing_key(),
),
)
.await
}
pub async fn generic_message(
&self,
user: &User,
title: String,
body: String,
icon: Option<String>,
) -> Result<(), AMQPError> {
let config = revolt_config::config().await;
let payload = GenericPayload {
title,
body,
icon,
user: user.to_owned(),
};
let payload = to_string(&payload).unwrap();
debug!(
"Sending generic payload on channel {}: {}",
config.pushd.get_generic_routing_key(),
payload
);
self.channel
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_generic_routing_key(),
),
)
.await
}
pub async fn message_sent(
&self,
recipients: Vec<String>,
payload: PushNotification,
) -> Result<(), AMQPError> {
if recipients.is_empty() {
return Ok(());
}
let config = revolt_config::config().await;
let online_ids = filter_online(&recipients).await;
let recipients = (&recipients.into_iter().collect::<HashSet<String>>() - &online_ids)
.into_iter()
.collect::<Vec<String>>();
let payload = MessageSentPayload {
notification: payload,
users: recipients,
};
let payload = to_string(&payload).unwrap();
debug!(
"Sending message payload on channel {}: {}",
config.pushd.get_message_routing_key(),
payload
);
self.channel
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
.finish(),
payload.into(),
BasicPublishArguments::new(
&config.pushd.exchange,
&config.pushd.get_message_routing_key(),
),
)
.await
}
pub async fn ack_message(
&self,
user_id: String,
channel_id: String,
message_id: String,
) -> Result<(), AMQPError> {
let config = revolt_config::config().await;
let payload = AckPayload {
user_id: user_id.clone(),
channel_id: channel_id.clone(),
message_id,
};
let payload = to_string(&payload).unwrap();
info!(
"Sending ack payload on channel {}: {}",
config.pushd.ack_queue, payload
);
let mut headers = FieldTable::new();
headers.insert(
"x-deduplication-header".try_into().unwrap(),
format!("{}-{}", &user_id, &channel_id).into(),
);
self.channel
.basic_publish(
BasicProperties::default()
.with_content_type("application/json")
.with_persistence(true)
//.with_headers(headers)
.finish(),
payload.into(),
BasicPublishArguments::new(&config.pushd.exchange, &config.pushd.ack_queue),
)
.await
}
}

View File

@@ -0,0 +1,2 @@
#[allow(clippy::module_inception)]
pub mod amqp;

View File

@@ -1,25 +1,16 @@
use authifier::AuthifierEvent;
use revolt_result::Error;
use serde::{Deserialize, Serialize};
use revolt_models::v0::{
AppendMessage, Channel, ChannelUnread, Emoji, FieldsChannel, FieldsMember, FieldsMessage, FieldsRole, FieldsServer, FieldsUser, FieldsWebhook, Member, MemberCompositeKey, Message, PartialChannel, PartialMember, PartialMessage, PartialRole, PartialServer, PartialUser, PartialWebhook, RemovalIntention, Report, Server, User, UserSettings, Webhook
AppendMessage, Channel, ChannelUnread, Emoji, FieldsChannel, FieldsMember, FieldsMessage,
FieldsRole, FieldsServer, FieldsUser, FieldsWebhook, Member, MemberCompositeKey, Message,
PartialChannel, PartialMember, PartialMessage, PartialRole, PartialServer, PartialUser,
PartialWebhook, RemovalIntention, Report, Server, User, UserSettings, Webhook,
};
use revolt_result::Error;
use crate::Database;
/// WebSocket Client Errors
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(tag = "error")]
pub enum WebSocketError {
LabelMe,
InternalError { at: String },
InvalidSession,
OnboardingNotFinished,
AlreadyAuthenticated,
MalformedData { msg: String },
}
/// Ping Packet
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(untagged)]
@@ -28,14 +19,6 @@ pub enum Ping {
Number(usize),
}
/// Untagged Error
#[derive(Serialize)]
#[serde(untagged)]
pub enum ErrorEvent {
Error(WebSocketError),
APIError(Error),
}
/// Fields provided in Ready payload
#[derive(PartialEq)]
pub enum ReadyPayloadFields {
@@ -55,6 +38,8 @@ pub enum ReadyPayloadFields {
pub enum EventV1 {
/// Multiple events
Bulk { v: Vec<EventV1> },
/// Error event
Error { data: Error },
/// Successfully authenticated
Authenticated,
@@ -89,6 +74,7 @@ pub enum EventV1 {
id: String,
channel: String,
data: PartialMessage,
#[serde(default)]
clear: Vec<FieldsMessage>,
},
@@ -140,6 +126,7 @@ pub enum EventV1 {
ServerUpdate {
id: String,
data: PartialServer,
#[serde(default)]
clear: Vec<FieldsServer>,
},
@@ -150,6 +137,7 @@ pub enum EventV1 {
ServerMemberUpdate {
id: MemberCompositeKey,
data: PartialMember,
#[serde(default)]
clear: Vec<FieldsMember>,
},
@@ -168,6 +156,7 @@ pub enum EventV1 {
id: String,
role_id: String,
data: PartialRole,
#[serde(default)]
clear: Vec<FieldsRole>,
},
@@ -178,6 +167,7 @@ pub enum EventV1 {
UserUpdate {
id: String,
data: PartialUser,
#[serde(default)]
clear: Vec<FieldsUser>,
event_id: Option<String>,
},
@@ -212,6 +202,7 @@ pub enum EventV1 {
ChannelUpdate {
id: String,
data: PartialChannel,
#[serde(default)]
clear: Vec<FieldsChannel>,
},

View File

@@ -1,2 +1,3 @@
pub mod client;
pub mod rabbit;
pub mod server;

View File

@@ -0,0 +1,59 @@
use std::collections::HashMap;
use revolt_models::v0::PushNotification;
use serde::{Deserialize, Serialize};
use crate::User;
#[derive(Serialize, Deserialize)]
pub struct MessageSentPayload {
pub notification: PushNotification,
pub users: Vec<String>,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct FRAcceptedPayload {
pub accepted_user: User,
pub user: String,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct FRReceivedPayload {
pub from_user: User,
pub user: String,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct GenericPayload {
pub title: String,
pub body: String,
pub icon: Option<String>,
pub user: User,
}
#[derive(Serialize, Deserialize)]
#[serde(tag = "type", content = "data")]
#[allow(clippy::large_enum_variant)]
pub enum PayloadKind {
MessageNotification(PushNotification),
FRAccepted(FRAcceptedPayload),
FRReceived(FRReceivedPayload),
BadgeUpdate(usize),
Generic(GenericPayload),
}
#[derive(Serialize, Deserialize)]
pub struct PayloadToService {
pub notification: PayloadKind,
pub user_id: String,
pub session_id: String,
pub token: String,
pub extras: HashMap<String, String>,
}
#[derive(Serialize, Deserialize)]
pub struct AckPayload {
pub user_id: String,
pub channel_id: String,
pub message_id: String,
}

View File

@@ -16,6 +16,8 @@ extern crate revolt_optional_struct;
#[macro_use]
extern crate revolt_result;
pub use iso8601_timestamp;
#[cfg(feature = "mongodb")]
pub use mongodb;
@@ -23,6 +25,26 @@ pub use mongodb;
#[macro_use]
extern crate bson;
#[macro_export]
#[cfg(debug_assertions)]
macro_rules! query {
( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
Ok($self.$type($collection, $($rest),+).await.unwrap())
};
}
#[macro_export]
#[cfg(not(debug_assertions))]
macro_rules! query {
( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
$self.$type($collection, $($rest),+).await
.map_err(|err| {
revolt_config::capture_internal_error!(err);
create_database_error!(stringify!($type), $collection)
})
};
}
macro_rules! database_derived {
( $( $item:item )+ ) => {
$(
@@ -83,6 +105,9 @@ pub use models::*;
pub mod events;
pub mod tasks;
mod amqp;
pub use amqp::amqp::AMQP;
/// Utility function to check if a boolean value is false
pub fn if_false(t: &bool) -> bool {
!t
@@ -91,4 +116,4 @@ pub fn if_false(t: &bool) -> bool {
/// Utility function to check if an option doesnt contain true
pub fn if_option_false(t: &Option<bool>) -> bool {
t != &Some(true)
}
}

View File

@@ -56,6 +56,10 @@ pub async fn create_database(db: &MongoDb) {
.await
.expect("Failed to create attachments collection.");
db.create_collection("attachment_hashes", None)
.await
.expect("Failed to create attachment_hashes collection.");
db.create_collection("user_settings", None)
.await
.expect("Failed to create user_settings collection.");
@@ -209,6 +213,46 @@ pub async fn create_database(db: &MongoDb) {
.await
.expect("Failed to create server_members index.");
db.run_command(
doc! {
"createIndexes": "attachments",
"indexes": [
{
"key": {
"hash": 1_i32
},
"name": "hash"
},
{
"key": {
"used_for.id": 1_i32
},
"name": "used_for_id"
}
]
},
None,
)
.await
.expect("Failed to create attachments index.");
db.run_command(
doc! {
"createIndexes": "attachment_hashes",
"indexes": [
{
"key": {
"processed_hash": 1_i32
},
"name": "processed_hash"
}
]
},
None,
)
.await
.expect("Failed to create attachment_hashes index.");
db.collection("migrations")
.insert_one(
doc! {

View File

@@ -5,12 +5,13 @@ use crate::{
bson::{doc, from_bson, from_document, to_document, Bson, DateTime, Document},
options::FindOptions,
},
Invite, MongoDb, DISCRIMINATOR_SEARCH_SPACE,
AbstractChannels, AbstractServers, Channel, Invite, MongoDb, DISCRIMINATOR_SEARCH_SPACE,
};
use bson::oid::ObjectId;
use futures::StreamExt;
use rand::seq::SliceRandom;
use revolt_permissions::DEFAULT_WEBHOOK_PERMISSIONS;
use revolt_result::{Error, ErrorType};
use serde::{Deserialize, Serialize};
use unicode_segmentation::UnicodeSegmentation;
@@ -20,7 +21,7 @@ struct MigrationInfo {
revision: i32,
}
pub const LATEST_REVISION: i32 = 28;
pub const LATEST_REVISION: i32 = 31;
pub async fn migrate_database(db: &MongoDb) {
let migrations = db.col::<Document>("migrations");
@@ -1094,7 +1095,137 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 {
.expect("Failed to create message index.");
}
// Need to migrate fields on attachments, change `user_id`, `object_id`, etc to `parent`.
if revision <= 28 {
info!("Running migration [revision 28 / 10-09-2024]: Add support for new Autumn.");
db.db()
.create_collection("attachment_hashes", None)
.await
.ok();
db.db()
.run_command(
doc! {
"createIndexes": "attachments",
"indexes": [
{
"key": {
"hash": 1_i32
},
"name": "hash"
}
]
},
None,
)
.await
.expect("Failed to create attachments index.");
db.db()
.run_command(
doc! {
"createIndexes": "attachment_hashes",
"indexes": [
{
"key": {
"processed_hash": 1_i32
},
"name": "processed_hash"
}
]
},
None,
)
.await
.expect("Failed to create attachment_hashes index.");
}
// Revision 29 omitted due to bug.
if revision <= 30 {
info!("Running migration [revision 30 / 29-09-2024]: Add index for used_for.id to attachments.");
db.db()
.run_command(
doc! {
"createIndexes": "attachments",
"indexes": [
{
"key": {
"used_for.id": 1_i32
},
"name": "used_for_id"
}
]
},
None,
)
.await
.expect("Failed to create attachments index.");
}
if revision <= 31 {
info!("Running migration [revision 31 / 31-10-2024]: Add creator_id to webhooks and delete those whose channels don't exist.");
#[derive(serde::Serialize, serde::Deserialize)]
struct WebhookShell {
_id: String,
channel_id: String,
}
let webhooks = db
.db()
.collection::<WebhookShell>("channel_webhooks")
.find(doc! {}, None)
.await
.expect("webhooks")
.filter_map(|s| async { s.ok() })
.collect::<Vec<WebhookShell>>()
.await;
for webhook in webhooks {
match db.fetch_channel(&webhook.channel_id).await {
Ok(channel) => {
let creator_id = match channel {
Channel::Group { owner, .. } => owner,
Channel::TextChannel { server, .. }
| Channel::VoiceChannel { server, .. } => {
let server = db.fetch_server(&server).await.expect("server");
server.owner
}
_ => unreachable!("not server or group channel!"),
};
db.db()
.collection::<Document>("channel_webhooks")
.update_one(
doc! {
"_id": webhook._id,
},
doc! {
"$set" : {
"creator_id": creator_id
}
},
None,
)
.await
.expect("update webhook");
}
Err(Error {
error_type: ErrorType::NotFound,
..
}) => {
db.db()
.collection::<WebhookShell>("channel_webhooks")
.delete_one(doc! { "_id": webhook._id }, None)
.await
.expect("failed to delete invalid webhook");
}
Err(err) => panic!("{err:?}"),
}
}
}
// Reminder to update LATEST_REVISION when adding new migrations.
LATEST_REVISION.max(revision)

View File

@@ -26,6 +26,9 @@ pub trait AbstractChannelUnreads: Sync + Send {
message_ids: &[String],
) -> Result<()>;
/// Fetch all unreads with mentions for a user.
async fn fetch_unread_mentions(&self, user_id: &str) -> Result<Vec<ChannelUnread>>;
/// Fetch all channel unreads for a user.
async fn fetch_unreads(&self, user_id: &str) -> Result<Vec<ChannelUnread>>;

View File

@@ -30,7 +30,7 @@ impl AbstractChannelUnreads for MongoDb {
doc! {
"$pull": {
"mentions": {
"$lt": message_id
"$lte": message_id
}
},
"$set": {
@@ -123,6 +123,18 @@ impl AbstractChannelUnreads for MongoDb {
)
}
async fn fetch_unread_mentions(&self, user_id: &str) -> Result<Vec<ChannelUnread>> {
query! {
self,
find,
COL,
doc! {
"_id.user": user_id,
"mentions": {"$ne": null}
}
}
}
/// Fetch unread for a specific user in a channel.
async fn fetch_unread(&self, user_id: &str, channel_id: &str) -> Result<Option<ChannelUnread>> {
query!(
@@ -135,5 +147,4 @@ impl AbstractChannelUnreads for MongoDb {
}
)
}
}

View File

@@ -78,6 +78,15 @@ impl AbstractChannelUnreads for ReferenceDb {
Ok(())
}
async fn fetch_unread_mentions(&self, user_id: &str) -> Result<Vec<ChannelUnread>> {
let unreads = self.channel_unreads.lock().await;
Ok(unreads
.values()
.filter(|unread| unread.id.user == user_id && unread.mentions.is_some())
.cloned()
.collect())
}
/// Fetch all channel unreads for a user.
async fn fetch_unreads(&self, user_id: &str) -> Result<Vec<ChannelUnread>> {
let unreads = self.channel_unreads.lock().await;
@@ -92,9 +101,11 @@ impl AbstractChannelUnreads for ReferenceDb {
async fn fetch_unread(&self, user_id: &str, channel_id: &str) -> Result<Option<ChannelUnread>> {
let unreads = self.channel_unreads.lock().await;
Ok(unreads.get(&ChannelCompositeKey {
channel: channel_id.to_string(),
user: user_id.to_string()
}).cloned())
Ok(unreads
.get(&ChannelCompositeKey {
channel: channel_id.to_string(),
user: user_id.to_string(),
})
.cloned())
}
}

View File

@@ -17,6 +17,9 @@ auto_derived_partial!(
#[serde(skip_serializing_if = "Option::is_none")]
pub avatar: Option<File>,
/// User that created this webhook
pub creator_id: String,
/// The channel this webhook belongs to
pub channel_id: String,
@@ -43,6 +46,7 @@ impl Default for Webhook {
id: Default::default(),
name: Default::default(),
avatar: None,
creator_id: Default::default(),
channel_id: Default::default(),
permissions: Default::default(),
token: Default::default(),
@@ -70,7 +74,7 @@ impl Webhook {
if self.token.as_deref() == Some(token) {
Ok(())
} else {
Err(create_error!(InvalidCredentials))
Err(create_error!(NotAuthenticated))
}
}

View File

@@ -9,7 +9,7 @@ use ulid::Ulid;
use crate::{
events::client::EventV1, tasks::ack::AckEvent, Database, File, IntoDocumentPath, PartialServer,
Server, SystemMessage, User,
Server, SystemMessage, User, AMQP,
};
auto_derived!(
@@ -337,6 +337,7 @@ impl Channel {
pub async fn add_user_to_group(
&mut self,
db: &Database,
amqp: &AMQP,
user: &User,
by_id: &str,
) -> Result<()> {
@@ -373,6 +374,7 @@ impl Channel {
.into_message(id.to_string())
.send(
db,
Some(amqp),
MessageAuthor::System {
username: &user.username,
avatar: user.avatar.as_ref().map(|file| file.id.as_ref()),
@@ -639,7 +641,7 @@ impl Channel {
.private(user.to_string())
.await;
crate::tasks::ack::queue(
crate::tasks::ack::queue_ack(
self.id().to_string(),
user.to_string(),
AckEvent::AckMessage {
@@ -655,6 +657,7 @@ impl Channel {
pub async fn remove_user_from_group(
&self,
db: &Database,
amqp: &AMQP,
user: &User,
by_id: Option<&str>,
silent: bool,
@@ -686,6 +689,7 @@ impl Channel {
.into_message(id.to_string())
.send(
db,
Some(amqp),
MessageAuthor::System {
username: name,
avatar: None,
@@ -725,6 +729,7 @@ impl Channel {
.into_message(id.to_string())
.send(
db,
Some(amqp),
MessageAuthor::System {
username: &user.username,
avatar: user.avatar.as_ref().map(|file| file.id.as_ref()),

View File

@@ -261,7 +261,7 @@ impl AbstractChannels for MongoDb {
// Delete associated attachments
self.delete_many_attachments(doc! {
"object_id": &id
"used_for.id": &id
})
.await?;

View File

@@ -1,5 +1,7 @@
use iso8601_timestamp::Timestamp;
use crate::File;
auto_derived_partial!(
/// File hash
pub struct FileHash {
@@ -43,7 +45,7 @@ auto_derived!(
Image {
width: isize,
height: isize,
// animated: bool // TODO: https://docs.rs/image/latest/image/trait.AnimationDecoder.html
// animated: bool // TODO: https://docs.rs/image/latest/image/trait.AnimationDecoder.html for APNG support
},
/// File is a video with specific dimensions
Video { width: isize, height: isize },
@@ -51,3 +53,40 @@ auto_derived!(
Audio,
}
);
impl FileHash {
/// Create a file from a file hash
pub fn into_file(
&self,
id: String,
tag: String,
filename: String,
uploader_id: String,
) -> File {
File {
id,
tag,
filename,
hash: Some(self.id.clone()),
uploaded_at: Some(Timestamp::now_utc()),
uploader_id: Some(uploader_id),
used_for: None,
deleted: None,
reported: None,
// TODO: remove this data
metadata: self.metadata.clone(),
content_type: self.content_type.clone(),
size: self.size,
// TODO: superseded by "used_for"
message_id: None,
object_id: None,
server_id: None,
user_id: None,
}
}
}

View File

@@ -7,6 +7,15 @@ mod reference;
#[async_trait]
pub trait AbstractAttachmentHashes: Sync + Send {
/// Insert a new attachment hash into the database.
async fn insert_attachment_hash(&self, hash: &FileHash) -> Result<()>;
/// Fetch an attachment hash entry by sha256 hash.
async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash>;
/// Update an attachment hash nonce value.
async fn set_attachment_hash_nonce(&self, hash: &str, nonce: &str) -> Result<()>;
/// Delete attachment hash by id.
async fn delete_attachment_hash(&self, id: &str) -> Result<()>;
}

View File

@@ -9,6 +9,11 @@ static COL: &str = "attachment_hashes";
#[async_trait]
impl AbstractAttachmentHashes for MongoDb {
/// Insert a new attachment hash into the database.
async fn insert_attachment_hash(&self, hash: &FileHash) -> Result<()> {
query!(self, insert_one, COL, &hash).map(|_| ())
}
/// Fetch an attachment hash entry by sha256 hash.
async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash> {
query!(
@@ -16,9 +21,36 @@ impl AbstractAttachmentHashes for MongoDb {
find_one,
COL,
doc! {
"_id": hash
"$or": [
{"_id": hash},
{"processed_hash": hash}
]
}
)?
.ok_or_else(|| create_error!(NotFound))
}
/// Update an attachment hash nonce value.
async fn set_attachment_hash_nonce(&self, hash: &str, nonce: &str) -> Result<()> {
self.col::<FileHash>(COL)
.update_one(
doc! {
"_id": hash
},
doc! {
"$set": {
"iv": nonce
}
},
None,
)
.await
.map(|_| ())
.map_err(|_| create_database_error!("update_one", COL))
}
/// Delete attachment hash by id.
async fn delete_attachment_hash(&self, id: &str) -> Result<()> {
query!(self, delete_one_by_id, COL, id).map(|_| ())
}
}

View File

@@ -7,15 +7,43 @@ use super::AbstractAttachmentHashes;
#[async_trait]
impl AbstractAttachmentHashes for ReferenceDb {
/// Insert a new attachment hash into the database.
async fn insert_attachment_hash(&self, hash: &FileHash) -> Result<()> {
let mut hashes = self.file_hashes.lock().await;
if hashes.contains_key(&hash.id) {
Err(create_database_error!("insert", "attachment"))
} else {
hashes.insert(hash.id.to_string(), hash.clone());
Ok(())
}
}
/// Fetch an attachment hash entry by sha256 hash.
async fn fetch_attachment_hash(&self, hash: &str) -> Result<FileHash> {
async fn fetch_attachment_hash(&self, hash_value: &str) -> Result<FileHash> {
let hashes = self.file_hashes.lock().await;
if let Some(file) = hashes.get(hash) {
if file.id == hash {
Ok(file.clone())
} else {
Err(create_error!(NotFound))
}
hashes
.values()
.find(|&hash| hash.id == hash_value || hash.processed_hash == hash_value)
.cloned()
.ok_or(create_error!(NotFound))
}
/// Update an attachment hash nonce value.
async fn set_attachment_hash_nonce(&self, hash: &str, nonce: &str) -> Result<()> {
let mut hashes = self.file_hashes.lock().await;
if let Some(file) = hashes.get_mut(hash) {
file.iv = nonce.to_owned();
Ok(())
} else {
Err(create_error!(NotFound))
}
}
/// Delete attachment hash by id.
async fn delete_attachment_hash(&self, id: &str) -> Result<()> {
let mut file_hashes = self.file_hashes.lock().await;
if file_hashes.remove(id).is_some() {
Ok(())
} else {
Err(create_error!(NotFound))
}

View File

@@ -19,9 +19,11 @@ auto_derived_partial!(
/// When this file was uploaded
pub uploaded_at: Option<Timestamp>, // these are Option<>s to not break file uploads on legacy Autumn
/// ID of user who uploaded this file
pub uploaded_id: Option<String>, // these are Option<>s to not break file uploads on legacy Autumn
#[serde(skip_serializing_if = "Option::is_none")]
pub uploader_id: Option<String>, // these are Option<>s to not break file uploads on legacy Autumn
/// What the file was used for
#[serde(skip_serializing_if = "Option::is_none")]
pub used_for: Option<FileUsedFor>,
/// Whether this file was deleted
@@ -59,15 +61,15 @@ auto_derived_partial!(
auto_derived!(
/// Type of object file was used for
pub enum FileUsedForType {
// TODO: changing this requires a db migration
message,
serverBanner,
emoji,
userAvatar,
userProfileBackground,
legacyGroupIcon,
channelIcon,
serverIcon,
Message,
ServerBanner,
Emoji,
UserAvatar,
WebhookAvatar,
UserProfileBackground,
LegacyGroupIcon,
ChannelIcon,
ServerIcon,
}
/// Information about what the file was used for
@@ -87,44 +89,154 @@ impl File {
}
/// Use a file for a message attachment
pub async fn use_attachment(db: &Database, id: &str, parent: &str) -> Result<File> {
db.find_and_use_attachment(id, "attachments", "message", parent)
.await
pub async fn use_attachment(
db: &Database,
id: &str,
parent: &str,
uploader_id: &str,
) -> Result<File> {
db.find_and_use_attachment(
id,
"attachments",
FileUsedFor {
id: parent.to_owned(),
object_type: FileUsedForType::Message,
},
uploader_id.to_owned(),
)
.await
}
/// Use a file for a user profile background
pub async fn use_background(db: &Database, id: &str, parent: &str) -> Result<File> {
db.find_and_use_attachment(id, "backgrounds", "user", parent)
.await
pub async fn use_background(
db: &Database,
id: &str,
parent: &str,
uploader_id: &str,
) -> Result<File> {
db.find_and_use_attachment(
id,
"backgrounds",
FileUsedFor {
id: parent.to_owned(),
object_type: FileUsedForType::UserProfileBackground,
},
uploader_id.to_owned(),
)
.await
}
/// Use a file for a user avatar
pub async fn use_avatar(db: &Database, id: &str, parent: &str) -> Result<File> {
db.find_and_use_attachment(id, "avatars", "user", parent)
.await
pub async fn use_user_avatar(
db: &Database,
id: &str,
parent: &str,
uploader_id: &str,
) -> Result<File> {
db.find_and_use_attachment(
id,
"avatars",
FileUsedFor {
id: parent.to_owned(),
object_type: FileUsedForType::UserAvatar,
},
uploader_id.to_owned(),
)
.await
}
/// Use a file for an icon
pub async fn use_icon(db: &Database, id: &str, parent: &str) -> Result<File> {
db.find_and_use_attachment(id, "icons", "object", parent)
.await
/// Use a file for a webhook avatar
pub async fn use_webhook_avatar(
db: &Database,
id: &str,
parent: &str,
uploader_id: &str,
) -> Result<File> {
db.find_and_use_attachment(
id,
"avatars",
FileUsedFor {
id: parent.to_owned(),
object_type: FileUsedForType::WebhookAvatar,
},
uploader_id.to_owned(),
)
.await
}
/// Use a file for a server icon
pub async fn use_server_icon(db: &Database, id: &str, parent: &str) -> Result<File> {
db.find_and_use_attachment(id, "icons", "object", parent)
.await
pub async fn use_server_icon(
db: &Database,
id: &str,
parent: &str,
uploader_id: &str,
) -> Result<File> {
db.find_and_use_attachment(
id,
"icons",
FileUsedFor {
id: parent.to_owned(),
object_type: FileUsedForType::ServerIcon,
},
uploader_id.to_owned(),
)
.await
}
/// Use a file for a channel icon
pub async fn use_channel_icon(
db: &Database,
id: &str,
parent: &str,
uploader_id: &str,
) -> Result<File> {
db.find_and_use_attachment(
id,
"icons",
FileUsedFor {
id: parent.to_owned(),
object_type: FileUsedForType::ChannelIcon,
},
uploader_id.to_owned(),
)
.await
}
/// Use a file for a server banner
pub async fn use_banner(db: &Database, id: &str, parent: &str) -> Result<File> {
db.find_and_use_attachment(id, "banners", "server", parent)
.await
pub async fn use_server_banner(
db: &Database,
id: &str,
parent: &str,
uploader_id: &str,
) -> Result<File> {
db.find_and_use_attachment(
id,
"banners",
FileUsedFor {
id: parent.to_owned(),
object_type: FileUsedForType::ServerBanner,
},
uploader_id.to_owned(),
)
.await
}
/// Use a file for an emoji
pub async fn use_emoji(db: &Database, id: &str, parent: &str) -> Result<File> {
db.find_and_use_attachment(id, "emojis", "object", parent)
.await
pub async fn use_emoji(
db: &Database,
id: &str,
parent: &str,
uploader_id: &str,
) -> Result<File> {
db.find_and_use_attachment(
id,
"emojis",
FileUsedFor {
id: parent.to_owned(),
object_type: FileUsedForType::Emoji,
},
uploader_id.to_owned(),
)
.await
}
}

View File

@@ -2,6 +2,8 @@ use revolt_result::Result;
use crate::File;
use super::FileUsedFor;
mod mongodb;
mod reference;
@@ -13,13 +15,22 @@ pub trait AbstractAttachments: Sync + Send {
/// Fetch an attachment by its id.
async fn fetch_attachment(&self, tag: &str, file_id: &str) -> Result<File>;
/// Fetch all deleted attachments.
async fn fetch_deleted_attachments(&self) -> Result<Vec<File>>;
/// Fetch all dangling attachments.
async fn fetch_dangling_files(&self) -> Result<Vec<File>>;
/// Count references to a given hash.
async fn count_file_hash_references(&self, hash: &str) -> Result<usize>;
/// Find an attachment by its details and mark it as used by a given parent.
async fn find_and_use_attachment(
&self,
id: &str,
tag: &str,
parent_type: &str,
parent_id: &str,
used_for: FileUsedFor,
uploader_id: String,
) -> Result<File>;
/// Mark an attachment as having been reported.
@@ -30,4 +41,7 @@ pub trait AbstractAttachments: Sync + Send {
/// Mark multiple attachments as having been deleted.
async fn mark_attachments_as_deleted(&self, ids: &[String]) -> Result<()>;
/// Delete the attachment entry.
async fn delete_attachment(&self, id: &str) -> Result<()>;
}

View File

@@ -1,7 +1,10 @@
use bson::to_document;
use bson::Document;
use revolt_config::report_internal_error;
use revolt_result::Result;
use crate::File;
use crate::FileUsedFor;
use crate::MongoDb;
use super::AbstractAttachments;
@@ -29,15 +32,59 @@ impl AbstractAttachments for MongoDb {
.ok_or_else(|| create_error!(NotFound))
}
/// Fetch all deleted attachments.
async fn fetch_deleted_attachments(&self) -> Result<Vec<File>> {
query!(
self,
find,
COL,
doc! {
"deleted": true,
"reported": {
"$ne": true
}
}
)
}
/// Fetch all dangling attachments.
async fn fetch_dangling_files(&self) -> Result<Vec<File>> {
query!(
self,
find,
COL,
doc! {
"used_for.type": {
"$exists": 0
},
"deleted": {
"$ne": true
}
}
)
}
/// Count references to a given hash.
async fn count_file_hash_references(&self, hash: &str) -> Result<usize> {
query!(
self,
count_documents,
COL,
doc! {
"hash": hash
}
)
.map(|count| count as usize)
}
/// Find an attachment by its details and mark it as used by a given parent.
async fn find_and_use_attachment(
&self,
id: &str,
tag: &str,
parent_type: &str,
parent_id: &str,
used_for: FileUsedFor,
uploader_id: String,
) -> Result<File> {
let key = format!("{parent_type}_id");
let file = query!(
self,
find_one,
@@ -45,7 +92,7 @@ impl AbstractAttachments for MongoDb {
doc! {
"_id": id,
"tag": tag,
&key: {
"used_for": {
"$exists": false
}
}
@@ -59,7 +106,8 @@ impl AbstractAttachments for MongoDb {
},
doc! {
"$set": {
key: parent_id
"used_for": report_internal_error!(to_document(&used_for))?,
"uploader_id": uploader_id
}
},
None,
@@ -111,7 +159,7 @@ impl AbstractAttachments for MongoDb {
/// Mark multiple attachments as having been deleted.
async fn mark_attachments_as_deleted(&self, ids: &[String]) -> Result<()> {
self.col::<Document>(COL)
.update_one(
.update_many(
doc! {
"_id": {
"$in": ids
@@ -126,7 +174,12 @@ impl AbstractAttachments for MongoDb {
)
.await
.map(|_| ())
.map_err(|_| create_database_error!("update_one", COL))
.map_err(|_| create_database_error!("update_many", COL))
}
/// Delete the attachment entry.
async fn delete_attachment(&self, id: &str) -> Result<()> {
query!(self, delete_one_by_id, COL, id).map(|_| ())
}
}

View File

@@ -1,6 +1,7 @@
use revolt_result::Result;
use crate::File;
use crate::FileUsedFor;
use crate::ReferenceDb;
use super::AbstractAttachments;
@@ -32,24 +33,54 @@ impl AbstractAttachments for ReferenceDb {
}
}
/// Fetch all deleted attachments.
async fn fetch_deleted_attachments(&self) -> Result<Vec<File>> {
let files = self.files.lock().await;
Ok(files
.values()
.filter(|file| {
// file has been marked as deleted
file.deleted.is_some_and(|v| v)
// and it has not been reported
&& !file.reported.is_some_and(|v| v)
})
.cloned()
.collect())
}
/// Fetch all dangling attachments.
async fn fetch_dangling_files(&self) -> Result<Vec<File>> {
let files = self.files.lock().await;
Ok(files
.values()
.filter(|file| file.used_for.is_none() && !file.deleted.is_some_and(|v| v))
.cloned()
.collect())
}
/// Count references to a given hash.
async fn count_file_hash_references(&self, hash: &str) -> Result<usize> {
let files = self.files.lock().await;
Ok(files
.values()
.filter(|file| file.hash.as_ref().is_some_and(|h| h == hash))
.cloned()
.count())
}
/// Find an attachment by its details and mark it as used by a given parent.
async fn find_and_use_attachment(
&self,
id: &str,
tag: &str,
parent_type: &str,
parent_id: &str,
used_for: FileUsedFor,
uploader_id: String,
) -> Result<File> {
let mut files = self.files.lock().await;
if let Some(file) = files.get_mut(id) {
if file.tag == tag {
match parent_type {
"message" => file.message_id = Some(parent_id.to_owned()),
"user" => file.user_id = Some(parent_id.to_owned()),
"object" => file.object_id = Some(parent_id.to_owned()),
"server" => file.server_id = Some(parent_id.to_owned()),
_ => unreachable!(),
}
file.uploader_id = Some(uploader_id);
file.used_for = Some(used_for);
Ok(file.clone())
} else {
@@ -100,4 +131,14 @@ impl AbstractAttachments for ReferenceDb {
Ok(())
}
/// Delete the attachment entry.
async fn delete_attachment(&self, id: &str) -> Result<()> {
let mut files = self.files.lock().await;
if files.remove(id).is_some() {
Ok(())
} else {
Err(create_error!(NotFound))
}
}
}

View File

@@ -1,4 +1,4 @@
use std::collections::HashSet;
use std::{collections::HashSet, hash::RandomState};
use indexmap::{IndexMap, IndexSet};
use iso8601_timestamp::Timestamp;
@@ -15,8 +15,8 @@ use validator::Validate;
use crate::{
events::client::EventV1,
tasks::{self, ack::AckEvent},
util::idempotency::IdempotencyKey,
Channel, Database, Emoji, File, User,
util::{bulk_permissions::BulkDatabasePermissionQuery, idempotency::IdempotencyKey},
Channel, Database, Emoji, File, User, AMQP,
};
auto_derived_partial!(
@@ -230,6 +230,7 @@ impl Message {
#[allow(clippy::too_many_arguments)]
pub async fn create_from_api(
db: &Database,
amqp: Option<&AMQP>,
channel: Channel,
data: DataMessageSend,
author: MessageAuthor<'_>,
@@ -337,6 +338,50 @@ impl Message {
}
}
// Validate the mentions go to users in the channel/server
if !mentions.is_empty() {
match channel {
Channel::DirectMessage { ref recipients, .. }
| Channel::Group { ref recipients, .. } => {
let recipients_hash: HashSet<&String, RandomState> =
HashSet::from_iter(recipients);
mentions.retain(|m| recipients_hash.contains(m));
}
Channel::TextChannel { ref server, .. }
| Channel::VoiceChannel { ref server, .. } => {
let mentions_vec = Vec::from_iter(mentions.iter().cloned());
let valid_members = db.fetch_members(server.as_str(), &mentions_vec[..]).await;
if let Ok(valid_members) = valid_members {
let valid_mentions: HashSet<&String, RandomState> =
HashSet::from_iter(valid_members.iter().map(|m| &m.id.user));
mentions.retain(|m| valid_mentions.contains(m)); // quick pass, validate mentions are in the server
if !mentions.is_empty() {
// if there are still mentions, drill down to a channel-level
let member_channel_view_perms =
BulkDatabasePermissionQuery::from_server_id(db, server)
.await
.channel(&channel)
.members(&valid_members)
.members_can_see_channel()
.await;
mentions
.retain(|m| *member_channel_view_perms.get(m).unwrap_or(&false));
}
} else {
revolt_config::capture_error(&valid_members.unwrap_err());
return Err(create_error!(InternalError));
}
}
Channel::SavedMessages { .. } => {
mentions.clear();
}
}
}
if !mentions.is_empty() {
message.mentions.replace(mentions.into_iter().collect());
}
@@ -370,10 +415,8 @@ impl Message {
}
for attachment_id in data.attachments.as_deref().unwrap_or_default() {
attachments.push(
db.find_and_use_attachment(attachment_id, "attachments", "message", &message_id)
.await?,
);
attachments
.push(File::use_attachment(db, attachment_id, &message_id, author.id()).await?);
}
if !attachments.is_empty() {
@@ -393,7 +436,7 @@ impl Message {
// Send the message
message
.send(db, author, user, member, &channel, generate_embeds)
.send(db, amqp, author, user, member, &channel, generate_embeds)
.await?;
Ok(message)
@@ -407,6 +450,9 @@ impl Message {
member: Option<v0::Member>,
is_dm: bool,
generate_embeds: bool,
// This determines if this function should queue the mentions task or if somewhere else will.
// If this is true, you MUST call tasks::ack::queue yourself.
mentions_elsewhere: bool,
) -> Result<()> {
db.insert_message(self).await?;
@@ -419,13 +465,12 @@ impl Message {
tasks::last_message_id::queue(self.channel.to_string(), self.id.to_string(), is_dm).await;
// Add mentions for affected users
if let Some(mentions) = &self.mentions {
for user in mentions {
tasks::ack::queue(
if !mentions_elsewhere {
if let Some(mentions) = &self.mentions {
tasks::ack::queue_message(
self.channel.to_string(),
user.to_string(),
AckEvent::AddMention {
ids: vec![self.id.to_string()],
AckEvent::ProcessMessage {
messages: vec![(None, self.clone(), mentions.clone(), true)],
},
)
.await;
@@ -448,9 +493,11 @@ impl Message {
}
/// Send a message
#[allow(clippy::too_many_arguments)]
pub async fn send(
&mut self,
db: &Database,
amqp: Option<&AMQP>, // this is optional mostly for tests.
author: MessageAuthor<'_>,
user: Option<v0::User>,
member: Option<v0::Member>,
@@ -463,26 +510,36 @@ impl Message {
member.clone(),
matches!(channel, Channel::DirectMessage { .. }),
generate_embeds,
true,
)
.await?;
if !self.has_suppressed_notifications() {
// Push out Web Push notifications
crate::tasks::web_push::queue(
{
match channel {
Channel::DirectMessage { recipients, .. }
| Channel::Group { recipients, .. } => recipients.clone(),
Channel::TextChannel { .. } => self.mentions.clone().unwrap_or_default(),
_ => vec![],
}
// send Push notifications
tasks::ack::queue_message(
self.channel.to_string(),
AckEvent::ProcessMessage {
messages: vec![(
Some(
PushNotification::from(
self.clone().into_model(user, member),
Some(author),
channel.to_owned().into(),
)
.await,
),
self.clone(),
match channel {
Channel::DirectMessage { recipients, .. }
| Channel::Group { recipients, .. } => recipients.clone(),
Channel::TextChannel { .. } => {
self.mentions.clone().unwrap_or_default()
}
_ => vec![],
},
self.has_suppressed_notifications(),
)],
},
PushNotification::from(
self.clone().into_model(user, member),
Some(author),
channel.id(),
)
.await,
)
.await;
}
@@ -499,10 +556,7 @@ impl Message {
})?;
let media = if let Some(id) = embed.media {
Some(
db.find_and_use_attachment(&id, "attachments", "message", &self.id)
.await?,
)
Some(File::use_attachment(db, &id, &self.id, &self.author).await?)
} else {
None
};
@@ -661,7 +715,7 @@ impl Message {
) -> Result<()> {
let media: Option<v0::File> = if let Some(id) = embed.media {
Some(
db.find_and_use_attachment(&id, "attachments", "message", &self.id)
File::use_attachment(db, &id, &self.id, &self.author)
.await?
.into(),
)

View File

@@ -95,7 +95,7 @@ impl AbstractMessages for MongoDb {
COL,
older_message_filter,
FindOptions::builder()
.limit(limit / 2)
.limit(limit / 2 + 1)
.sort(doc! {
"_id": -1_i32
})
@@ -171,7 +171,7 @@ impl AbstractMessages for MongoDb {
self.find_with_options(
COL,
doc! {
"ids": {
"_id": {
"$in": ids
}
},

View File

@@ -150,7 +150,7 @@ impl Member {
id: user.id.clone(),
}
.into_message(id.to_string())
.send_without_notifications(db, None, None, false, false)
.send_without_notifications(db, None, None, false, false, false)
.await
.ok();
}
@@ -251,7 +251,7 @@ impl Member {
}
.into_message(id.to_string())
// TODO: support notifications here in the future?
.send_without_notifications(db, None, None, false, false)
.send_without_notifications(db, None, None, false, false, false)
.await
.ok();
}

View File

@@ -53,17 +53,17 @@ impl AbstractServerMembers for ReferenceDb {
/// Fetch multiple members by their ids
async fn fetch_members<'a>(&self, server_id: &str, ids: &'a [String]) -> Result<Vec<Member>> {
let server_members = self.server_members.lock().await;
ids.iter()
.map(|id| {
Ok(ids
.iter()
.filter_map(|id| {
server_members
.get(&MemberCompositeKey {
server: server_id.to_string(),
user: id.to_string(),
})
.cloned()
.ok_or_else(|| create_error!(NotFound))
})
.collect()
.collect())
}
/// Fetch member count of a server

View File

@@ -260,7 +260,7 @@ impl MongoDb {
// Update many attachments with parent id.
self.delete_many_attachments(doc! {
"object_id": &server_id
"used_for.id": &server_id
})
.await?;

View File

@@ -0,0 +1,24 @@
use axum::{extract::FromRequestParts, http::request::Parts};
use revolt_result::{create_error, Error, Result};
use crate::{Database, User};
#[async_trait::async_trait]
impl FromRequestParts<Database> for User {
type Rejection = Error;
async fn from_request_parts(parts: &mut Parts, db: &Database) -> Result<User> {
if let Some(Ok(bot_token)) = parts.headers.get("x-bot-token").map(|v| v.to_str()) {
let bot = db.fetch_bot_by_token(bot_token).await?;
db.fetch_user(&bot.id).await
} else if let Some(Ok(session_token)) =
parts.headers.get("x-session-token").map(|v| v.to_str())
{
let session = db.fetch_session_by_token(session_token).await?;
db.fetch_user(&session.user_id).await
} else {
Err(create_error!(NotAuthenticated))
}
}
}

View File

@@ -1,3 +1,5 @@
#[cfg(feature = "axum-impl")]
mod axum;
mod model;
mod ops;
#[cfg(feature = "rocket-impl")]
@@ -5,6 +7,8 @@ mod rocket;
#[cfg(feature = "rocket-impl")]
mod schema;
#[cfg(feature = "axum-impl")]
pub use self::axum::*;
#[cfg(feature = "rocket-impl")]
pub use self::rocket::*;
#[cfg(feature = "rocket-impl")]

View File

@@ -1,6 +1,6 @@
use std::{collections::HashSet, str::FromStr, time::Duration};
use crate::{events::client::EventV1, Database, File, RatelimitEvent};
use crate::{events::client::EventV1, Database, File, RatelimitEvent, AMQP};
use authifier::config::{EmailVerificationConfig, Template};
use iso8601_timestamp::Timestamp;
@@ -152,7 +152,7 @@ pub static DISCRIMINATOR_SEARCH_SPACE: Lazy<HashSet<String>> = Lazy::new(|| {
.collect::<HashSet<String>>();
for discrim in [
123, 1234, 1111, 2222, 3333, 4444, 5555, 6666, 7777, 8888, 9999,
123, 1234, 1111, 2222, 3333, 4444, 5555, 6666, 7777, 8888, 9999, 1488,
] {
set.remove(&format!("{:0>4}", discrim));
}
@@ -218,7 +218,7 @@ impl User {
.datetime()
.elapsed()
.expect("time went backwards")
<= Duration::from_secs(86400u64 * config.features.limits.global.new_user_days as u64)
<= Duration::from_secs(3600u64 * config.features.limits.global.new_user_hours as u64)
{
config.features.limits.new_user
} else {
@@ -293,7 +293,14 @@ impl User {
}
// Ensure none of the following substrings show up in the username
const BLOCKED_SUBSTRINGS: &[&str] = &["```"];
const BLOCKED_SUBSTRINGS: &[&str] = &[
"```",
"discord.gg",
"rvlt.gg",
"guilded.gg",
"https://",
"http://",
];
for substr in BLOCKED_SUBSTRINGS {
if username_lowercase.contains(substr) {
@@ -497,7 +504,12 @@ impl User {
}
/// Add another user as a friend
pub async fn add_friend(&mut self, db: &Database, target: &mut User) -> Result<()> {
pub async fn add_friend(
&mut self,
db: &Database,
amqp: &AMQP,
target: &mut User,
) -> Result<()> {
match self.relationship_with(&target.id) {
RelationshipStatus::User => Err(create_error!(NoEffect)),
RelationshipStatus::Friend => Err(create_error!(AlreadyFriends)),
@@ -506,6 +518,8 @@ impl User {
RelationshipStatus::BlockedOther => Err(create_error!(BlockedByOther)),
RelationshipStatus::Incoming => {
// Accept incoming friend request
_ = amqp.friend_request_accepted(self, target).await;
self.apply_relationship(
db,
target,
@@ -534,6 +548,8 @@ impl User {
}));
}
_ = amqp.friend_request_received(target, self).await;
// Send the friend request
self.apply_relationship(
db,

View File

@@ -38,7 +38,7 @@ impl<'r> FromRequest<'r> for User {
if let Some(user) = user {
Outcome::Success(user.clone())
} else {
Outcome::Failure((Status::Unauthorized, authifier::Error::InvalidSession))
Outcome::Error((Status::Unauthorized, authifier::Error::InvalidSession))
}
}
}

View File

@@ -1,21 +1,27 @@
// Queue Type: Debounced
use crate::Database;
use crate::{Database, Message, AMQP};
use deadqueue::limited::Queue;
use once_cell::sync::Lazy;
use std::{collections::HashMap, time::Duration};
use revolt_models::v0::PushNotification;
use rocket::form::validate::Contains;
use std::{
collections::{HashMap, HashSet},
time::Duration,
};
use validator::HasLen;
use revolt_result::Result;
use super::{apple_notifications::{self, ApnJob}, DelayedTask};
use super::DelayedTask;
/// Enumeration of possible events
#[derive(Debug, Eq, PartialEq)]
pub enum AckEvent {
/// Add mentions for a user in a channel
AddMention {
/// Message IDs
ids: Vec<String>,
/// Add mentions for a channel
ProcessMessage {
/// push notification, message, recipients, push silenced
messages: Vec<(Option<PushNotification>, Message, Vec<String>, bool)>,
},
/// Acknowledge message in a channel for a user
@@ -30,7 +36,7 @@ struct Data {
/// Channel to ack
channel: String,
/// User to ack for
user: String,
user: Option<String>,
/// Event
event: AckEvent,
}
@@ -43,21 +49,49 @@ struct Task {
static Q: Lazy<Queue<Data>> = Lazy::new(|| Queue::new(10_000));
/// Queue a new task for a worker
pub async fn queue(channel: String, user: String, event: AckEvent) {
pub async fn queue_ack(channel: String, user: String, event: AckEvent) {
Q.try_push(Data {
channel,
user,
user: Some(user),
event,
})
.ok();
info!("Queue is using {} slots from {}.", Q.len(), Q.capacity());
info!(
"Queue is using {} slots from {}. Queued type: ACK",
Q.len(),
Q.capacity()
);
}
pub async fn handle_ack_event(event: &AckEvent, db: &Database, authifier_db: &authifier::Database, user: &str, channel: &str) -> Result<()> {
pub async fn queue_message(channel: String, event: AckEvent) {
Q.try_push(Data {
channel,
user: None,
event,
})
.ok();
info!(
"Queue is using {} slots from {}. Queued type: MENTION",
Q.len(),
Q.capacity()
);
}
pub async fn handle_ack_event(
event: &AckEvent,
db: &Database,
amqp: &AMQP,
user: &Option<String>,
channel: &str,
) -> Result<()> {
match &event {
#[allow(clippy::disallowed_methods)] // event is sent by higher level function
AckEvent::AckMessage { id } => {
let user = user.as_ref().unwrap();
let user: &str = user.as_str();
let unread = db.fetch_unread(user, channel).await?;
let updated = db.acknowledge_message(channel, user, id).await?;
@@ -68,21 +102,67 @@ pub async fn handle_ack_event(event: &AckEvent, db: &Database, authifier_db: &au
let mentions_acked = before_mentions - after_mentions;
if mentions_acked > 0 {
if let Ok(sessions) = authifier_db.find_sessions(user).await {
for session in sessions {
if let Some(sub) = session.subscription {
if sub.endpoint == "apn" {
apple_notifications::queue(ApnJob::from_ack(session.id, user.to_string(), sub.auth)).await;
}
}
}
if let Err(err) = amqp
.ack_message(user.to_string(), channel.to_string(), id.to_owned())
.await
{
revolt_config::capture_error(&err);
}
};
}
},
AckEvent::AddMention { ids } => {
db.add_mention_to_unread(channel, user, ids).await?;
}
AckEvent::ProcessMessage { messages } => {
let mut users: HashSet<&String> = HashSet::new();
debug!(
"Processing {} messages from channel {}",
messages.len(),
messages[0].1.channel
);
// find all the users we'll be notifying
messages
.iter()
.for_each(|(_, _, recipents, _)| users.extend(recipents.iter()));
debug!("Found {} users to notify.", users.len());
for user in users {
let message_ids: Vec<String> = messages
.iter()
.filter(|(_, _, recipients, _)| recipients.contains(user))
.map(|(_, message, _, _)| message.id.clone())
.collect();
if !message_ids.is_empty() {
db.add_mention_to_unread(channel, user, &message_ids)
.await?;
}
debug!("Added {} mentions for user {}", message_ids.len(), &user);
}
for (push, _, recipients, silenced) in messages {
if *silenced || recipients.is_empty() || push.is_none() {
debug!(
"Rejecting push: silenced: {}, recipient count: {}, push exists: {:?}",
*silenced,
recipients.length(),
push
);
continue;
}
debug!(
"Sending push event to AMQP; message {} for {} users",
push.as_ref().unwrap().message.id,
recipients.len()
);
if let Err(err) = amqp
.message_sent(recipients.clone(), push.clone().unwrap())
.await
{
revolt_config::capture_error(&err);
}
}
}
};
@@ -90,9 +170,9 @@ pub async fn handle_ack_event(event: &AckEvent, db: &Database, authifier_db: &au
}
/// Start a new worker
pub async fn worker(db: Database, authifier_db: authifier::Database) {
let mut tasks = HashMap::<(String, String), DelayedTask<Task>>::new();
let mut keys = vec![];
pub async fn worker(db: Database, amqp: AMQP) {
let mut tasks = HashMap::<(Option<String>, String, u8), DelayedTask<Task>>::new();
let mut keys: Vec<(Option<String>, String, u8)> = vec![];
loop {
// Find due tasks.
@@ -106,12 +186,13 @@ pub async fn worker(db: Database, authifier_db: authifier::Database) {
for key in &keys {
if let Some(task) = tasks.remove(key) {
let Task { event } = task.data;
let (user, channel) = key;
let (user, channel, _) = key;
if let Err(err) = handle_ack_event(&event, &db, &authifier_db, user, channel).await {
error!("{err:?} for {event:?}. ({user}, {channel})");
if let Err(err) = handle_ack_event(&event, &db, &amqp, user, channel).await {
revolt_config::capture_error(&err);
error!("{err:?} for {event:?}. ({user:?}, {channel})");
} else {
info!("User {user} ack in {channel} with {event:?}");
info!("User {user:?} ack in {channel} with {event:?}");
}
}
}
@@ -126,20 +207,41 @@ pub async fn worker(db: Database, authifier_db: authifier::Database) {
mut event,
}) = Q.try_pop()
{
let key = (user, channel);
let key: (Option<String>, String, u8) = (
user,
channel,
match &event {
AckEvent::AckMessage { .. } => 0,
AckEvent::ProcessMessage { .. } => 1,
},
);
if let Some(task) = tasks.get_mut(&key) {
task.delay();
match &mut event {
AckEvent::AddMention { ids } => {
if let AckEvent::AddMention { ids: existing } = &mut task.data.event {
existing.append(ids);
AckEvent::ProcessMessage { messages: new_data } => {
if let AckEvent::ProcessMessage { messages: existing } =
&mut task.data.event
{
// add the new message to the list of messages to be processed.
existing.append(new_data);
// put a cap on the amount of messages that can be queued, for particularly active channels
if (existing.length() as u16)
< revolt_config::config()
.await
.features
.advanced
.process_message_delay_limit
{
task.delay();
}
} else {
task.data.event = event;
panic!("Somehow got an ack message in the add mention arm");
}
}
AckEvent::AckMessage { .. } => {
// replace the last acked message with the new acked message
task.data.event = event;
task.delay();
}
}
} else {

View File

@@ -1,312 +0,0 @@
use std::io::Cursor;
use base64::{
engine::{self},
Engine as _,
};
use deadqueue::limited::Queue;
use once_cell::sync::Lazy;
use revolt_a2::{
request::{
notification::{DefaultAlert, NotificationOptions},
payload::{APSAlert, APSSound, PayloadLike, APS},
},
Client, ClientConfig, Endpoint, Error, ErrorBody, ErrorReason, Priority, PushType, Response,
};
use revolt_config::config;
use revolt_models::v0::{Message, PushNotification};
use crate::Database;
/// Payload information, before assembly
#[derive(Debug)]
pub struct ApnPayload {
message: Message,
url: String,
authorAvatar: String,
authorDisplayName: String,
channelName: String,
}
#[derive(Serialize, Debug)]
struct Payload<'a> {
aps: APS<'a>,
#[serde(skip_serializing)]
options: NotificationOptions<'a>,
#[serde(skip_serializing)]
device_token: &'a str,
message: &'a Message,
url: &'a str,
authorAvatar: &'a str,
authorDisplayName: &'a str,
channelName: &'a str,
}
impl<'a> PayloadLike for Payload<'a> {
fn get_device_token(&self) -> &'a str {
self.device_token
}
fn get_options(&self) -> &NotificationOptions {
&self.options
}
}
/// Task information
#[derive(Debug)]
pub struct AlertJob {
/// Session Id
session_id: String,
/// Device token
device_token: String,
/// User Id
user_id: String,
/// Title
title: String,
/// Body
body: String,
/// Thread Id
thread_id: String,
/// Category (informs the client what kind of notification is being sent.)
category: String,
/// Payload used by the iOS client to modify the notification
custom_payload: ApnPayload,
}
impl AlertJob {
fn format_title(notification: &PushNotification) -> String {
// ideally this changes depending on context
// in a server, it would look like "Sendername, #channelname in servername"
// in a group, it would look like "Sendername in groupname"
// in a dm it should just be "Sendername".
// not sure how feasible all those are given the PushNotification object as it currently stands.
format!(
"{} in {}",
notification.author, notification.message.channel
) // TODO: this absolutely needs a channel name
}
}
#[derive(Debug)]
pub struct BadgeJob {
/// Session Id
session_id: String,
/// Device token
device_token: String,
/// User Id
user_id: String,
}
#[derive(Debug)]
pub enum JobType {
Alert(AlertJob),
Badge(BadgeJob),
}
#[derive(Debug)]
pub struct ApnJob {
job_type: JobType,
}
impl ApnJob {
pub fn from_notification(
session_id: String,
user_id: String,
device_token: String,
notification: &PushNotification,
) -> ApnJob {
ApnJob {
job_type: JobType::Alert(AlertJob {
session_id,
device_token,
user_id,
title: AlertJob::format_title(notification),
body: notification.body.to_string(),
thread_id: notification.tag.to_string(),
category: "ALERT_MESSAGE".to_string(),
custom_payload: ApnPayload {
message: notification.message.clone(),
url: notification.url.clone(),
authorAvatar: notification.icon.clone(),
authorDisplayName: notification.author.clone(),
channelName: "#fetchchannelnamehere".to_string(), // TODO: get actual channel name
},
}),
}
}
pub fn from_ack(session_id: String, user_id: String, device_token: String) -> ApnJob {
ApnJob {
job_type: JobType::Badge(BadgeJob {
session_id,
device_token,
user_id,
}),
}
}
}
enum AssembledPayload<'a> {
Alert(Payload<'a>),
Default(revolt_a2::request::payload::Payload<'a>),
}
static Q: Lazy<Queue<ApnJob>> = Lazy::new(|| Queue::new(10_000));
/// Queue a new task for a worker
pub async fn queue(task: ApnJob) {
Q.try_push(task).ok();
info!("Queue is using {} slots from {}.", Q.len(), Q.capacity());
}
async fn get_badge_count(db: &Database, user: &str) -> Option<u32> {
if let Ok(unreads) = db.fetch_unreads(user).await {
let mut mention_count = 0;
for channel in unreads {
if let Some(mentions) = channel.mentions {
mention_count += mentions.len() as u32
}
}
return Some(mention_count);
}
None
}
/// Start a new worker
pub async fn worker(db: Database) {
let config = config().await;
if config.api.apn.pkcs8.is_empty()
|| config.api.apn.key_id.is_empty()
|| config.api.apn.team_id.is_empty()
{
eprintln!("Missing APN keys.");
return;
}
let endpoint = if config.api.apn.sandbox {
Endpoint::Sandbox
} else {
Endpoint::Production
};
let pkcs8 = engine::general_purpose::STANDARD
.decode(config.api.apn.pkcs8)
.expect("valid `pcks8`");
let client_config = ClientConfig::new(endpoint);
let client = Client::token(
&mut Cursor::new(pkcs8),
config.api.apn.key_id,
config.api.apn.team_id,
client_config,
)
.expect("could not create APN client");
let payload_options = NotificationOptions {
apns_id: None,
apns_push_type: Some(PushType::Alert),
apns_expiration: None,
apns_priority: Some(Priority::High),
apns_topic: Some("chat.revolt.app"),
apns_collapse_id: None,
};
loop {
let task = Q.pop().await;
let payload: AssembledPayload;
match task.job_type {
JobType::Alert(ref alert) => {
payload = AssembledPayload::Alert(Payload {
aps: APS {
alert: Some(APSAlert::Default(DefaultAlert {
title: Some(&alert.title),
subtitle: None,
body: Some(&alert.body),
title_loc_key: None,
title_loc_args: None,
action_loc_key: None,
loc_key: None,
loc_args: None,
launch_image: None,
})),
badge: get_badge_count(&db, &alert.user_id).await,
sound: Some(APSSound::Sound("default")),
thread_id: Some(&alert.thread_id),
content_available: None,
category: Some(&alert.category),
mutable_content: Some(1),
url_args: None,
},
device_token: &alert.device_token,
options: payload_options.clone(),
message: &alert.custom_payload.message,
url: &alert.custom_payload.url,
authorAvatar: &alert.custom_payload.authorAvatar,
authorDisplayName: &alert.custom_payload.authorDisplayName,
channelName: &alert.custom_payload.channelName,
});
}
JobType::Badge(ref alert) => {
payload = AssembledPayload::Default(revolt_a2::request::payload::Payload {
aps: APS {
alert: None,
badge: get_badge_count(&db, &alert.user_id).await,
sound: None,
thread_id: None,
content_available: None,
category: None,
mutable_content: None,
url_args: None,
},
device_token: &alert.device_token,
options: payload_options.clone(),
data: std::collections::BTreeMap::new(),
})
}
}
let resp = match payload {
AssembledPayload::Alert(p) => client.send(p).await,
AssembledPayload::Default(p) => client.send(p).await,
};
//println!("response from APNS: {:?}", resp);
if let Err(err) = resp {
match err {
Error::ResponseError(Response {
error:
Some(ErrorBody {
reason: ErrorReason::BadDeviceToken | ErrorReason::Unregistered,
..
}),
..
}) => {
if let Err(err) = db
.remove_push_subscription_by_session_id(match task.job_type {
JobType::Alert(ref a) => &a.session_id.as_str(),
JobType::Badge(ref a) => &a.session_id.as_str(),
})
.await
{
revolt_config::capture_error(&err);
}
}
err => {
revolt_config::capture_error(&err);
}
}
}
}
}

View File

@@ -1,6 +1,6 @@
//! Semi-important background task management
use crate::Database;
use crate::{Database, AMQP};
use async_std::task;
use std::time::Instant;
@@ -8,22 +8,18 @@ use std::time::Instant;
const WORKER_COUNT: usize = 5;
pub mod ack;
pub mod apple_notifications;
pub mod authifier_relay;
pub mod last_message_id;
pub mod process_embeds;
pub mod web_push;
/// Spawn background workers
pub fn start_workers(db: Database, authifier_db: authifier::Database) {
pub fn start_workers(db: Database, amqp: AMQP) {
task::spawn(authifier_relay::worker());
task::spawn(apple_notifications::worker(db.clone()));
for _ in 0..WORKER_COUNT {
task::spawn(ack::worker(db.clone(), authifier_db.clone()));
task::spawn(ack::worker(db.clone(), amqp.clone()));
task::spawn(last_message_id::worker(db.clone()));
task::spawn(process_embeds::worker(db.clone()));
task::spawn(web_push::worker(db.clone(), authifier_db.clone()));
}
}

View File

@@ -1,199 +0,0 @@
use std::{
collections::{HashMap, HashSet},
time::Duration,
};
use authifier::Database as AuthifierDatabase;
use base64::{
engine::{self},
Engine as _,
};
use deadqueue::limited::Queue;
use fcm_v1::auth::{Authenticator, ServiceAccountKey};
use once_cell::sync::Lazy;
use revolt_config::config;
use revolt_models::v0::PushNotification;
use revolt_presence::filter_online;
use serde_json::json;
use web_push::{
ContentEncoding, IsahcWebPushClient, SubscriptionInfo, SubscriptionKeys, VapidSignatureBuilder,
WebPushClient, WebPushMessageBuilder,
};
use crate::Database;
use super::apple_notifications;
/// Task information
#[derive(Debug)]
struct PushTask {
/// User IDs of the targets that are to receive this notification
recipients: Vec<String>,
/// Push Notification
payload: PushNotification,
}
static Q: Lazy<Queue<PushTask>> = Lazy::new(|| Queue::new(10_000));
/// Queue a new task for a worker
pub async fn queue(recipients: Vec<String>, payload: PushNotification) {
if recipients.is_empty() {
return;
}
let online_ids = filter_online(&recipients).await;
let recipients = (&recipients.into_iter().collect::<HashSet<String>>() - &online_ids)
.into_iter()
.collect::<Vec<String>>();
Q.try_push(PushTask {
recipients,
payload,
})
.ok();
info!("Queue is using {} slots from {}.", Q.len(), Q.capacity());
}
/// Start a new worker
pub async fn worker(db: Database, authifier_db: AuthifierDatabase) {
let config = config().await;
let web_push_client = IsahcWebPushClient::new().unwrap();
let fcm_client = if config.api.fcm.key_type.is_empty() {
None
} else {
Some(fcm_v1::Client::new(
Authenticator::service_account::<&str>(ServiceAccountKey {
key_type: Some(config.api.fcm.key_type),
project_id: Some(config.api.fcm.project_id.clone()),
private_key_id: Some(config.api.fcm.private_key_id),
private_key: config.api.fcm.private_key,
client_email: config.api.fcm.client_email,
client_id: Some(config.api.fcm.client_id),
auth_uri: Some(config.api.fcm.auth_uri),
token_uri: config.api.fcm.token_uri,
auth_provider_x509_cert_url: Some(config.api.fcm.auth_provider_x509_cert_url),
client_x509_cert_url: Some(config.api.fcm.client_x509_cert_url),
})
.await
.unwrap(),
config.api.fcm.project_id,
false,
Duration::from_secs(5),
))
};
let web_push_private_key = engine::general_purpose::URL_SAFE_NO_PAD
.decode(config.api.vapid.private_key)
.expect("valid `VAPID_PRIVATE_KEY`");
loop {
let task = Q.pop().await;
if let Ok(sessions) = authifier_db
.find_sessions_with_subscription(&task.recipients)
.await
{
for session in sessions {
if let Some(sub) = session.subscription {
if sub.endpoint == "fcm" {
// Use Firebase Cloud Messaging
if let Some(client) = &fcm_client {
let message = fcm_v1::message::Message {
token: Some(sub.auth),
data: Some(HashMap::from([(
"payload".to_owned(),
serde_json::Value::String(json!(&task.payload).to_string()),
)])),
..Default::default()
};
if let Err(err) = client.send(&message).await {
error!("Failed to send FCM notification! {:?}", err);
if let fcm_v1::Error::FCM(fcm_error) = err {
if fcm_error.contains("404 (Not Found)") {
println!("Unregistering {:?}", session.id);
if let Err(err) = db
.remove_push_subscription_by_session_id(&session.id)
.await
{
revolt_config::capture_error(&err);
}
}
}
} else {
info!("Sent FCM notification to {:?}.", session.id);
}
} else {
info!("No FCM token was specified!");
}
} else if sub.endpoint == "apn" {
apple_notifications::queue(apple_notifications::ApnJob::from_notification(
session.id,
session.user_id,
sub.auth,
&task.payload,
))
.await;
} else {
// Use Web Push Standard
let subscription = SubscriptionInfo {
endpoint: sub.endpoint,
keys: SubscriptionKeys {
auth: sub.auth,
p256dh: sub.p256dh,
},
};
match VapidSignatureBuilder::from_pem(
std::io::Cursor::new(&web_push_private_key),
&subscription,
) {
Ok(sig_builder) => match sig_builder.build() {
Ok(signature) => {
let mut builder = WebPushMessageBuilder::new(&subscription);
builder.set_vapid_signature(signature);
let payload = json!(task.payload).to_string();
builder
.set_payload(ContentEncoding::AesGcm, payload.as_bytes());
match builder.build() {
Ok(msg) => match web_push_client.send(msg).await {
Ok(_) => {
info!(
"Sent Web Push notification to {:?}.",
session.id
)
}
Err(err) => {
error!("Hit error sending Web Push! {:?}", err)
}
},
Err(err) => {
error!(
"Failed to build message for {}! {:?}",
session.user_id, err
)
}
}
}
Err(err) => error!(
"Failed to build signature for {}! {:?}",
session.user_id, err
),
},
Err(err) => error!(
"Failed to create signature builder for {}! {:?}",
session.user_id, err
),
}
}
}
}
}
}
}

View File

@@ -108,6 +108,7 @@ impl From<crate::Webhook> for Webhook {
id: value.id,
name: value.name,
avatar: value.avatar.map(|file| file.into()),
creator_id: value.creator_id,
channel_id: value.channel_id,
token: value.token,
permissions: value.permissions,
@@ -121,6 +122,7 @@ impl From<crate::PartialWebhook> for PartialWebhook {
id: value.id,
name: value.name,
avatar: value.avatar.map(|file| file.into()),
creator_id: value.creator_id,
channel_id: value.channel_id,
token: value.token,
permissions: value.permissions,
@@ -421,7 +423,7 @@ impl From<File> for crate::File {
object_id: value.object_id,
hash: None,
uploaded_at: None,
uploaded_id: None,
uploader_id: None,
used_for: None,
}
}

View File

@@ -0,0 +1,337 @@
use std::{collections::HashMap, hash::RandomState};
use revolt_permissions::{
ChannelPermission, ChannelType, Override, OverrideField, PermissionValue, ALLOW_IN_TIMEOUT,
DEFAULT_PERMISSION_DIRECT_MESSAGE,
};
use crate::{Channel, Database, Member, Server, User};
#[derive(Clone)]
pub struct BulkDatabasePermissionQuery<'a> {
#[allow(dead_code)]
database: &'a Database,
server: Server,
channel: Option<Channel>,
users: Option<Vec<User>>,
members: Option<Vec<Member>>,
// In case the users or members are fetched as part of the permissions checking operation
pub(crate) cached_users: Option<Vec<User>>,
pub(crate) cached_members: Option<Vec<Member>>,
cached_member_perms: Option<HashMap<String, PermissionValue>>,
}
impl<'z, 'x> BulkDatabasePermissionQuery<'x> {
pub async fn members_can_see_channel(&'z mut self) -> HashMap<String, bool>
where
'z: 'x,
{
let member_perms = if self.cached_member_perms.is_some() {
// This isn't done as an if let to prevent borrow checker errors with the mut self call when the perms aren't cached.
let perms = self.cached_member_perms.as_ref().unwrap();
perms
.iter()
.map(|(m, p)| {
(
m.clone(),
p.has_channel_permission(ChannelPermission::ViewChannel),
)
})
.collect()
} else {
calculate_members_permissions(self)
.await
.iter()
.map(|(m, p)| {
(
m.clone(),
p.has_channel_permission(ChannelPermission::ViewChannel),
)
})
.collect()
};
member_perms
}
}
impl<'z> BulkDatabasePermissionQuery<'z> {
pub fn new(database: &Database, server: Server) -> BulkDatabasePermissionQuery<'_> {
BulkDatabasePermissionQuery {
database,
server,
channel: None,
users: None,
members: None,
cached_members: None,
cached_users: None,
cached_member_perms: None,
}
}
pub async fn from_server_id<'a>(
db: &'a Database,
server: &str,
) -> BulkDatabasePermissionQuery<'a> {
BulkDatabasePermissionQuery {
database: db,
server: db.fetch_server(server).await.unwrap(),
channel: None,
users: None,
members: None,
cached_members: None,
cached_users: None,
cached_member_perms: None,
}
}
pub fn channel(self, channel: &'z Channel) -> BulkDatabasePermissionQuery {
BulkDatabasePermissionQuery {
channel: Some(channel.clone()),
..self
}
}
pub fn members(self, members: &'z [Member]) -> BulkDatabasePermissionQuery {
BulkDatabasePermissionQuery {
members: Some(members.to_owned()),
..self
}
}
pub fn users(self, users: &'z [User]) -> BulkDatabasePermissionQuery {
BulkDatabasePermissionQuery {
users: Some(users.to_owned()),
..self
}
}
/// Get the default channel permissions
/// Group channel defaults should be mapped to an allow-only override
#[allow(dead_code)]
async fn get_default_channel_permissions(&mut self) -> Override {
if let Some(channel) = &self.channel {
match channel {
Channel::Group { permissions, .. } => Override {
allow: permissions.unwrap_or(*DEFAULT_PERMISSION_DIRECT_MESSAGE as i64) as u64,
deny: 0,
},
Channel::TextChannel {
default_permissions,
..
}
| Channel::VoiceChannel {
default_permissions,
..
} => default_permissions.unwrap_or_default().into(),
_ => Default::default(),
}
} else {
Default::default()
}
}
#[allow(dead_code)]
fn get_channel_type(&mut self) -> ChannelType {
if let Some(channel) = &self.channel {
match channel {
Channel::DirectMessage { .. } => ChannelType::DirectMessage,
Channel::Group { .. } => ChannelType::Group,
Channel::SavedMessages { .. } => ChannelType::SavedMessages,
Channel::TextChannel { .. } | Channel::VoiceChannel { .. } => {
ChannelType::ServerChannel
}
}
} else {
ChannelType::Unknown
}
}
/// Get the ordered role overrides (from lowest to highest) for this member in this channel
#[allow(dead_code)]
async fn get_channel_role_overrides(&mut self) -> &HashMap<String, OverrideField> {
if let Some(channel) = &self.channel {
match channel {
Channel::TextChannel {
role_permissions, ..
}
| Channel::VoiceChannel {
role_permissions, ..
} => role_permissions,
_ => panic!("Not supported for non-server channels"),
}
} else {
panic!("No channel added to query")
}
}
}
/// Calculate members permissions in a server channel.
async fn calculate_members_permissions<'a>(
query: &'a mut BulkDatabasePermissionQuery<'a>,
) -> HashMap<String, PermissionValue> {
let mut resp = HashMap::new();
let (_, channel_role_permissions, channel_default_permissions) = match query
.channel
.as_ref()
.expect("A channel must be assigned to calculate channel permissions")
.clone()
{
Channel::TextChannel {
id,
role_permissions,
default_permissions,
..
}
| Channel::VoiceChannel {
id,
role_permissions,
default_permissions,
..
} => (id, role_permissions, default_permissions),
_ => panic!("Calculation of member permissions must be done on a server channel"),
};
if query.users.is_none() {
let ids: Vec<String> = query
.members
.as_ref()
.expect("No users or members added to the query")
.iter()
.map(|m| m.id.user.clone())
.collect();
query.cached_users = Some(
query
.database
.fetch_users(&ids[..])
.await
.expect("Failed to get data from the db"),
);
query.users = Some(query.cached_users.as_ref().unwrap().to_vec())
}
let users = query.users.as_ref().unwrap();
if query.members.is_none() {
let ids: Vec<String> = query
.users
.as_ref()
.expect("No users or members added to the query")
.iter()
.map(|m| m.id.clone())
.collect();
query.cached_members = Some(
query
.database
.fetch_members(&query.server.id, &ids[..])
.await
.expect("Failed to get data from the db"),
);
query.members = Some(query.cached_members.as_ref().unwrap().to_vec())
}
let members: HashMap<&String, &Member, RandomState> = HashMap::from_iter(
query
.members
.as_ref()
.unwrap()
.iter()
.map(|m| (&m.id.user, m)),
);
for user in users {
let member = members.get(&user.id);
// User isn't a part of the server
if member.is_none() {
resp.insert(user.id.clone(), 0_u64.into());
continue;
}
let member = *member.unwrap();
if user.privileged {
resp.insert(
user.id.clone(),
PermissionValue::from(ChannelPermission::GrantAllSafe),
);
continue;
}
if user.id == query.server.owner {
resp.insert(
user.id.clone(),
PermissionValue::from(ChannelPermission::GrantAllSafe),
);
continue;
}
// Get the user's server permissions
let mut permission = calculate_server_permissions(&query.server, user, member);
if let Some(defaults) = channel_default_permissions {
permission.apply(defaults.into());
}
// Get the applicable role overrides
let mut roles = channel_role_permissions
.iter()
.filter(|(id, _)| member.roles.contains(id))
.filter_map(|(id, permission)| {
query.server.roles.get(id).map(|role| {
let v: Override = (*permission).into();
(role.rank, v)
})
})
.collect::<Vec<(i64, Override)>>();
roles.sort_by(|a, b| b.0.cmp(&a.0));
let overrides = roles.into_iter().map(|(_, v)| v);
for role_override in overrides {
permission.apply(role_override)
}
resp.insert(user.id.clone(), permission);
}
resp
}
/// Calculates a member's server permissions
fn calculate_server_permissions(server: &Server, user: &User, member: &Member) -> PermissionValue {
if user.privileged || server.owner == user.id {
return ChannelPermission::GrantAllSafe.into();
}
let mut permissions: PermissionValue = server.default_permissions.into();
let mut roles = server
.roles
.iter()
.filter(|(id, _)| member.roles.contains(id))
.map(|(_, role)| {
let v: Override = role.permissions.into();
(role.rank, v)
})
.collect::<Vec<(i64, Override)>>();
roles.sort_by(|a, b| b.0.cmp(&a.0));
let role_overrides: Vec<Override> = roles.into_iter().map(|(_, v)| v).collect();
for role in role_overrides {
permissions.apply(role);
}
if member.in_timeout() {
permissions.restrict(*ALLOW_IN_TIMEOUT);
}
permissions
}

View File

@@ -102,7 +102,7 @@ impl<'r> FromRequest<'r> for IdempotencyKey {
.map(|k| k.to_string())
{
if key.len() > 64 {
return Outcome::Failure((
return Outcome::Error((
Status::BadRequest,
create_error!(FailedValidation {
error: "idempotency key too long".to_string(),
@@ -113,7 +113,7 @@ impl<'r> FromRequest<'r> for IdempotencyKey {
let idempotency = IdempotencyKey { key };
let mut cache = TOKEN_CACHE.lock().await;
if cache.get(&idempotency.key).is_some() {
return Outcome::Failure((Status::Conflict, create_error!(DuplicateNonce)));
return Outcome::Error((Status::Conflict, create_error!(DuplicateNonce)));
}
cache.put(idempotency.key.clone(), ());

View File

@@ -1,4 +1,5 @@
pub mod bridge;
pub mod bulk_permissions;
pub mod idempotency;
pub mod permissions;
pub mod reference;

View File

@@ -1,12 +1,18 @@
[package]
name = "revolt-files"
version = "0.7.16"
version = "0.8.2"
edition = "2021"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <me@insrt.uk>"]
description = "Revolt Backend: S3 and encryption subroutines"
[dependencies]
tracing = "0.1"
ffprobe = "0.4.0"
imagesize = "0.13.0"
tempfile = "3.12.0"
base64 = "0.22.1"
aes-gcm = "0.10.3"
typenum = "1.17.0"
@@ -14,5 +20,19 @@ typenum = "1.17.0"
aws-config = "1.5.5"
aws-sdk-s3 = { version = "1.46.0", features = ["behavior-version-latest"] }
revolt-config = { version = "0.7.16", path = "../config" }
revolt-result = { version = "0.7.16", path = "../result" }
revolt-config = { version = "0.8.2", path = "../config", features = [
"report-macros",
] }
revolt-result = { version = "0.8.2", path = "../result" }
# image processing
jxl-oxide = "0.8.1"
image = { version = "0.25.2" }
# svg rendering
usvg = "0.44.0"
resvg = "0.44.0"
tiny-skia = "0.11.4"
# encoding
webp = "0.3.0"

View File

@@ -1,7 +1,11 @@
use std::io::Write;
use std::io::{BufRead, Read, Seek, Write};
use aes_gcm::{aead::AeadMutInPlace, Aes256Gcm, Key, KeyInit, Nonce};
use revolt_config::{config, FilesS3};
use aes_gcm::{
aead::{AeadCore, AeadMutInPlace, OsRng},
Aes256Gcm, Key, KeyInit, Nonce,
};
use image::{DynamicImage, ImageBuffer};
use revolt_config::{config, report_internal_error, FilesS3};
use revolt_result::{create_error, Result};
use aws_sdk_s3::{
@@ -10,8 +14,11 @@ use aws_sdk_s3::{
};
use base64::prelude::*;
use tempfile::NamedTempFile;
use tiny_skia::Pixmap;
pub static AUTHENTICATION_TAG_SIZE_BYTES: usize = 16;
/// Size of the authentication tag in the buffer
pub const AUTHENTICATION_TAG_SIZE_BYTES: usize = 16;
/// Create an S3 client
pub fn create_client(s3_config: FilesS3) -> Client {
@@ -27,6 +34,7 @@ pub fn create_client(s3_config: FilesS3) -> Client {
let config = Config::builder()
.region(Region::new(s3_config.region))
.endpoint_url(s3_config.endpoint)
.force_path_style(s3_config.path_style_buckets)
.credentials_provider(creds)
.build();
@@ -35,7 +43,7 @@ pub fn create_client(s3_config: FilesS3) -> Client {
/// Create an AES-256-GCM cipher
pub fn create_cipher(key: &str) -> Aes256Gcm {
let key = &BASE64_STANDARD.decode(key).unwrap()[..];
let key = &BASE64_STANDARD.decode(key).expect("valid base64 string")[..];
let key: &Key<Aes256Gcm> = key.into();
Aes256Gcm::new(key)
}
@@ -46,27 +54,23 @@ pub async fn fetch_from_s3(bucket_id: &str, path: &str, nonce: &str) -> Result<V
let client = create_client(config.files.s3);
// Send a request for the file
let mut obj = client
.get_object()
.bucket(bucket_id)
.key(path)
.send()
.await
.inspect_err(|err| {
revolt_config::capture_error(err);
})
.map_err(|_| create_error!(InternalError))?;
let mut obj =
report_internal_error!(client.get_object().bucket(bucket_id).key(path).send().await)?;
// Read the file from remote
let mut buf = vec![];
while let Some(bytes) = obj.body.next().await {
let data = bytes.map_err(|_| create_error!(InternalError))?;
buf.write_all(&data)
.map_err(|_| create_error!(InternalError))?;
let data = report_internal_error!(bytes)?;
report_internal_error!(buf.write_all(&data))?;
// is there a more efficient way to do this?
// we just want the Vec<u8>
}
// File is not encrypted
if nonce.is_empty() {
return Ok(buf);
}
// Recover nonce as bytes
let nonce = &BASE64_STANDARD.decode(nonce).unwrap()[..];
let nonce: &Nonce<typenum::consts::U12> = nonce.into();
@@ -78,3 +82,209 @@ pub async fn fetch_from_s3(bucket_id: &str, path: &str, nonce: &str) -> Result<V
Ok(buf)
}
/// Encrypt and upload a file to S3 (returning its nonce/IV)
pub async fn upload_to_s3(bucket_id: &str, path: &str, buf: &[u8]) -> Result<String> {
let config = config().await;
let client = create_client(config.files.s3);
// Generate a nonce
let nonce = Aes256Gcm::generate_nonce(&mut OsRng);
// Extend the buffer for in-place encryption
let mut buf = [buf, &[0; AUTHENTICATION_TAG_SIZE_BYTES]].concat();
// Encrypt the file in place
create_cipher(&config.files.encryption_key)
.encrypt_in_place(&nonce, b"", &mut buf)
.map_err(|_| create_error!(InternalError))?;
// Upload the file to remote
report_internal_error!(
client
.put_object()
.bucket(bucket_id)
.key(path)
.body(buf.into())
.send()
.await
)?;
Ok(BASE64_STANDARD.encode(nonce))
}
/// Delete a file from S3 by path
pub async fn delete_from_s3(bucket_id: &str, path: &str) -> Result<()> {
let config = config().await;
let client = create_client(config.files.s3);
report_internal_error!(
client
.delete_object()
.bucket(bucket_id)
.key(path)
.send()
.await
)?;
Ok(())
}
/// Determine size of image at temp file
pub fn image_size(f: &NamedTempFile) -> Option<(usize, usize)> {
if let Ok(size) = imagesize::size(f.path())
.inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}"))
{
Some((size.width, size.height))
} else {
None
}
}
/// Determine size of image with buffer
pub fn image_size_vec(v: &[u8], mime: &str) -> Option<(usize, usize)> {
match mime {
"image/svg+xml" => {
let tree =
report_internal_error!(usvg::Tree::from_data(v, &Default::default())).ok()?;
let size = tree.size();
Some((size.width() as usize, size.height() as usize))
}
_ => {
if let Ok(size) = imagesize::blob_size(v)
.inspect_err(|err| tracing::error!("Failed to generate image size! {err:?}"))
{
Some((size.width, size.height))
} else {
None
}
}
}
}
/// Determine size of video at temp file
pub fn video_size(f: &NamedTempFile) -> Option<(i64, i64)> {
if let Ok(data) = ffprobe::ffprobe(f.path())
.inspect_err(|err| tracing::error!("Failed to ffprobe file! {err:?}"))
{
// Use first valid stream
for stream in data.streams {
if let (Some(w), Some(h)) = (stream.width, stream.height) {
return Some((w, h));
}
}
None
} else {
None
}
}
/// Decode image from reader
pub fn decode_image<R: Read + BufRead + Seek>(reader: &mut R, mime: &str) -> Result<DynamicImage> {
match mime {
// Read image using jxl-oxide crate
"image/jxl" => {
let jxl_image = report_internal_error!(jxl_oxide::JxlImage::builder().read(reader))?;
if let Ok(frame) = jxl_image.render_frame(0) {
match frame.color_channels().len() {
3 => Ok(DynamicImage::ImageRgb8(
DynamicImage::ImageRgb32F(
ImageBuffer::from_vec(
jxl_image.width(),
jxl_image.height(),
frame.image().buf().to_vec(),
)
.ok_or_else(|| create_error!(ImageProcessingFailed))?,
)
.to_rgb8(),
)),
4 => Ok(DynamicImage::ImageRgba8(
DynamicImage::ImageRgba32F(
ImageBuffer::from_vec(
jxl_image.width(),
jxl_image.height(),
frame.image().buf().to_vec(),
)
.ok_or_else(|| create_error!(ImageProcessingFailed))?,
)
.to_rgba8(),
)),
_ => Err(create_error!(ImageProcessingFailed)),
}
} else {
Err(create_error!(ImageProcessingFailed))
}
}
// Read image using resvg
"image/svg+xml" => {
// usvg doesn't support Read trait so copy to buffer
let mut buf = Vec::new();
report_internal_error!(reader.read_to_end(&mut buf))?;
let tree = report_internal_error!(usvg::Tree::from_data(&buf, &Default::default()))?;
let size = tree.size();
let mut pixmap = Pixmap::new(size.width() as u32, size.height() as u32)
.ok_or_else(|| create_error!(ImageProcessingFailed))?;
let mut pixmap_mut = pixmap.as_mut();
resvg::render(&tree, Default::default(), &mut pixmap_mut);
Ok(DynamicImage::ImageRgba8(
ImageBuffer::from_vec(
size.width() as u32,
size.height() as u32,
pixmap.data().to_vec(),
)
.ok_or_else(|| create_error!(ImageProcessingFailed))?,
))
}
// Check if we can read using image-rs crate
_ => report_internal_error!(report_internal_error!(
image::ImageReader::new(reader).with_guessed_format()
)?
.decode()),
}
}
/// Check whether given reader has a valid image
pub fn is_valid_image<R: Read + BufRead + Seek>(reader: &mut R, mime: &str) -> bool {
match mime {
// Check if we can read using jxl-oxide crate
"image/jxl" => jxl_oxide::JxlImage::builder()
.read(reader)
.inspect_err(|err| tracing::error!("Failed to read JXL! {err:?}"))
.is_ok(),
// Check if we can read using image-rs crate
_ => !matches!(
image::ImageReader::new(reader)
.with_guessed_format()
.inspect_err(|err| tracing::error!("Failed to read image! {err:?}"))
.map(|f| f.decode()),
Err(_) | Ok(Err(_))
),
}
}
/// Create thumbnail from given image
pub async fn create_thumbnail(image: DynamicImage, tag: &str) -> Vec<u8> {
// Load configuration
let config = config().await;
let [w, h] = config.files.preview.get(tag).unwrap();
// Create thumbnail
//.resize(width as u32, height as u32, image::imageops::FilterType::Gaussian)
// resize is about 2.5x slower,
// thumbnail doesn't have terrible quality
// so we use thumbnail
let image = image.thumbnail(image.width().min(*w as u32), image.height().min(*h as u32));
// Encode it into WEBP
let encoder = webp::Encoder::from_image(&image).expect("Could not create encoder.");
if config.files.webp_quality != 100.0 {
encoder.encode(config.files.webp_quality).to_vec()
} else {
encoder.encode_lossless().to_vec()
}
}

View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-models"
version = "0.7.16"
version = "0.8.2"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -20,8 +20,8 @@ default = ["serde", "partials", "rocket"]
[dependencies]
# Core
revolt-config = { version = "0.7.16", path = "../config" }
revolt-permissions = { version = "0.7.16", path = "../permissions" }
revolt-config = { version = "0.8.2", path = "../config" }
revolt-permissions = { version = "0.8.2", path = "../permissions" }
# Utility
regex = "1"

View File

@@ -16,6 +16,9 @@ auto_derived_partial!(
#[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
pub avatar: Option<File>,
/// User that created this webhook
pub creator_id: String,
/// The channel this webhook belongs to
pub channel_id: String,

View File

@@ -306,4 +306,18 @@ impl Channel {
| Channel::VoiceChannel { id, .. } => id,
}
}
/// This returns a Result because the recipient name can't be determined here without a db call,
/// which can't be done since this is models, which can't reference the database crate.
///
/// If it returns Err, you need to fetch the name from the db.
pub fn name(&self) -> Result<&str, ()> {
match self {
Channel::DirectMessage { .. } => Err(()),
Channel::SavedMessages { .. } => Ok("Saved Messages"),
Channel::TextChannel { name, .. }
| Channel::Group { name, .. }
| Channel::VoiceChannel { name, .. } => Ok(name),
}
}
}

View File

@@ -14,9 +14,9 @@ auto_derived!(
/// URL to the original image
pub url: String,
/// Width of the image
pub width: isize,
pub width: usize,
/// Height of the image
pub height: isize,
pub height: usize,
/// Positioning and size
pub size: ImageSize,
}
@@ -26,9 +26,9 @@ auto_derived!(
/// URL to the original video
pub url: String,
/// Width of the video
pub width: isize,
pub width: usize,
/// Height of the video
pub height: isize,
pub height: usize,
}
/// Type of remote Twitch content
@@ -86,7 +86,7 @@ auto_derived!(
},
AppleMusic {
album_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
track_id: Option<String>,
},
@@ -119,8 +119,6 @@ auto_derived!(
#[serde(skip_serializing_if = "Option::is_none")]
pub video: Option<Video>,
// #[serde(skip_serializing_if = "Option::is_none")]
// opengraph_type: Option<String>,
/// Site name
#[serde(skip_serializing_if = "Option::is_none")]
pub site_name: Option<String>,
@@ -156,11 +154,56 @@ auto_derived!(
/// Embed
#[serde(tag = "type")]
#[derive(Default)]
pub enum Embed {
Website(WebsiteMetadata),
Image(Image),
Video(Video),
Text(Text),
#[default]
None,
}
);
impl WebsiteMetadata {
/// Truncate strings in metadata
pub fn truncate(&mut self) {
if let Some(s) = self.url.as_mut() {
s.truncate(256);
}
if let Some(s) = self.original_url.as_mut() {
s.truncate(256);
}
if let Some(s) = self.title.as_mut() {
s.truncate(100);
}
if let Some(s) = self.description.as_mut() {
s.truncate(1000);
}
if let Some(s) = self.site_name.as_mut() {
s.truncate(32);
}
if let Some(s) = self.icon_url.as_mut() {
s.truncate(256);
}
if let Some(s) = self.colour.as_mut() {
s.truncate(32);
}
}
/// Check if this is considered "empty"
pub fn is_empty(&self) -> bool {
(self.title.is_none() || self.title.as_ref().is_some_and(|f| f.is_empty()))
&& (self.description.is_none()
|| self.description.as_ref().is_some_and(|f| f.is_empty()))
&& self.special.is_none()
&& self.video.is_none()
&& self.image.is_none()
}
}

View File

@@ -13,7 +13,7 @@ use rocket::{FromForm, FromFormField};
use iso8601_timestamp::Timestamp;
use super::{Embed, File, Member, MessageWebhook, User, Webhook, RE_COLOUR};
use super::{Channel, Embed, File, Member, MessageWebhook, User, Webhook, RE_COLOUR};
pub static RE_MENTION: Lazy<Regex> =
Lazy::new(|| Regex::new(r"<@([0-9A-HJKMNP-TV-Z]{26})>").unwrap());
@@ -209,6 +209,8 @@ auto_derived!(
pub url: String,
/// The message object itself, to send to clients for processing
pub message: Message,
/// The channel object itself, for clients to process
pub channel: Channel,
}
/// Representation of a text embed before it is sent.
@@ -278,7 +280,7 @@ auto_derived!(
pub struct OptionsQueryMessages {
/// Maximum number of messages to fetch
///
/// For fetching nearby messages, this is \`(limit + 1)\`.
/// For fetching nearby messages, this is \`(limit + 2)\`.
#[cfg_attr(feature = "validator", validate(range(min = 1, max = 100)))]
pub limit: Option<i64>,
/// Message id before which messages should be fetched
@@ -369,7 +371,7 @@ auto_derived!(
/// Optional fields on message
pub enum FieldsMessage {
Pinned
Pinned,
}
);
@@ -442,7 +444,7 @@ impl From<SystemMessage> for String {
impl PushNotification {
/// Create a new notification from a given message, author and channel ID
pub async fn from(msg: Message, author: Option<MessageAuthor<'_>>, channel_id: &str) -> Self {
pub async fn from(msg: Message, author: Option<MessageAuthor<'_>>, channel: Channel) -> Self {
let config = config().await;
let icon = if let Some(author) = &author {
@@ -496,10 +498,11 @@ impl PushNotification {
icon,
image,
body,
tag: channel_id.to_string(),
tag: channel.id().to_string(),
timestamp,
url: format!("{}/channel/{}/{}", config.hosts.app, channel_id, msg.id),
url: format!("{}/channel/{}/{}", config.hosts.app, channel.id(), msg.id),
message: msg,
channel,
}
}
}

View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-permissions"
version = "0.7.16"
version = "0.8.2"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -21,7 +21,7 @@ async-std = { version = "1.8.0", features = ["attributes"] }
[dependencies]
# Core
revolt-result = { version = "0.7.16", path = "../result" }
revolt-result = { version = "0.8.2", path = "../result" }
# Utility
auto_ops = "0.3.0"

View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-presence"
version = "0.7.16"
version = "0.8.2"
edition = "2021"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <me@insrt.uk>"]

View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-result"
version = "0.7.16"
version = "0.8.2"
edition = "2021"
license = "MIT"
authors = ["Paul Makles <me@insrt.uk>"]
@@ -29,7 +29,7 @@ utoipa = { version = "4.2.3", optional = true }
# Rocket
rocket = { optional = true, version = "0.5.0-rc.2", default-features = false }
revolt_rocket_okapi = { version = "0.9.1", optional = true }
revolt_rocket_okapi = { version = "0.10.0", optional = true }
revolt_okapi = { version = "0.9.1", optional = true }
# Axum

View File

@@ -1,4 +1,5 @@
use axum::{http::StatusCode, response::IntoResponse, Json};
use rocket::http::Status;
use crate::{Error, ErrorType};
@@ -41,6 +42,7 @@ impl IntoResponse for Error {
ErrorType::InvalidRole => StatusCode::NOT_FOUND,
ErrorType::Banned => StatusCode::FORBIDDEN,
ErrorType::AlreadyInServer => StatusCode::CONFLICT,
ErrorType::CannotTimeoutYourself => StatusCode::BAD_REQUEST,
ErrorType::TooManyServers { .. } => StatusCode::BAD_REQUEST,
ErrorType::TooManyEmbeds { .. } => StatusCode::BAD_REQUEST,
@@ -67,6 +69,7 @@ impl IntoResponse for Error {
ErrorType::InvalidCredentials => StatusCode::UNAUTHORIZED,
ErrorType::InvalidProperty => StatusCode::BAD_REQUEST,
ErrorType::InvalidSession => StatusCode::UNAUTHORIZED,
ErrorType::NotAuthenticated => StatusCode::UNAUTHORIZED,
ErrorType::DuplicateNonce => StatusCode::CONFLICT,
ErrorType::VosoUnavailable => StatusCode::BAD_REQUEST,
ErrorType::NotFound => StatusCode::NOT_FOUND,
@@ -74,6 +77,11 @@ impl IntoResponse for Error {
ErrorType::FailedValidation { .. } => StatusCode::BAD_REQUEST,
ErrorType::ProxyError => StatusCode::BAD_REQUEST,
ErrorType::FileTooSmall => StatusCode::UNPROCESSABLE_ENTITY,
ErrorType::FileTooLarge { .. } => StatusCode::UNPROCESSABLE_ENTITY,
ErrorType::FileTypeNotAllowed => StatusCode::BAD_REQUEST,
ErrorType::ImageProcessingFailed => StatusCode::INTERNAL_SERVER_ERROR,
ErrorType::NoEmbedData => StatusCode::BAD_REQUEST,
};
(status, Json(&self)).into_response()

View File

@@ -116,6 +116,7 @@ pub enum ErrorType {
max: usize,
},
AlreadyInServer,
CannotTimeoutYourself,
// ? Bot related errors
ReachedMaximumBots,
@@ -147,6 +148,7 @@ pub enum ErrorType {
InvalidCredentials,
InvalidProperty,
InvalidSession,
NotAuthenticated,
DuplicateNonce,
NotFound,
NoEffect,
@@ -156,6 +158,13 @@ pub enum ErrorType {
// ? Micro-service errors
ProxyError,
FileTooSmall,
FileTooLarge {
max: usize,
},
FileTypeNotAllowed,
ImageProcessingFailed,
NoEmbedData,
// ? Legacy errors
VosoUnavailable,
@@ -181,23 +190,6 @@ macro_rules! create_database_error {
};
}
#[macro_export]
#[cfg(debug_assertions)]
macro_rules! query {
( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
Ok($self.$type($collection, $($rest),+).await.unwrap())
};
}
#[macro_export]
#[cfg(not(debug_assertions))]
macro_rules! query {
( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
$self.$type($collection, $($rest),+).await
.map_err(|_| create_database_error!(stringify!($type), $collection))
};
}
#[cfg(test)]
mod tests {
use crate::ErrorType;

View File

@@ -47,6 +47,7 @@ impl<'r> Responder<'r, 'static> for Error {
ErrorType::InvalidRole => Status::NotFound,
ErrorType::Banned => Status::Forbidden,
ErrorType::AlreadyInServer => Status::Conflict,
ErrorType::CannotTimeoutYourself => Status::BadRequest,
ErrorType::TooManyServers { .. } => Status::BadRequest,
ErrorType::TooManyEmbeds { .. } => Status::BadRequest,
@@ -73,6 +74,7 @@ impl<'r> Responder<'r, 'static> for Error {
ErrorType::InvalidCredentials => Status::Unauthorized,
ErrorType::InvalidProperty => Status::BadRequest,
ErrorType::InvalidSession => Status::Unauthorized,
ErrorType::NotAuthenticated => Status::Unauthorized,
ErrorType::DuplicateNonce => Status::Conflict,
ErrorType::VosoUnavailable => Status::BadRequest,
ErrorType::NotFound => Status::NotFound,
@@ -80,6 +82,11 @@ impl<'r> Responder<'r, 'static> for Error {
ErrorType::FailedValidation { .. } => Status::BadRequest,
ErrorType::ProxyError => Status::BadRequest,
ErrorType::FileTooSmall => Status::UnprocessableEntity,
ErrorType::FileTooLarge { .. } => Status::UnprocessableEntity,
ErrorType::FileTypeNotAllowed => Status::BadRequest,
ErrorType::ImageProcessingFailed => Status::InternalServerError,
ErrorType::NoEmbedData => Status::BadRequest,
};
// Serialize the error data structure into JSON.

View File

@@ -0,0 +1,22 @@
[package]
name = "revolt-crond"
version = "0.8.2"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <me@insrt.uk>"]
edition = "2021"
description = "Revolt Daemon Service: Timed data clean up tasks"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
# Utility
log = "0.4"
# Async
tokio = { version = "1" }
# Core
revolt-database = { version = "0.8.2", path = "../../core/database" }
revolt-result = { version = "0.8.2", path = "../../core/result" }
revolt-config = { version = "0.8.2", path = "../../core/config" }
revolt-files = { version = "0.8.2", path = "../../core/files" }

View File

@@ -0,0 +1,10 @@
# Build Stage
FROM ghcr.io/revoltchat/base:latest AS builder
FROM debian:12 AS debian
# Bundle Stage
FROM gcr.io/distroless/cc-debian12:nonroot
COPY --from=builder /home/rust/src/target/release/revolt-crond ./
USER nonroot
CMD ["./revolt-crond"]

View File

@@ -0,0 +1,19 @@
use revolt_config::configure;
use revolt_database::DatabaseInfo;
use revolt_result::Result;
use tasks::{file_deletion, prune_dangling_files};
use tokio::try_join;
pub mod tasks;
#[tokio::main]
async fn main() -> Result<()> {
configure!(crond);
let db = DatabaseInfo::Auto.connect().await.expect("database");
try_join!(
file_deletion::task(db.clone()),
prune_dangling_files::task(db)
)
.map(|_| ())
}

View File

@@ -0,0 +1,39 @@
use std::time::Duration;
use log::info;
use revolt_database::Database;
use revolt_files::delete_from_s3;
use revolt_result::Result;
use tokio::time::sleep;
pub async fn task(db: Database) -> Result<()> {
loop {
let files = db.fetch_deleted_attachments().await?;
for file in files {
let count = db
.count_file_hash_references(file.hash.as_ref().expect("no `hash` present"))
.await?;
// No other files reference this file on disk anymore
if count <= 1 {
let file_hash = db
.fetch_attachment_hash(file.hash.as_ref().expect("no `hash` present"))
.await?;
// Delete from S3
delete_from_s3(&file_hash.bucket_id, &file_hash.path).await?;
// Delete the hash
db.delete_attachment_hash(&file_hash.id).await?;
info!("Deleted file hash {}", file_hash.id);
}
// Delete the file
db.delete_attachment(&file.id).await?;
info!("Deleted file {}", file.id);
}
sleep(Duration::from_secs(60)).await;
}
}

View File

@@ -0,0 +1,2 @@
pub mod file_deletion;
pub mod prune_dangling_files;

View File

@@ -0,0 +1,36 @@
use std::time::Duration;
use revolt_database::{iso8601_timestamp::Timestamp, Database};
use revolt_result::Result;
use tokio::time::sleep;
use log::info;
pub async fn task(db: Database) -> Result<()> {
loop {
// This could just be a single database query
// ... but timestamps are inconsistently serialised
// ... sometimes they are dates/numbers, hard to query
// ... in the future, we could use Postgres instead! :D
// ...
// ... on the plus side, it's still only 2 queries
let files = db.fetch_dangling_files().await?;
let file_ids: Vec<String> = files
.into_iter()
.filter(|file| {
file.uploaded_at.is_some_and(|uploaded_at| {
Timestamp::now_utc().duration_since(uploaded_at) > Duration::from_secs(60 * 60)
})
})
.map(|file| file.id)
.collect();
if !file_ids.is_empty() {
db.mark_attachments_as_deleted(&file_ids).await?;
info!("Marked {} dangling files for deletion", file_ids.len());
}
sleep(Duration::from_secs(60)).await;
}
}

View File

@@ -0,0 +1,31 @@
[package]
name = "revolt-pushd"
version = "0.8.2"
edition = "2021"
[dependencies]
revolt-config = { version = "0.8.2", path = "../../core/config" }
revolt-database = { version = "0.8.2", path = "../../core/database" }
revolt-models = { version = "0.8.2", path = "../../core/models", features = [
"validator",
] }
amqprs = { version = "1.7.0" }
fcm_v1 = "0.3.0"
web-push = "0.10.0"
isahc = { optional = true, version = "1.7", features = ["json"] }
revolt_a2 = { version = "0.10", default-features = false, features = ["ring"] }
tokio = "1.39.2"
async-trait = "0.1.81"
ulid = "1.0.0"
authifier = "1.0.8"
log = "0.4.11"
#serialization
serde_json = "1"
revolt_optional_struct = "0.2.0"
serde = { version = "1", features = ["derive"] }
iso8601-timestamp = { version = "0.2.10", features = ["serde", "bson"] }
base64 = "0.22.1"

View File

@@ -0,0 +1,14 @@
# Build Stage
# BASE_IMAGE is the compiled workspace base image. Defaults to the upstream
# revolt base, but CI overrides it with our own base built from repo.
ARG BASE_IMAGE=ghcr.io/revoltchat/base:latest
FROM ${BASE_IMAGE} AS builder
FROM debian:12 AS debian
# Bundle Stage
FROM gcr.io/distroless/cc-debian12:nonroot
COPY --from=builder /home/rust/src/target/release/revolt-pushd ./
COPY --from=debian /usr/bin/uname /usr/bin/uname
USER nonroot
CMD ["./revolt-pushd"]

Binary file not shown.

View File

@@ -0,0 +1,138 @@
use crate::consumers::inbound::internal::*;
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use async_trait::async_trait;
use revolt_database::{events::rabbit::*, Database};
pub struct AckConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
}
impl Channeled for AckConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl AckConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> AckConsumer {
AckConsumer {
db,
authifier_db,
conn: None,
channel: None,
}
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for AckConsumer {
/// This consumer processes all acks the platform receives, and sends relevant badge updates to apple platforms.
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
let content = String::from_utf8(content).unwrap();
let payload: AckPayload = serde_json::from_str(content.as_str()).unwrap();
// Step 1: fetch unreads and don't continue if there's no unreads
#[allow(clippy::disallowed_methods)]
let unreads = self.db.fetch_unread_mentions(&payload.user_id).await;
if let Ok(u) = &unreads {
if u.is_empty() {
return;
}
} else {
return;
}
if let Ok(sessions) = self.authifier_db.find_sessions(&payload.user_id).await {
let config = revolt_config::config().await;
// Step 2: find any apple sessions, since we don't need to calculate this for anything else.
// If there's no apple sessions, we can return early
let apple_sessions: Vec<&authifier::models::Session> = sessions
.iter()
.filter(|session| {
if let Some(sub) = &session.subscription {
sub.endpoint == "apn"
} else {
false
}
})
.collect();
if apple_sessions.is_empty() {
return;
}
// Step 3: calculate the actual mention count, since we have to send it out
let mut mention_count = 0;
for u in &unreads.unwrap() {
mention_count += u.mentions.as_ref().unwrap().len()
}
// Step 4: loop through each apple session and send the badge update
for session in apple_sessions {
let service_payload = PayloadToService {
notification: PayloadKind::BadgeUpdate(mention_count),
user_id: payload.user_id.clone(),
session_id: session.id.clone(),
token: session.subscription.as_ref().unwrap().auth.clone(),
extras: Default::default(),
};
let raw_service_payload = serde_json::to_string(&service_payload);
if let Ok(p) = raw_service_payload {
let args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
log::debug!(
"Publishing ack to apn session {}",
session.subscription.as_ref().unwrap().auth
);
publish_message(self, p.into(), args).await;
} else {
log::warn!("Failed to serialize ack badge update payload!");
revolt_config::capture_error(&raw_service_payload.unwrap_err());
}
}
}
}
}

View File

@@ -0,0 +1,121 @@
use std::collections::HashMap;
use crate::consumers::inbound::internal::*;
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use async_trait::async_trait;
use log::debug;
use revolt_database::{events::rabbit::*, Database};
pub struct FRAcceptedConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
}
impl Channeled for FRAcceptedConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl FRAcceptedConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> FRAcceptedConsumer {
FRAcceptedConsumer {
db,
authifier_db,
conn: None,
channel: None,
}
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for FRAcceptedConsumer {
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
let content = String::from_utf8(content).unwrap();
let payload: FRAcceptedPayload = serde_json::from_str(content.as_str()).unwrap();
debug!("Received FR accept event");
if let Ok(sessions) = self.authifier_db.find_sessions(&payload.user).await {
let config = revolt_config::config().await;
for session in sessions {
if let Some(sub) = session.subscription {
let mut sendable = PayloadToService {
notification: PayloadKind::FRAccepted(payload.clone()),
token: sub.auth,
user_id: session.user_id,
session_id: session.id,
extras: HashMap::new(),
};
let args: BasicPublishArguments;
if sub.endpoint == "apn" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
} else if sub.endpoint == "fcm" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.fcm.queue.as_str(),
)
.finish();
} else {
// web push (vapid)
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.vapid.queue.as_str(),
)
.finish();
sendable.extras.insert("p265dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), sub.endpoint.clone());
}
let payload = serde_json::to_string(&sendable).unwrap();
publish_message(self, payload.into(), args).await;
}
}
}
}
}

View File

@@ -0,0 +1,121 @@
use std::collections::HashMap;
use crate::consumers::inbound::internal::*;
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use async_trait::async_trait;
use log::debug;
use revolt_database::{events::rabbit::*, Database};
pub struct FRReceivedConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
}
impl Channeled for FRReceivedConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl FRReceivedConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> FRReceivedConsumer {
FRReceivedConsumer {
db,
authifier_db,
conn: None,
channel: None,
}
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for FRReceivedConsumer {
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
let content = String::from_utf8(content).unwrap();
let payload: FRReceivedPayload = serde_json::from_str(content.as_str()).unwrap();
debug!("Received FR received event");
if let Ok(sessions) = self.authifier_db.find_sessions(&payload.user).await {
let config = revolt_config::config().await;
for session in sessions {
if let Some(sub) = session.subscription {
let mut sendable = PayloadToService {
notification: PayloadKind::FRReceived(payload.clone()),
token: sub.auth,
user_id: session.user_id,
session_id: session.id,
extras: HashMap::new(),
};
let args: BasicPublishArguments;
if sub.endpoint == "apn" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
} else if sub.endpoint == "fcm" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.fcm.queue.as_str(),
)
.finish();
} else {
// web push (vapid)
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.vapid.queue.as_str(),
)
.finish();
sendable.extras.insert("p265dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), sub.endpoint.clone());
}
let payload = serde_json::to_string(&sendable).unwrap();
publish_message(self, payload.into(), args).await;
}
}
}
}
}

View File

@@ -0,0 +1,127 @@
use std::collections::HashMap;
use crate::consumers::inbound::internal::*;
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use async_trait::async_trait;
use log::debug;
use revolt_database::{events::rabbit::*, Database};
pub struct GenericConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
}
impl Channeled for GenericConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl GenericConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> GenericConsumer {
GenericConsumer {
db,
authifier_db,
conn: None,
channel: None,
}
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for GenericConsumer {
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
let content = String::from_utf8(content).unwrap();
let payload: MessageSentPayload = serde_json::from_str(content.as_str()).unwrap();
debug!("Received message event on origin");
if let Ok(sessions) = self
.authifier_db
.find_sessions_with_subscription(&payload.users)
.await
{
let config = revolt_config::config().await;
for session in sessions {
if let Some(sub) = session.subscription {
let mut sendable = PayloadToService {
notification: PayloadKind::MessageNotification(
payload.notification.clone(),
),
token: sub.auth,
user_id: session.user_id,
session_id: session.id,
extras: HashMap::new(),
};
let args: BasicPublishArguments;
if sub.endpoint == "apn" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
} else if sub.endpoint == "fcm" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.fcm.queue.as_str(),
)
.finish();
} else {
// web push (vapid)
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.vapid.queue.as_str(),
)
.finish();
sendable.extras.insert("p265dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), sub.endpoint.clone());
}
let payload = serde_json::to_string(&sendable).unwrap();
publish_message(self, payload.into(), args).await;
}
}
}
}
}

View File

@@ -0,0 +1,53 @@
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::{Connection, OpenConnectionArguments},
BasicProperties,
};
use log::{debug, warn};
pub(crate) trait Channeled {
#[allow(unused)]
fn get_connection(&self) -> Option<&Connection>;
fn get_channel(&self) -> Option<&Channel>;
fn set_connection(&mut self, conn: Connection);
fn set_channel(&mut self, channel: Channel);
}
pub(crate) async fn make_channel<T: Channeled>(consumer: &mut T) {
let config = revolt_config::config().await;
let args = OpenConnectionArguments::new(
&config.rabbit.host,
config.rabbit.port,
&config.rabbit.username,
&config.rabbit.password,
);
let conn = amqprs::connection::Connection::open(&args).await.unwrap();
let channel = conn.open_channel(None).await.unwrap();
consumer.set_connection(conn);
consumer.set_channel(channel);
}
pub(crate) async fn publish_message<T: Channeled>(
consumer: &mut T,
payload: Vec<u8>,
args: BasicPublishArguments,
) {
let routing_key = &args.routing_key.clone();
let mut channel = consumer.get_channel();
if channel.is_none() {
make_channel(consumer).await;
channel = consumer.get_channel();
}
if let Some(chnl) = channel {
chnl.basic_publish(BasicProperties::default(), payload.clone(), args.clone())
.await
.unwrap();
debug!("Sent message to queue for target {}", routing_key);
} else {
warn!("Failed to unwrap channel (including attempt to make a channel)!")
}
}

View File

@@ -0,0 +1,127 @@
use std::collections::HashMap;
use crate::consumers::inbound::internal::*;
use amqprs::{
channel::{BasicPublishArguments, Channel},
connection::Connection,
consumer::AsyncConsumer,
BasicProperties, Deliver,
};
use async_trait::async_trait;
use log::debug;
use revolt_database::{events::rabbit::*, Database};
pub struct MessageConsumer {
#[allow(dead_code)]
db: Database,
authifier_db: authifier::Database,
conn: Option<Connection>,
channel: Option<Channel>,
}
impl Channeled for MessageConsumer {
fn get_connection(&self) -> Option<&Connection> {
if self.conn.is_none() {
None
} else {
Some(self.conn.as_ref().unwrap())
}
}
fn get_channel(&self) -> Option<&Channel> {
if self.channel.is_none() {
None
} else {
Some(self.channel.as_ref().unwrap())
}
}
fn set_connection(&mut self, conn: Connection) {
self.conn = Some(conn);
}
fn set_channel(&mut self, channel: Channel) {
self.channel = Some(channel)
}
}
impl MessageConsumer {
pub fn new(db: Database, authifier_db: authifier::Database) -> MessageConsumer {
MessageConsumer {
db,
authifier_db,
conn: None,
channel: None,
}
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for MessageConsumer {
/// This consumer handles delegating messages into their respective platform queues.
async fn consume(
&mut self,
channel: &Channel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
let content = String::from_utf8(content).unwrap();
let payload: MessageSentPayload = serde_json::from_str(content.as_str()).unwrap();
debug!("Received message event on origin");
if let Ok(sessions) = self
.authifier_db
.find_sessions_with_subscription(&payload.users)
.await
{
let config = revolt_config::config().await;
for session in sessions {
if let Some(sub) = session.subscription {
let mut sendable = PayloadToService {
notification: PayloadKind::MessageNotification(
payload.notification.clone(),
),
token: sub.auth,
user_id: session.user_id,
session_id: session.id,
extras: HashMap::new(),
};
let args: BasicPublishArguments;
if sub.endpoint == "apn" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.apn.queue.as_str(),
)
.finish();
} else if sub.endpoint == "fcm" {
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.fcm.queue.as_str(),
)
.finish();
} else {
// web push (vapid)
args = BasicPublishArguments::new(
config.pushd.exchange.as_str(),
config.pushd.vapid.queue.as_str(),
)
.finish();
sendable.extras.insert("p256dh".to_string(), sub.p256dh);
sendable
.extras
.insert("endpoint".to_string(), sub.endpoint.clone());
}
let payload = serde_json::to_string(&sendable).unwrap();
publish_message(self, payload.into(), args).await;
}
}
}
}
}

View File

@@ -0,0 +1,6 @@
pub mod ack;
pub mod fr_accepted;
pub mod fr_received;
pub mod generic;
mod internal;
pub mod message;

View File

@@ -0,0 +1,2 @@
pub mod inbound;
pub mod outbound;

View File

@@ -0,0 +1,338 @@
use std::{borrow::Cow, collections::BTreeMap, io::Cursor};
use amqprs::{channel::Channel as AmqpChannel, consumer::AsyncConsumer, BasicProperties, Deliver};
use async_trait::async_trait;
use base64::{
engine::{self},
Engine as _,
};
use revolt_a2::{
request::{
notification::{DefaultAlert, NotificationOptions},
payload::{APSAlert, APSSound, Payload, PayloadLike, APS},
},
Client, ClientConfig, Endpoint, Error, ErrorBody, ErrorReason, Priority, PushType, Response,
};
use revolt_database::{events::rabbit::*, Database};
use revolt_models::v0::{Channel, Message, PushNotification};
use serde::Serialize;
// region: payload
#[derive(Serialize, Debug)]
struct MessagePayload<'a> {
aps: APS<'a>,
#[serde(skip_serializing)]
options: NotificationOptions<'a>,
#[serde(skip_serializing)]
device_token: &'a str,
message: &'a Message,
url: &'a str,
#[serde(rename = "camelCase")]
author_avatar: &'a str,
#[serde(rename = "camelCase")]
author_display_name: &'a str,
#[serde(rename = "camelCase")]
channel_name: &'a str,
}
impl<'a> PayloadLike for MessagePayload<'a> {
fn get_device_token(&self) -> &'a str {
self.device_token
}
fn get_options(&self) -> &NotificationOptions {
&self.options
}
}
// region: consumer
pub struct ApnsOutboundConsumer {
#[allow(dead_code)]
db: Database,
client: Client,
}
impl ApnsOutboundConsumer {
fn format_title(&self, notification: &PushNotification) -> String {
// ideally this changes depending on context
// in a server, it would look like "Sendername, #channelname in servername"
// in a group, it would look like "Sendername in groupname"
// in a dm it should just be "Sendername".
// not sure how feasible all those are given the PushNotification object as it currently stands.
match &notification.channel {
Channel::DirectMessage { .. } => notification.author.clone(),
Channel::Group { name, .. } => format!("{}, #{}", notification.author, name),
Channel::TextChannel { name, .. } | Channel::VoiceChannel { name, .. } => {
format!("{} in #{}", notification.author, name)
}
_ => "Unknown".to_string(),
}
}
async fn get_badge_count(&self, user: &str) -> Option<u32> {
if let Ok(unreads) = self.db.fetch_unread_mentions(user).await {
let mut mention_count = 0;
for channel in unreads {
if let Some(mentions) = channel.mentions {
mention_count += mentions.len() as u32
}
}
println!("Got badge count for APN: {}", mention_count);
return Some(mention_count);
}
None
}
}
impl ApnsOutboundConsumer {
pub async fn new(db: Database) -> Result<ApnsOutboundConsumer, &'static str> {
let config = revolt_config::config().await;
if config.pushd.apn.pkcs8.is_empty()
|| config.pushd.apn.key_id.is_empty()
|| config.pushd.apn.team_id.is_empty()
{
return Err("Missing APN keys.");
}
let endpoint = if config.pushd.apn.sandbox {
Endpoint::Sandbox
} else {
Endpoint::Production
};
let pkcs8 = engine::general_purpose::STANDARD
.decode(config.pushd.apn.pkcs8.clone())
.expect("valid `pcks8`");
let client_config = ClientConfig::new(endpoint);
let client = Client::token(
&mut Cursor::new(pkcs8),
config.pushd.apn.key_id.clone(),
config.pushd.apn.team_id.clone(),
client_config,
)
.expect("could not create APN client");
Ok(ApnsOutboundConsumer { db, client })
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for ApnsOutboundConsumer {
async fn consume(
&mut self,
channel: &AmqpChannel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
let content = String::from_utf8(content).unwrap();
let payload: PayloadToService = serde_json::from_str(content.as_str()).unwrap();
let payload_options = NotificationOptions {
apns_id: None,
apns_push_type: Some(PushType::Alert),
apns_expiration: None,
apns_priority: Some(Priority::High),
apns_topic: Some("chat.revolt.app"),
apns_collapse_id: None,
};
let resp: Result<Response, Error>;
match payload.notification {
PayloadKind::FRReceived(alert) => {
let loc_args = vec![Cow::from(
alert
.from_user
.display_name
.or(Some(format!(
"{}#{}",
alert.from_user.username, alert.from_user.discriminator
)))
.clone()
.unwrap(),
)];
let apn_payload = Payload {
aps: APS {
alert: Some(APSAlert::Default(DefaultAlert {
title: None,
subtitle: None,
body: None,
title_loc_key: None,
title_loc_args: None,
action_loc_key: None,
loc_key: Some("push.fr.received"),
loc_args: Some(loc_args),
launch_image: None,
})),
badge: self.get_badge_count(&payload.user_id).await,
sound: Some(APSSound::Sound("default")),
thread_id: None,
content_available: None,
category: None,
mutable_content: Some(1),
url_args: None,
},
device_token: &payload.token,
options: payload_options.clone(),
data: BTreeMap::new(),
};
resp = self.client.send(apn_payload).await;
}
PayloadKind::FRAccepted(alert) => {
let loc_args = vec![Cow::from(
alert
.accepted_user
.display_name
.or(Some(format!(
"{}#{}",
alert.accepted_user.username, alert.accepted_user.discriminator
)))
.clone()
.unwrap(),
)];
let apn_payload = Payload {
aps: APS {
alert: Some(APSAlert::Default(DefaultAlert {
title: None,
subtitle: None,
body: None,
title_loc_key: None,
title_loc_args: None,
action_loc_key: None,
loc_key: Some("push.fr.accepted"),
loc_args: Some(loc_args),
launch_image: None,
})),
badge: self.get_badge_count(&payload.user_id).await,
sound: Some(APSSound::Sound("default")),
thread_id: None,
content_available: None,
category: None,
mutable_content: Some(1),
url_args: None,
},
device_token: &payload.token,
options: payload_options.clone(),
data: BTreeMap::new(),
};
resp = self.client.send(apn_payload).await;
}
PayloadKind::Generic(alert) => {
let apn_payload = Payload {
aps: APS {
alert: Some(APSAlert::Default(DefaultAlert {
title: Some(&alert.title),
subtitle: None,
body: Some(&alert.body),
title_loc_key: None,
title_loc_args: None,
action_loc_key: None,
loc_key: None,
loc_args: None,
launch_image: None,
})),
badge: self.get_badge_count(&payload.user_id).await,
sound: Some(APSSound::Sound("default")),
thread_id: None,
content_available: None,
category: None,
mutable_content: Some(1),
url_args: None,
},
device_token: &payload.token,
options: payload_options.clone(),
data: BTreeMap::new(),
};
resp = self.client.send(apn_payload).await;
}
PayloadKind::MessageNotification(alert) => {
let title = self.format_title(&alert);
let apn_payload = MessagePayload {
aps: APS {
alert: Some(APSAlert::Default(DefaultAlert {
title: Some(&title),
subtitle: None,
body: Some(&alert.body),
title_loc_key: None,
title_loc_args: None,
action_loc_key: None,
loc_key: None,
loc_args: None,
launch_image: None,
})),
badge: self.get_badge_count(&payload.user_id).await,
sound: Some(APSSound::Sound("default")),
thread_id: Some(alert.channel.id()),
content_available: None,
category: None,
mutable_content: Some(1),
url_args: None,
},
device_token: &payload.token,
options: payload_options.clone(),
message: &alert.message,
url: &alert.url,
author_avatar: &alert.icon,
author_display_name: &alert.author,
channel_name: alert.channel.name().unwrap_or(&title),
};
resp = self.client.send(apn_payload).await;
}
PayloadKind::BadgeUpdate(badge) => {
let apn_payload = Payload {
aps: APS {
badge: Some(badge as u32),
..Default::default()
},
device_token: &payload.token,
options: payload_options.clone(),
data: BTreeMap::new(),
};
resp = self.client.send(apn_payload).await;
}
}
if let Err(err) = resp {
match err {
Error::ResponseError(Response {
error:
Some(ErrorBody {
reason: ErrorReason::BadDeviceToken | ErrorReason::Unregistered,
..
}),
..
}) => {
if let Err(err) = self
.db
.remove_push_subscription_by_session_id(&payload.session_id)
.await
{
revolt_config::capture_error(&err);
}
}
err => {
revolt_config::capture_error(&err);
}
}
}
}
}

View File

@@ -0,0 +1,199 @@
use std::{collections::HashMap, time::Duration};
use amqprs::{channel::Channel as AmqpChannel, consumer::AsyncConsumer, BasicProperties, Deliver};
use async_trait::async_trait;
use fcm_v1::{
android::AndroidConfig,
auth::{Authenticator, ServiceAccountKey},
message::{Message, Notification},
Client, Error as FcmError,
};
use revolt_database::{events::rabbit::*, Database};
use revolt_models::v0::{Channel, PushNotification};
use serde_json::Value;
pub struct FcmOutboundConsumer {
db: Database,
client: Client,
}
impl FcmOutboundConsumer {
fn format_title(&self, notification: &PushNotification) -> String {
// ideally this changes depending on context
// in a server, it would look like "Sendername, #channelname in servername"
// in a group, it would look like "Sendername in groupname"
// in a dm it should just be "Sendername".
// not sure how feasible all those are given the PushNotification object as it currently stands.
match &notification.channel {
Channel::DirectMessage { .. } => notification.author.clone(),
Channel::Group { name, .. } => format!("{}, #{}", notification.author, name),
Channel::TextChannel { name, .. } | Channel::VoiceChannel { name, .. } => {
format!("{} in #{}", notification.author, name)
}
_ => "Unknown".to_string(),
}
}
}
impl FcmOutboundConsumer {
pub async fn new(db: Database) -> Result<FcmOutboundConsumer, &'static str> {
let config = revolt_config::config().await;
Ok(FcmOutboundConsumer {
db,
client: Client::new(
Authenticator::service_account::<&str>(ServiceAccountKey {
key_type: Some(config.pushd.fcm.key_type),
project_id: Some(config.pushd.fcm.project_id.clone()),
private_key_id: Some(config.pushd.fcm.private_key_id),
private_key: config.pushd.fcm.private_key,
client_email: config.pushd.fcm.client_email,
client_id: Some(config.pushd.fcm.client_id),
auth_uri: Some(config.pushd.fcm.auth_uri),
token_uri: config.pushd.fcm.token_uri,
auth_provider_x509_cert_url: Some(config.pushd.fcm.auth_provider_x509_cert_url),
client_x509_cert_url: Some(config.pushd.fcm.client_x509_cert_url),
})
.await
.unwrap(),
config.pushd.fcm.project_id,
false,
Duration::from_secs(5),
),
})
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for FcmOutboundConsumer {
async fn consume(
&mut self,
channel: &AmqpChannel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
let content = String::from_utf8(content).unwrap();
let payload: PayloadToService = serde_json::from_str(content.as_str()).unwrap();
let config = revolt_config::config().await;
#[allow(clippy::needless_late_init)]
let resp: Result<Message, FcmError>;
match payload.notification {
PayloadKind::FRReceived(alert) => {
let name = alert
.from_user
.display_name
.or(Some(format!(
"{}#{}",
alert.from_user.username, alert.from_user.discriminator
)))
.clone()
.unwrap();
let mut data = HashMap::new();
data.insert(
"type".to_string(),
Value::String("push.fr.receive".to_string()),
);
data.insert("id".to_string(), Value::String(alert.from_user.id));
data.insert("username".to_string(), Value::String(name));
let msg = Message {
token: Some(payload.token),
data: Some(data),
..Default::default()
};
resp = self.client.send(&msg).await;
}
PayloadKind::FRAccepted(alert) => {
let name = alert
.accepted_user
.display_name
.or(Some(format!(
"{}#{}",
alert.accepted_user.username, alert.accepted_user.discriminator
)))
.clone()
.unwrap();
let mut data: HashMap<String, Value> = HashMap::new();
data.insert(
"type".to_string(),
Value::String("push.fr.accept".to_string()),
);
data.insert("id".to_string(), Value::String(alert.accepted_user.id));
data.insert("username".to_string(), Value::String(name));
let msg = Message {
token: Some(payload.token),
data: Some(data),
..Default::default()
};
resp = self.client.send(&msg).await;
}
PayloadKind::Generic(alert) => {
let msg = Message {
token: Some(payload.token),
notification: Some(Notification {
title: Some(alert.title),
body: Some(alert.body),
image: alert.icon,
}),
..Default::default()
};
resp = self.client.send(&msg).await;
}
PayloadKind::MessageNotification(alert) => {
let title = self.format_title(&alert);
let msg = Message {
token: Some(payload.token),
notification: Some(Notification {
title: Some(title),
body: Some(alert.body),
image: Some(alert.icon),
}),
android: Some(AndroidConfig {
collapse_key: Some(alert.tag),
..Default::default()
}),
..Default::default()
};
resp = self.client.send(&msg).await;
}
PayloadKind::BadgeUpdate(_) => {
panic!("FCM cannot handle badge updates, and they should not be sent here.")
}
}
if let Err(err) = resp {
match err {
FcmError::Auth => {
if let Err(err) = self
.db
.remove_push_subscription_by_session_id(&payload.session_id)
.await
{
revolt_config::capture_error(&err);
}
}
err => {
revolt_config::capture_error(&err);
}
}
}
}
}

View File

@@ -0,0 +1,3 @@
pub mod apn;
pub mod fcm;
pub mod vapid;

View File

@@ -0,0 +1,162 @@
use std::collections::HashMap;
use amqprs::{channel::Channel as AmqpChannel, consumer::AsyncConsumer, BasicProperties, Deliver};
use async_trait::async_trait;
use base64::{
engine::{self},
Engine as _,
};
use revolt_database::{events::rabbit::*, Database};
// use revolt_models::v0::{Channel, PushNotification};
use web_push::{
ContentEncoding, IsahcWebPushClient, SubscriptionInfo, SubscriptionKeys, VapidSignatureBuilder,
WebPushClient, WebPushError, WebPushMessageBuilder,
};
pub struct VapidOutboundConsumer {
db: Database,
client: IsahcWebPushClient,
pkey: Vec<u8>,
}
impl VapidOutboundConsumer {
pub async fn new(db: Database) -> Result<VapidOutboundConsumer, &'static str> {
let config = revolt_config::config().await;
if config.pushd.vapid.private_key.is_empty() | config.pushd.vapid.public_key.is_empty() {
return Err("No Vapid keys present");
}
let web_push_private_key = engine::general_purpose::URL_SAFE_NO_PAD
.decode(config.pushd.vapid.private_key)
.expect("valid `VAPID_PRIVATE_KEY`");
Ok(VapidOutboundConsumer {
db,
client: IsahcWebPushClient::new().unwrap(),
pkey: web_push_private_key,
})
}
}
#[allow(unused_variables)]
#[async_trait]
impl AsyncConsumer for VapidOutboundConsumer {
async fn consume(
&mut self,
channel: &AmqpChannel,
deliver: Deliver,
basic_properties: BasicProperties,
content: Vec<u8>,
) {
let content = String::from_utf8(content).unwrap();
let payload: PayloadToService = serde_json::from_str(content.as_str()).unwrap();
let config = revolt_config::config().await;
// A VAPID web push payload must carry the subscription endpoint and the
// p256dh public key in `extras`. Dropped malformed requests instead
// of panicking.
let (Some(endpoint), Some(p256dh)) =
(payload.extras.get("endpoint"), payload.extras.get("p256dh"))
else {
log::warn!(
"Dropping VAPID push for session {}: payload missing endpoint/p256dh in extras.",
payload.session_id
);
return;
};
let subscription = SubscriptionInfo {
endpoint: endpoint.clone(),
keys: SubscriptionKeys {
auth: payload.token,
p256dh: p256dh.clone(),
},
};
#[allow(clippy::needless_late_init)]
let payload_body: String;
match payload.notification {
PayloadKind::FRReceived(alert) => {
let name = alert
.from_user
.display_name
.or(Some(format!(
"{}#{}",
alert.from_user.username, alert.from_user.discriminator
)))
.clone()
.unwrap();
let mut body = HashMap::new();
body.insert("body", format!("{} sent you a friend request", name));
payload_body = serde_json::to_string(&body).unwrap();
}
PayloadKind::FRAccepted(alert) => {
let name = alert
.accepted_user
.display_name
.or(Some(format!(
"{}#{}",
alert.accepted_user.username, alert.accepted_user.discriminator
)))
.clone()
.unwrap();
let mut body = HashMap::new();
body.insert("body", format!("{} accepted your friend request", name));
payload_body = serde_json::to_string(&body).unwrap();
}
PayloadKind::Generic(alert) => {
payload_body = serde_json::to_string(&alert).unwrap();
}
PayloadKind::MessageNotification(alert) => {
payload_body = serde_json::to_string(&alert).unwrap();
}
PayloadKind::BadgeUpdate(_) => {
panic!("Vapid cannot handle badge updates, and they should not be sent here.")
}
}
match VapidSignatureBuilder::from_pem(std::io::Cursor::new(&self.pkey), &subscription) {
Ok(sig_builder) => match sig_builder.build() {
Ok(signature) => {
let mut builder = WebPushMessageBuilder::new(&subscription);
builder.set_vapid_signature(signature);
builder.set_payload(ContentEncoding::AesGcm, payload_body.as_bytes());
match builder.build() {
Ok(msg) => {
if let Err(err) = self.client.send(msg).await {
if err == WebPushError::Unauthorized {
if let Err(err) = self
.db
.remove_push_subscription_by_session_id(&payload.session_id)
.await
{
revolt_config::capture_error(&err);
}
}
}
}
Err(err) => {
revolt_config::capture_error(&err);
}
}
}
Err(err) => {
revolt_config::capture_error(&err);
}
},
Err(err) => {
revolt_config::capture_error(&err);
}
}
}
}

View File

@@ -0,0 +1,233 @@
use amqprs::{
channel::{
BasicConsumeArguments, Channel, ExchangeDeclareArguments, QueueBindArguments,
QueueDeclareArguments,
},
connection::{Connection, OpenConnectionArguments},
consumer::AsyncConsumer,
FieldTable,
};
use revolt_config::{config, Settings};
use tokio::sync::Notify;
mod consumers;
use consumers::{
inbound::{
ack::AckConsumer, fr_accepted::FRAcceptedConsumer, fr_received::FRReceivedConsumer,
generic::GenericConsumer, message::MessageConsumer,
},
outbound::{apn::ApnsOutboundConsumer, fcm::FcmOutboundConsumer, vapid::VapidOutboundConsumer},
};
#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
async fn main() {
let config = config().await;
// Setup database
let db = revolt_database::DatabaseInfo::Auto.connect().await.unwrap();
let authifier: authifier::Database;
if let Some(client) = match &db {
revolt_database::Database::Reference(_) => None,
revolt_database::Database::MongoDb(mongo) => Some(mongo),
} {
authifier =
authifier::Database::MongoDb(authifier::database::MongoDb(client.database("revolt")));
} else {
panic!("Mongo is not in use, can't connect via authifier!")
}
let mut connections: Vec<(Channel, Connection)> = Vec::new();
// An explainer of how this works:
// The inbound connections are on separate routing keys, such that they only receive the proper payload
// from their respective api (prod or test).
// However, the outbound queues that go to the services are routed to receive from both, so that messages
// sent from beta are still notified on prod, and vice versa.
// This'll require some interesting shimming if we need to add more events once this is in prod (different payloads between prod and test),
// but that sounds like a problem for future us.
// inbound: generic
connections.push(
make_queue_and_consume(
&config,
&config.pushd.generic_queue,
config.pushd.get_generic_routing_key().as_str(),
None,
GenericConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
// inbound: messages
connections.push(
make_queue_and_consume(
&config,
&config.pushd.message_queue,
config.pushd.get_message_routing_key().as_str(),
None,
MessageConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
// inbound: FR received
connections.push(
make_queue_and_consume(
&config,
&config.pushd.fr_received_queue,
config.pushd.get_fr_received_routing_key().as_str(),
None,
FRReceivedConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
// inbound: FR accepted
connections.push(
make_queue_and_consume(
&config,
&config.pushd.fr_accepted_queue,
config.pushd.get_fr_accepted_routing_key().as_str(),
None,
FRAcceptedConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
if !config.pushd.apn.pkcs8.is_empty() {
connections.push(
make_queue_and_consume(
&config,
&config.pushd.apn.queue,
&config.pushd.apn.queue,
None,
ApnsOutboundConsumer::new(db.clone()).await.unwrap(),
)
.await,
);
let mut table = FieldTable::new();
table.insert("x-message-deduplication".try_into().unwrap(), "true".into());
connections.push(
make_queue_and_consume(
&config,
&config.pushd.ack_queue,
&config.pushd.ack_queue,
Some(table),
AckConsumer::new(db.clone(), authifier.clone()),
)
.await,
);
}
if !config.pushd.fcm.auth_uri.is_empty() {
connections.push(
make_queue_and_consume(
&config,
&config.pushd.fcm.queue,
&config.pushd.fcm.queue,
None,
FcmOutboundConsumer::new(db.clone()).await.unwrap(),
)
.await,
)
}
if !config.pushd.vapid.public_key.is_empty() {
connections.push(
make_queue_and_consume(
&config,
&config.pushd.vapid.queue,
&config.pushd.vapid.queue,
None,
VapidOutboundConsumer::new(db.clone()).await.unwrap(),
)
.await,
)
}
let guard = Notify::new();
guard.notified().await;
for (channel, conn) in connections {
channel.close().await.expect("Unable to close channel");
conn.close().await.expect("Unable to close connection");
}
}
async fn make_queue_and_consume<F>(
config: &Settings,
queue_name: &str,
routing_key: &str,
queue_args: Option<FieldTable>,
consumer: F,
) -> (Channel, Connection)
where
F: AsyncConsumer + Send + 'static,
{
let connection = Connection::open(&OpenConnectionArguments::new(
&config.rabbit.host,
config.rabbit.port,
&config.rabbit.username,
&config.rabbit.password,
))
.await
.unwrap();
let channel = connection.open_channel(None).await.unwrap();
channel
.exchange_declare(
ExchangeDeclareArguments::new(&config.pushd.exchange, "direct")
.durable(true)
.finish(),
)
.await
.expect("Failed to declare pushd exchange");
let mut queue_name = queue_name.to_string();
if config.pushd.production {
queue_name += "-prd";
} else {
queue_name += "-tst";
}
let queue_name = queue_name.as_str();
let mut args = QueueDeclareArguments::new(queue_name);
args.durable(true);
if let Some(arg) = queue_args {
args.arguments(arg);
}
let args = args.finish();
_ = channel.queue_declare(args).await.unwrap().unwrap();
channel
.queue_bind(QueueBindArguments::new(
queue_name,
&config.pushd.exchange,
routing_key,
))
.await
.expect(
"This probably means the revolt.notifications exchange does not exist in rabbitmq!",
);
let args = BasicConsumeArguments::new(queue_name, "")
.manual_ack(false)
.finish();
channel.basic_consume(consumer, args).await.unwrap();
log::info!(
"Consuming routing key {} as queue {}",
routing_key,
queue_name
);
(channel, connection)
}

View File

@@ -1,6 +1,6 @@
[package]
name = "revolt-delta"
version = "0.7.16"
version = "0.8.2"
license = "AGPL-3.0-or-later"
authors = ["Paul Makles <paulmakles@gmail.com>"]
edition = "2018"
@@ -16,7 +16,6 @@ redis-kiss = "0.1.4"
lru = "0.7.0"
url = "2.2.2"
log = "0.4.11"
dotenv = "0.15.0"
dashmap = "5.2.0"
linkify = "0.6.0"
once_cell = "1.17.1"
@@ -53,20 +52,21 @@ async-std = { version = "1.8.0", features = [
lettre = "0.10.0-alpha.4"
# web
rocket = { version = "0.5.0-rc.2", default-features = false, features = [
"json",
] }
rocket_cors = { git = "https://github.com/lawliet89/rocket_cors", rev = "c17e8145baa4790319fdb6a473e465b960f55e7c" }
rocket = { version = "0.5.1", default-features = false, features = ["json"] }
rocket_cors = { git = "https://github.com/lawliet89/rocket_cors", rev = "072d90359b23e9b291df6b672c07c93de9c46011" }
rocket_empty = { version = "0.1.1", features = ["schema"] }
rocket_authifier = { version = "1.0.8" }
rocket_authifier = { version = "1.0.9" }
rocket_prometheus = "0.10.0-rc.3"
# spec generation
schemars = "0.8.8"
revolt_rocket_okapi = { version = "0.9.1", features = ["swagger"] }
revolt_rocket_okapi = { version = "0.10.0", features = ["swagger"] }
# rabbit
amqprs = { version = "1.7.0" }
# core
authifier = "1.0.8"
authifier = "1.0.9"
revolt-config = { path = "../core/config" }
revolt-database = { path = "../core/database", features = [
"rocket-impl",

View File

@@ -1,12 +1,13 @@
# Build Stage
FROM ghcr.io/revoltchat/base:latest AS builder
FROM debian:12 AS debian
# Bundle Stage
FROM gcr.io/distroless/cc-debian12:nonroot
COPY --from=builder /home/rust/src/target/release/revolt-delta ./
COPY --from=debian /usr/bin/uname /usr/bin/uname
EXPOSE 8000
ENV ROCKET_ADDRESS 0.0.0.0
ENV ROCKET_PORT 8000
EXPOSE 14702
ENV ROCKET_ADDRESS=0.0.0.0
USER nonroot
CMD ["./revolt-delta"]

View File

@@ -10,12 +10,17 @@ pub mod util;
use revolt_config::config;
use revolt_database::events::client::EventV1;
use revolt_database::AMQP;
use rocket::{Build, Rocket};
use rocket_cors::{AllowedOrigins, CorsOptions};
use rocket_prometheus::PrometheusMetrics;
use std::net::Ipv4Addr;
use std::str::FromStr;
use amqprs::{
channel::ExchangeDeclareArguments,
connection::{Connection, OpenConnectionArguments},
};
use async_std::channel::unbounded;
use authifier::AuthifierEvent;
use rocket::data::ToByteUnit;
@@ -32,7 +37,7 @@ pub async fn web() -> Rocket<Build> {
db.migrate_database().await.unwrap();
// Setup Authifier event channel
let (sender, receiver) = unbounded();
let (_, receiver) = unbounded();
// Setup Authifier
let authifier = db.clone().to_authifier().await;
@@ -53,9 +58,6 @@ pub async fn web() -> Rocket<Build> {
}
});
// Launch background task workers
revolt_database::tasks::start_workers(db.clone(), authifier.database.clone());
// Configure CORS
let cors = CorsOptions {
allowed_origins: AllowedOrigins::All,
@@ -73,12 +75,49 @@ pub async fn web() -> Rocket<Build> {
// Configure Swagger
let swagger = revolt_rocket_okapi::swagger_ui::make_swagger_ui(
&revolt_rocket_okapi::swagger_ui::SwaggerUIConfig {
url: "../openapi.json".to_owned(),
url: "/openapi.json".to_owned(),
..Default::default()
},
)
.into();
let swagger_0_8 = revolt_rocket_okapi::swagger_ui::make_swagger_ui(
&revolt_rocket_okapi::swagger_ui::SwaggerUIConfig {
url: "/0.8/openapi.json".to_owned(),
..Default::default()
},
)
.into();
// Configure Rabbit
let connection = Connection::open(&OpenConnectionArguments::new(
&config.rabbit.host,
config.rabbit.port,
&config.rabbit.username,
&config.rabbit.password,
))
.await
.expect("Failed to connect to RabbitMQ");
let channel = connection
.open_channel(None)
.await
.expect("Failed to open RabbitMQ channel");
channel
.exchange_declare(
ExchangeDeclareArguments::new(&config.pushd.exchange, "direct")
.durable(true)
.finish(),
)
.await
.expect("Failed to declare exchange");
let amqp = AMQP::new(connection, channel);
// Launch background task workers
revolt_database::tasks::start_workers(db.clone(), amqp.clone());
// Configure Rocket
let rocket = rocket::build();
let prometheus = PrometheusMetrics::new();
@@ -89,8 +128,10 @@ pub async fn web() -> Rocket<Build> {
.mount("/", rocket_cors::catch_all_options_routes())
.mount("/", util::ratelimiter::routes())
.mount("/swagger/", swagger)
.mount("/0.8/swagger/", swagger_0_8)
.manage(authifier)
.manage(db)
.manage(amqp)
.manage(cors.clone())
.attach(util::ratelimiter::RatelimitFairing)
.attach(cors)

Some files were not shown because too many files have changed in this diff Show More