From 32a294a64a6d77c77221066a82287f42da5ac6d1 Mon Sep 17 00:00:00 2001 From: Tim Davis Date: Wed, 15 Mar 2023 14:39:54 -0700 Subject: [PATCH 01/56] feat: add support for arm64 (#142) - set explicit action permissions - make package owner dynamic so forks can test workflow - build multi platform images using cross compilation amd64 -> arm64 - add a scope to buildx gha cache --- .github/workflows/docker.yaml | 52 ++++++++++++++++++++--------------- Dockerfile | 25 +++++++++++++---- crates/bonfire/Dockerfile | 7 +++-- crates/delta/Dockerfile | 7 +++-- scripts/build-image-layer.sh | 47 +++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 33 deletions(-) create mode 100644 scripts/build-image-layer.sh diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 7a5e6471..67b06bd3 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -20,47 +20,52 @@ on: - "Dockerfile" workflow_dispatch: +permissions: + contents: read + packages: write + jobs: base: runs-on: ubuntu-latest - name: Build base image (amd64) + name: Build base image steps: # Configure build environment - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 # Authenticate with GHCR - name: Login to Github Container Registry - uses: docker/login-action@v1 + if: ${{ github.event_name != 'pull_request' }} + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # Build all projects and cache - - name: Build Base Image - uses: docker/build-push-action@v3 + - name: Build base image + uses: docker/build-push-action@v4 with: context: . - push: true - tags: ghcr.io/revoltchat/base:latest - cache-from: type=gha - cache-to: type=gha,mode=max + push: ${{ github.event_name != 'pull_request' }} + platforms: linux/amd64,linux/arm64 + tags: ghcr.io/${{ github.repository_owner }}/base:latest + cache-from: type=gha,scope=buildx-base-multi-arch + cache-to: type=gha,scope=buildx-base-multi-arch,mode=max - publish_amd64: + publish: needs: [base] runs-on: ubuntu-latest if: github.event_name != 'pull_request' strategy: matrix: project: [delta, bonfire] - name: Build ${{ matrix.project }} image (amd64) + name: Build ${{ matrix.project }} image steps: # Configure build environment - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -68,10 +73,11 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v2 with: + registry: docker.io username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Github Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} @@ -86,11 +92,11 @@ jobs: { "delta": { "path": "crates/delta", - "tag": "revoltchat/server" + "tag": "${{ github.repository_owner }}/server" }, "bonfire": { "path": "crates/bonfire", - "tag": "revoltchat/bonfire" + "tag": "${{ github.repository_owner }}/bonfire" } } export_to: output @@ -98,19 +104,21 @@ jobs: # Configure metadata - name: Docker meta id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: - images: ${{ steps.export.outputs.tag }}, ghcr.io/${{ steps.export.outputs.tag }} + images: | + docker.io/${{ steps.export.outputs.tag }} + ghcr.io/${{ steps.export.outputs.tag }} # Build crate image - name: Publish - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: context: . push: true - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 file: ${{ steps.export.outputs.path }}/Dockerfile tags: ${{ steps.meta.outputs.tags }} + build-args: | + BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile index 877ed09e..748757cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,27 @@ # Build Stage -FROM rustlang/rust:nightly-slim AS builder +FROM --platform="${BUILDPLATFORM}" rustlang/rust:nightly-slim USER 0:0 WORKDIR /home/rust/src -# Install build requirements -RUN apt-get update && apt-get install -y libssl-dev pkg-config +ARG TARGETARCH -# Build all crates +# Install build requirements +RUN dpkg --add-architecture "${TARGETARCH}" +RUN apt-get update && \ + apt-get install -y \ + make \ + pkg-config \ + libssl-dev:"${TARGETARCH}" +COPY scripts/build-image-layer.sh /tmp/ +RUN sh /tmp/build-image-layer.sh tools + +# Build all dependencies COPY Cargo.toml Cargo.lock ./ +COPY crates/bonfire/Cargo.toml ./crates/bonfire/ +COPY crates/delta/Cargo.toml ./crates/delta/ +COPY crates/quark/Cargo.toml ./crates/quark/ +RUN sh /tmp/build-image-layer.sh deps + +# Build all apps COPY crates ./crates -RUN cargo build --locked --release +RUN sh /tmp/build-image-layer.sh apps diff --git a/crates/bonfire/Dockerfile b/crates/bonfire/Dockerfile index e0b41afd..272ed03d 100644 --- a/crates/bonfire/Dockerfile +++ b/crates/bonfire/Dockerfile @@ -1,10 +1,11 @@ # Build Stage FROM ghcr.io/revoltchat/base:latest AS builder -RUN cargo install --locked --path crates/bonfire # Bundle Stage FROM debian:buster-slim -RUN apt-get update && apt-get install -y ca-certificates -COPY --from=builder /usr/local/cargo/bin/revolt-bonfire ./ +RUN apt-get update && \ + apt-get install -y ca-certificates && \ + apt-get clean +COPY --from=builder /home/rust/src/target/release/revolt-bonfire ./ EXPOSE 9000 CMD ["./revolt-bonfire"] diff --git a/crates/delta/Dockerfile b/crates/delta/Dockerfile index f02812b4..78489315 100644 --- a/crates/delta/Dockerfile +++ b/crates/delta/Dockerfile @@ -1,11 +1,12 @@ # Build Stage FROM ghcr.io/revoltchat/base:latest AS builder -RUN cargo install --locked --path crates/delta # Bundle Stage FROM debian:buster-slim -RUN apt-get update && apt-get install -y ca-certificates -COPY --from=builder /usr/local/cargo/bin/revolt-delta ./ +RUN apt-get update && \ + apt-get install -y ca-certificates && \ + apt-get clean +COPY --from=builder /home/rust/src/target/release/revolt-delta ./ EXPOSE 8000 ENV ROCKET_ADDRESS 0.0.0.0 diff --git a/scripts/build-image-layer.sh b/scripts/build-image-layer.sh new file mode 100644 index 00000000..69fca820 --- /dev/null +++ b/scripts/build-image-layer.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +set -eu + +case "${TARGETARCH}" in + "amd64") + LINKER_NAME="x86_64-linux-gnu-gcc" + LINKER_PACKAGE="gcc-x86-64-linux-gnu" + BUILD_TARGET="x86_64-unknown-linux-gnu" ;; + "arm64") + LINKER_NAME="aarch64-linux-gnu-gcc" + LINKER_PACKAGE="gcc-aarch64-linux-gnu" + BUILD_TARGET="aarch64-unknown-linux-gnu" ;; +esac + +tools() { + apt-get install -y "${LINKER_PACKAGE}" + rustup target add "${BUILD_TARGET}" +} + +deps() { + mkdir -p \ + crates/bonfire/src \ + crates/delta/src \ + crates/quark/src + echo 'fn main() { panic!("stub"); }' | + tee crates/bonfire/src/main.rs | + tee crates/delta/src/main.rs + echo '' | + tee crates/quark/src/lib.rs + cargo build --locked --release --target "${BUILD_TARGET}" +} + +apps() { + touch -am \ + crates/bonfire/src/main.rs \ + crates/delta/src/main.rs \ + crates/quark/src/lib.rs + cargo build --locked --release --target "${BUILD_TARGET}" + mv target _target && mv _target/"${BUILD_TARGET}" target +} + +export RUSTFLAGS="-C linker=${LINKER_NAME}" +export PKG_CONFIG_ALLOW_CROSS="1" +export PKG_CONFIG_PATH="/usr/lib/pkgconfig:/usr/lib/aarch64-linux-gnu/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig" + +"$@" From b9d813d8f0a800d0162f1c4cb9c00d1045ec09f1 Mon Sep 17 00:00:00 2001 From: ToastXC <100072983+toastxc@users.noreply.github.com> Date: Thu, 20 Apr 2023 18:04:51 +0800 Subject: [PATCH 02/56] fix: deny access for bot creating a server (#238) --- crates/delta/src/routes/servers/server_create.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/delta/src/routes/servers/server_create.rs b/crates/delta/src/routes/servers/server_create.rs index 85ba1e21..3a7ed5ce 100644 --- a/crates/delta/src/routes/servers/server_create.rs +++ b/crates/delta/src/routes/servers/server_create.rs @@ -44,6 +44,10 @@ pub async fn req( user: User, info: Json, ) -> Result> { + if user.bot.is_some() { + return Err(Error::IsBot); + } + let info = info.into_inner(); info.validate() .map_err(|error| Error::FailedValidation { error })?; From 487b979f0dcb868ed6fd56f3652a023ca455f887 Mon Sep 17 00:00:00 2001 From: ToastXC <100072983+toastxc@users.noreply.github.com> Date: Thu, 20 Apr 2023 18:05:13 +0800 Subject: [PATCH 03/56] fix: remove check for bot account fetching invites from a server (#237) --- crates/delta/src/routes/servers/invites_fetch.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/delta/src/routes/servers/invites_fetch.rs b/crates/delta/src/routes/servers/invites_fetch.rs index d7f7de4f..86927866 100644 --- a/crates/delta/src/routes/servers/invites_fetch.rs +++ b/crates/delta/src/routes/servers/invites_fetch.rs @@ -11,10 +11,6 @@ use rocket::serde::json::Json; #[openapi(tag = "Server Members")] #[get("//invites")] pub async fn req(db: &Db, user: User, target: Ref) -> Result>> { - if user.bot.is_some() { - return Err(Error::IsBot); - } - let server = target.as_server(db).await?; perms(&user) .server(&server) From 1df90ff53bd1285c9f62f68ad3529372e8ad5efa Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Fri, 21 Apr 2023 17:27:45 +0100 Subject: [PATCH 04/56] chore: update DSN for Sentry --- crates/quark/src/util/log.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/quark/src/util/log.rs b/crates/quark/src/util/log.rs index 063ad744..04964194 100644 --- a/crates/quark/src/util/log.rs +++ b/crates/quark/src/util/log.rs @@ -14,7 +14,7 @@ pub fn setup_logging(release: &'static str) -> sentry::ClientInitGuard { info!("Starting {release}"); sentry::init(( - "https://62fd0e02c5354905b4e286757f4beb16@sentry.insert.moe/4", + "https://d1d2a6f15c6245a987c532bbbcb30a04@glitchtip.insert.moe/2", sentry::ClientOptions { release: Some(release.into()), ..Default::default() From 32542a822e3de0fc8cc7b29af46c54a9284ee2de Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Fri, 21 Apr 2023 17:28:13 +0100 Subject: [PATCH 05/56] feat: rewrite presence system to use sets --- Cargo.lock | 7 +- crates/bonfire/Cargo.toml | 2 +- crates/delta/Cargo.toml | 2 +- crates/quark/Cargo.toml | 3 +- crates/quark/src/presence/entry.rs | 64 ++--------------- crates/quark/src/presence/mod.rs | 96 ++++++++++++------------- crates/quark/src/presence/operations.rs | 71 ++++++++---------- docker-compose.db.yml | 6 ++ 8 files changed, 93 insertions(+), 158 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae1e92d6..3a310ecf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2792,7 +2792,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.5.17" +version = "0.5.18" dependencies = [ "async-std", "async-tungstenite", @@ -2808,7 +2808,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.5.17" +version = "0.5.18" dependencies = [ "async-channel", "async-std", @@ -2847,7 +2847,7 @@ dependencies = [ [[package]] name = "revolt-quark" -version = "0.5.17" +version = "0.5.18" dependencies = [ "async-lock", "async-recursion", @@ -2874,6 +2874,7 @@ dependencies = [ "once_cell", "optional_struct", "pretty_env_logger", + "rand 0.8.5", "redis-kiss", "regex", "reqwest", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 9aaf7c72..3ba4f82a 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.5.17" +version = "0.5.18" license = "AGPL-3.0-or-later" edition = "2021" diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index c488923d..de9bdc53 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.5.17" +version = "0.5.18" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" diff --git a/crates/quark/Cargo.toml b/crates/quark/Cargo.toml index 916d9b33..0467d89b 100644 --- a/crates/quark/Cargo.toml +++ b/crates/quark/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-quark" -version = "0.5.17" +version = "0.5.18" license = "AGPL-3.0-or-later" edition = "2021" @@ -57,6 +57,7 @@ log = "0.4.14" pretty_env_logger = "0.4.0" # Util +rand = "0.8.5" ulid = "0.5.0" regex = "1.5.5" nanoid = "0.4.0" diff --git a/crates/quark/src/presence/entry.rs b/crates/quark/src/presence/entry.rs index 4d43cd2e..9fa61c6d 100644 --- a/crates/quark/src/presence/entry.rs +++ b/crates/quark/src/presence/entry.rs @@ -1,64 +1,12 @@ use std::env; -use serde::{Deserialize, Serialize}; use once_cell::sync::Lazy; -pub static REGION_ID: Lazy = Lazy::new(|| env::var("REGION_ID") - .unwrap_or_else(|_| "0".to_string()) - .parse() - .unwrap()); +pub static REGION_ID: Lazy = Lazy::new(|| { + env::var("REGION_ID") + .unwrap_or_else(|_| "0".to_string()) + .parse() + .unwrap() +}); pub static REGION_KEY: Lazy = Lazy::new(|| format!("region{}", &*REGION_ID)); - -/// Compact presence information for a user -#[derive(Serialize, Deserialize, Debug)] -pub struct PresenceEntry { - /// Region this session exists in - /// - /// We can have up to 65535 regions - pub region_id: u16, - - /// Unique session ID - pub session_id: u8, - - /// Known flags about session - pub flags: u8, -} - -impl PresenceEntry { - /// Create a new presence entry from a given session ID and known flags - pub fn from(session_id: u8, flags: u8) -> Self { - Self { - region_id: *REGION_ID, - session_id, - flags, - } - } -} - -pub trait PresenceOp { - /// Find next available session ID - fn find_next_id(&self) -> u8; -} - -impl PresenceOp for Vec { - fn find_next_id(&self) -> u8 { - // O(n^2) scan algorithm - // should be relatively fast at low numbers anyways - for i in 0..255 { - let mut found = false; - for entry in self { - if entry.session_id == i { - found = true; - break; - } - } - - if !found { - return i; - } - } - - 255 - } -} diff --git a/crates/quark/src/presence/mod.rs b/crates/quark/src/presence/mod.rs index 73c14b6a..827ec193 100644 --- a/crates/quark/src/presence/mod.rs +++ b/crates/quark/src/presence/mod.rs @@ -2,87 +2,77 @@ use std::collections::HashSet; use redis_kiss::{get_connection, AsyncCommands}; +use rand::Rng; mod entry; mod operations; -use entry::{PresenceEntry, PresenceOp}; use operations::{ - __add_to_set_sessions, __delete_key_presence_entry, __get_key_presence_entry, - __get_set_sessions, __remove_from_set_sessions, __set_key_presence_entry, + __add_to_set_string, __add_to_set_u32, __delete_key, __get_set_members_as_string, + __get_set_size, __remove_from_set_string, __remove_from_set_u32, }; -use crate::presence::operations::__delete_set_sessions; - use self::entry::REGION_KEY; /// Create a new presence session, returns the ID of this session -pub async fn presence_create_session(user_id: &str, flags: u8) -> (bool, u8) { +pub async fn presence_create_session(user_id: &str, flags: u8) -> (bool, u32) { info!("Creating a presence session for {user_id} with flags {flags}"); - // Try to find the presence entry for this user. - let mut conn = get_connection().await.unwrap(); - let mut entry: Vec = __get_key_presence_entry(&mut conn, user_id) - .await - .unwrap_or_default(); + if let Ok(mut conn) = get_connection().await { + // Check whether this is the first session + let was_empty = __get_set_size(&mut conn, user_id).await == 0; - // Return whether this was the first session. - let was_empty = entry.is_empty(); - info!("User ID {} just came online.", &user_id); + // A session ID is comprised of random data and any flags ORed to the end + let session_id = { + let mut rng = rand::thread_rng(); + (rng.gen::() ^ 1) | (flags as u32 & 1) + }; - // Generate session ID and push new entry. - let session_id = entry.find_next_id(); - entry.push(PresenceEntry::from(session_id, flags)); - __set_key_presence_entry(&mut conn, user_id, entry).await; + // Add session to user's sessions and to the region + __add_to_set_u32(&mut conn, user_id, session_id).await; + __add_to_set_string(&mut conn, ®ION_KEY, &format!("{user_id}:{session_id}")).await; + info!("Created session for {user_id}, assigned them a session ID of {session_id}."); - // Add to region set in case of failure. - __add_to_set_sessions(&mut conn, ®ION_KEY, user_id, session_id).await; - (was_empty, session_id) + (was_empty, session_id) + } else { + // Fail through + (false, 0) + } } /// Delete existing presence session -pub async fn presence_delete_session(user_id: &str, session_id: u8) -> bool { +pub async fn presence_delete_session(user_id: &str, session_id: u32) -> bool { presence_delete_session_internal(user_id, session_id, false).await } /// Delete existing presence session (but also choose whether to skip region) async fn presence_delete_session_internal( user_id: &str, - session_id: u8, + session_id: u32, skip_region: bool, ) -> bool { info!("Deleting presence session for {user_id} with id {session_id}"); - // Return whether this was the last session. - let mut is_empty = false; + if let Ok(mut conn) = get_connection().await { + // Remove the session + __remove_from_set_u32(&mut conn, user_id, session_id).await; - // Only continue if we can actually find one. - let mut conn = get_connection().await.unwrap(); - let entry: Option> = __get_key_presence_entry(&mut conn, user_id).await; - if let Some(entry) = entry { - let entries = entry - .into_iter() - .filter(|x| x.session_id != session_id) - .collect::>(); - - // If entry is empty, then just delete it. - if entries.is_empty() { - __delete_key_presence_entry(&mut conn, user_id).await; - is_empty = true; - } else { - __set_key_presence_entry(&mut conn, user_id, entries).await; - } - - // Remove from region set. + // Remove from the region if !skip_region { - __remove_from_set_sessions(&mut conn, ®ION_KEY, user_id, session_id).await; + __remove_from_set_string(&mut conn, ®ION_KEY, &format!("{user_id}:{session_id}")) + .await; } - } - if is_empty { - info!("User ID {} just went offline.", &user_id); - } + // Return whether this was the last session + let is_empty = __get_set_size(&mut conn, user_id).await == 0; + if is_empty { + info!("User ID {} just went offline.", &user_id); + } - is_empty + is_empty + } else { + // Fail through + false + } } /// Check whether a given user ID is online @@ -102,6 +92,10 @@ pub async fn presence_filter_online(user_ids: &'_ [String]) -> HashSet { return set; } + // NOTE: at the point that we need mobile indicators + // you can interpret the data here and return a new data + // structure like HashMap + // We need to handle a special case where only one is present // as for some reason or another, Redis does not like us sending // a list of just one ID to the server. @@ -136,7 +130,7 @@ pub async fn presence_clear_region(region_id: Option<&str>) { let region_id = region_id.unwrap_or(&*REGION_KEY); let mut conn = get_connection().await.expect("Redis connection"); - let sessions = __get_set_sessions(&mut conn, region_id).await; + let sessions = __get_set_members_as_string(&mut conn, region_id).await; if !sessions.is_empty() { info!( "Cleaning up {} sessions, this may take a while...", @@ -155,7 +149,7 @@ pub async fn presence_clear_region(region_id: Option<&str>) { } // Then clear the set in Redis. - __delete_set_sessions(&mut conn, region_id).await; + __delete_key(&mut conn, region_id).await; info!("Clean up complete."); } diff --git a/crates/quark/src/presence/operations.rs b/crates/quark/src/presence/operations.rs index b97b65c7..0196fd08 100644 --- a/crates/quark/src/presence/operations.rs +++ b/crates/quark/src/presence/operations.rs @@ -1,57 +1,42 @@ use redis_kiss::{AsyncCommands, Conn}; -use super::entry::PresenceEntry; - -/// Set presence entry by given ID -pub async fn __set_key_presence_entry(conn: &mut Conn, id: &str, data: Vec) { - let _: Option<()> = conn.set(id, bincode::serialize(&data).unwrap()).await.ok(); +/// Add to set (string) +pub async fn __add_to_set_string(conn: &mut Conn, key: &str, value: &str) { + let _: Option<()> = conn.sadd(key, value).await.ok(); } -/// Delete presence entry by given ID -pub async fn __delete_key_presence_entry(conn: &mut Conn, id: &str) { - let _: Option<()> = conn.del(id).await.ok(); +/// Add to set (u32) +pub async fn __add_to_set_u32(conn: &mut Conn, key: &str, value: u32) { + let _: Option<()> = conn.sadd(key, value).await.ok(); } -/// Get presence entry by given ID -pub async fn __get_key_presence_entry(conn: &mut Conn, id: &str) -> Option> { - conn.get::<_, Option>>(id) +/// Remove from set (string) +pub async fn __remove_from_set_string(conn: &mut Conn, key: &str, value: &str) { + let _: Option<()> = conn.srem(key, value).await.ok(); +} + +/// Remove from set (u32) +pub async fn __remove_from_set_u32(conn: &mut Conn, key: &str, value: u32) { + let _: Option<()> = conn.srem(key, value).await.ok(); +} + +/// Get set members as string +pub async fn __get_set_members_as_string(conn: &mut Conn, key: &str) -> Vec { + conn.smembers::<_, Vec>(key) .await - .unwrap() - .map(|entry| bincode::deserialize(&entry[..]).unwrap()) + .expect("could not get set members as string") } -/// Add to region session set -pub async fn __add_to_set_sessions( - conn: &mut Conn, - region_id: &str, - user_id: &str, - session_id: u8, -) { - let _: Option<()> = conn - .sadd(region_id, format!("{user_id}:{session_id}")) +/// Get set size +pub async fn __get_set_size(conn: &mut Conn, id: &str) -> u32 { + conn.scard::<_, u32>(id) .await - .ok(); + .expect("could not get set size") } -/// Remove from region session set -pub async fn __remove_from_set_sessions( - conn: &mut Conn, - region_id: &str, - user_id: &str, - session_id: u8, -) { - let _: Option<()> = conn - .srem(region_id, format!("{user_id}:{session_id}")) +/// Delete key by id +pub async fn __delete_key(conn: &mut Conn, id: &str) { + conn.del::<_, ()>(id) .await - .ok(); -} - -/// Get region session set as list -pub async fn __get_set_sessions(conn: &mut Conn, region_id: &str) -> Vec { - conn.smembers::<_, Vec>(region_id).await.unwrap() -} - -/// Delete region session set -pub async fn __delete_set_sessions(conn: &mut Conn, region_id: &str) { - conn.del::<_, ()>(region_id).await.unwrap(); + .expect("could not delete key by id"); } diff --git a/docker-compose.db.yml b/docker-compose.db.yml index 4801e7d4..222a1790 100644 --- a/docker-compose.db.yml +++ b/docker-compose.db.yml @@ -1,5 +1,11 @@ version: "3.3" services: + # Redis + redis: + image: eqalpha/keydb + ports: + - "6379:6379" + # MongoDB database: image: mongo From 03a28dbb3ec7d0577b98ac25fd2dfd70486b662d Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Fri, 21 Apr 2023 18:11:06 +0100 Subject: [PATCH 06/56] fix: use debian:bullseye image for glibc 2.29 --- crates/bonfire/Dockerfile | 2 +- crates/delta/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/bonfire/Dockerfile b/crates/bonfire/Dockerfile index 272ed03d..fb5d540f 100644 --- a/crates/bonfire/Dockerfile +++ b/crates/bonfire/Dockerfile @@ -2,7 +2,7 @@ FROM ghcr.io/revoltchat/base:latest AS builder # Bundle Stage -FROM debian:buster-slim +FROM debian:bullseye-slim RUN apt-get update && \ apt-get install -y ca-certificates && \ apt-get clean diff --git a/crates/delta/Dockerfile b/crates/delta/Dockerfile index 78489315..308e00db 100644 --- a/crates/delta/Dockerfile +++ b/crates/delta/Dockerfile @@ -2,7 +2,7 @@ FROM ghcr.io/revoltchat/base:latest AS builder # Bundle Stage -FROM debian:buster-slim +FROM debian:bullseye-slim RUN apt-get update && \ apt-get install -y ca-certificates && \ apt-get clean From a326cdc7361b658a946d1e6b367bd5d8535fe162 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Fri, 21 Apr 2023 18:40:00 +0100 Subject: [PATCH 07/56] refactor: remove unused import --- crates/delta/src/routes/servers/invites_fetch.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/delta/src/routes/servers/invites_fetch.rs b/crates/delta/src/routes/servers/invites_fetch.rs index 86927866..b5a77bf9 100644 --- a/crates/delta/src/routes/servers/invites_fetch.rs +++ b/crates/delta/src/routes/servers/invites_fetch.rs @@ -1,6 +1,6 @@ use revolt_quark::{ models::{Invite, User}, - perms, Db, Error, Permission, Ref, Result, + perms, Db, Permission, Ref, Result, }; use rocket::serde::json::Json; From 59a644891d64dfb214a30b3e8784d37dc6ebd413 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Fri, 21 Apr 2023 18:52:41 +0100 Subject: [PATCH 08/56] fix: collect IDs as set first to deduplicate entries --- crates/quark/src/events/impl.rs | 7 ++++--- crates/quark/src/impl/generic/channels/message.rs | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/quark/src/events/impl.rs b/crates/quark/src/events/impl.rs index 6010e6f0..ffac793e 100644 --- a/crates/quark/src/events/impl.rs +++ b/crates/quark/src/events/impl.rs @@ -99,7 +99,7 @@ impl State { let mut user = self.clone_user(); // Find all relationships to the user. - let mut user_ids: Vec = user + let mut user_ids: HashSet = user .relations .as_ref() .map(|arr| arr.iter().map(|x| x.id.to_string()).collect()) @@ -128,14 +128,15 @@ impl State { for channel in &channels { match channel { Channel::DirectMessage { recipients, .. } | Channel::Group { recipients, .. } => { - user_ids.append(&mut recipients.clone()); + user_ids.extend(&mut recipients.clone().into_iter()); } _ => {} } } // Fetch presence data for known users. - let online_ids = presence_filter_online(&user_ids).await; + let online_ids = + presence_filter_online(&user_ids.iter().cloned().collect::>()).await; user.online = Some(true); // Fetch user data. diff --git a/crates/quark/src/impl/generic/channels/message.rs b/crates/quark/src/impl/generic/channels/message.rs index 8e4a0e6e..aaee8e2d 100644 --- a/crates/quark/src/impl/generic/channels/message.rs +++ b/crates/quark/src/impl/generic/channels/message.rs @@ -305,12 +305,12 @@ impl IntoUsers for Message { impl IntoUsers for Vec { fn get_user_ids(&self) -> Vec { - let mut ids = vec![]; + let mut ids = HashSet::new(); for message in self { - ids.append(&mut message.get_user_ids()); + ids.extend(&mut message.get_user_ids().into_iter()); } - ids + ids.into_iter().collect() } } From c5880596a4ca63a1564bf97e25437c20f7018617 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Fri, 21 Apr 2023 19:36:25 +0100 Subject: [PATCH 09/56] refactor: use a separate online user ID set --- crates/quark/src/presence/entry.rs | 1 + crates/quark/src/presence/mod.rs | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/quark/src/presence/entry.rs b/crates/quark/src/presence/entry.rs index 9fa61c6d..c177a5ab 100644 --- a/crates/quark/src/presence/entry.rs +++ b/crates/quark/src/presence/entry.rs @@ -10,3 +10,4 @@ pub static REGION_ID: Lazy = Lazy::new(|| { }); pub static REGION_KEY: Lazy = Lazy::new(|| format!("region{}", &*REGION_ID)); +pub static ONLINE_SET: &str = "online"; diff --git a/crates/quark/src/presence/mod.rs b/crates/quark/src/presence/mod.rs index 827ec193..6505ee3c 100644 --- a/crates/quark/src/presence/mod.rs +++ b/crates/quark/src/presence/mod.rs @@ -11,7 +11,7 @@ use operations::{ __get_set_size, __remove_from_set_string, __remove_from_set_u32, }; -use self::entry::REGION_KEY; +use self::entry::{ONLINE_SET, REGION_KEY}; /// Create a new presence session, returns the ID of this session pub async fn presence_create_session(user_id: &str, flags: u8) -> (bool, u32) { @@ -29,6 +29,7 @@ pub async fn presence_create_session(user_id: &str, flags: u8) -> (bool, u32) { // Add session to user's sessions and to the region __add_to_set_u32(&mut conn, user_id, session_id).await; + __add_to_set_string(&mut conn, ONLINE_SET, user_id).await; __add_to_set_string(&mut conn, ®ION_KEY, &format!("{user_id}:{session_id}")).await; info!("Created session for {user_id}, assigned them a session ID of {session_id}."); @@ -65,6 +66,7 @@ async fn presence_delete_session_internal( // Return whether this was the last session let is_empty = __get_set_size(&mut conn, user_id).await == 0; if is_empty { + __remove_from_set_string(&mut conn, ONLINE_SET, user_id).await; info!("User ID {} just went offline.", &user_id); } @@ -109,14 +111,19 @@ pub async fn presence_filter_online(user_ids: &'_ [String]) -> HashSet { // Otherwise, go ahead as normal. if let Ok(mut conn) = get_connection().await { - let data: Vec>> = conn.get(user_ids).await.unwrap_or_default(); + // Ok so, if this breaks, that means we've lost the Redis patch which adds SMISMEMBER + // Currently it's patched in through a forked repository, investigate what happen to it + let data: Vec = conn + .sismember("online", user_ids) + .await + .expect("this shouldn't happen, please read this code! presence/mod.rs"); if data.is_empty() { return set; } // We filter known values to figure out who is online. for i in 0..user_ids.len() { - if data[i].is_some() { + if data[i] { set.insert(user_ids[i].to_string()); } } From 43e45aef3f285e2b487be088cf7f43c31f6d7d30 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Fri, 21 Apr 2023 19:37:33 +0100 Subject: [PATCH 10/56] fix: patch redis-rs for `SMISMEMBER` support --- Cargo.lock | 89 +++++++++++++++++++++------------------ Cargo.toml | 4 ++ crates/bonfire/Cargo.toml | 2 +- crates/delta/Cargo.toml | 7 +-- crates/quark/Cargo.toml | 4 +- 5 files changed, 55 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a310ecf..19e0577f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -871,12 +871,6 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" -[[package]] -name = "dtoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" - [[package]] name = "dyn-clone" version = "1.0.5" @@ -1831,6 +1825,27 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "metrics" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e52eb6380b6d2a10eb3434aec0885374490f5b82c8aaf5cd487a183c98be834" +dependencies = [ + "ahash", + "metrics-macros", +] + +[[package]] +name = "metrics-macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e30813093f757be5cf21e50389a24dc7dbb22c49f23b7e8f51d69b508a5ffa" +dependencies = [ + "proc-macro2", + "quote 1.0.18", + "syn 1.0.107", +] + [[package]] name = "mime" version = "0.3.16" @@ -1897,9 +1912,9 @@ dependencies = [ [[package]] name = "mobc" -version = "0.7.3" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f76d2f2e2dcbb00a8d3b2b09f026a74a82693ea52cd071647aa6cfa7f1ff37e" +checksum = "fc79c4a77e312fee9c7bd4b957c12ad1196db73c4a81e5c0b13f02083c4f7f2f" dependencies = [ "async-std", "async-trait", @@ -1908,17 +1923,21 @@ dependencies = [ "futures-timer", "futures-util", "log", + "metrics", + "thiserror", "tokio 1.18.2", + "tracing", + "tracing-subscriber", ] [[package]] name = "mobc-redis" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b5db77b37c9224d5b9949b214041ea3e1c15b6b1e5dd24a5acb8e73975d6d6" +checksum = "7bd8e2fd6bf7e35263b86662e663a9496a0352ceddd413b6c33313c36d5068fd" dependencies = [ "mobc", - "redis 0.19.0", + "redis 0.22.3", ] [[package]] @@ -2624,57 +2643,55 @@ dependencies = [ [[package]] name = "redis" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a6ddfecac9391fed21cce10e83c65fa4abafd77df05c98b1c647c65374ce9b3" +version = "0.22.3" +source = "git+https://github.com/insertish/redis-rs?rev=6575a6b1c09eb8c9cc7f0082d95fe6b8f903c4d7#6575a6b1c09eb8c9cc7f0082d95fe6b8f903c4d7" dependencies = [ "async-std", "async-trait", "bytes 1.1.0", "combine", - "dtoa", "futures-util", - "itoa 0.4.8", + "itoa 1.0.2", "percent-encoding", "pin-project-lite 0.2.9", - "sha1", + "ryu", + "sha1_smol", "tokio 1.18.2", - "tokio-util 0.6.10", + "tokio-util 0.7.2", "url", ] [[package]] name = "redis" -version = "0.21.5" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a80b5f38d7f5a020856a0e16e40a9cfabf88ae8f0e4c2dcd8a3114c1e470852" +checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f" dependencies = [ "async-std", "async-trait", "bytes 1.1.0", "combine", - "dtoa", "futures-util", - "itoa 0.4.8", + "itoa 1.0.2", "percent-encoding", "pin-project-lite 0.2.9", - "sha1", + "ryu", "tokio 1.18.2", - "tokio-util 0.6.10", + "tokio-util 0.7.2", "url", ] [[package]] name = "redis-kiss" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4f2f48fb776a308331c4a1ecb3a7b7b99f4d46c6beb2fc03db2da0f63fcad6" +checksum = "58605cfd83a2146161de13bb253f24db25eef6919b35403fc90f508ef89bec92" dependencies = [ "bincode", "lazy_static", "mobc", "mobc-redis", - "redis 0.21.5", + "redis 0.23.0", "rmp-serde", "serde", "serde_json", @@ -2792,7 +2809,7 @@ dependencies = [ [[package]] name = "revolt-bonfire" -version = "0.5.18" +version = "0.5.19" dependencies = [ "async-std", "async-tungstenite", @@ -2808,7 +2825,7 @@ dependencies = [ [[package]] name = "revolt-delta" -version = "0.5.18" +version = "0.5.19" dependencies = [ "async-channel", "async-std", @@ -2823,12 +2840,9 @@ dependencies = [ "linkify 0.6.0", "log", "lru", - "mobc", - "mobc-redis", "nanoid", "num_enum", "once_cell", - "redis 0.21.5", "regex", "reqwest", "revolt-quark", @@ -2847,7 +2861,7 @@ dependencies = [ [[package]] name = "revolt-quark" -version = "0.5.18" +version = "0.5.19" dependencies = [ "async-lock", "async-recursion", @@ -3471,15 +3485,6 @@ dependencies = [ "digest 0.10.3", ] -[[package]] -name = "sha1" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" -dependencies = [ - "sha1_smol", -] - [[package]] name = "sha1_smol" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index c66a4d73..68903beb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,2 +1,6 @@ [workspace] members = ["crates/*"] + +[patch.crates-io] +# mobc-redis = { git = "https://github.com/insertish/mobc", rev = "8b880bb59f2ba80b4c7bc40c649c113d8857a186" } +redis = { git = "https://github.com/insertish/redis-rs", rev = "6575a6b1c09eb8c9cc7f0082d95fe6b8f903c4d7" } diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 3ba4f82a..14dca258 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-bonfire" -version = "0.5.18" +version = "0.5.19" license = "AGPL-3.0-or-later" edition = "2021" diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index de9bdc53..84d4ffdd 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-delta" -version = "0.5.18" +version = "0.5.19" license = "AGPL-3.0-or-later" authors = ["Paul Makles "] edition = "2018" @@ -43,11 +43,6 @@ async-std = { version = "1.8.0", features = ["tokio1", "tokio02", "attributes"] # internal util lettre = "0.10.0-alpha.4" -# redis -redis = { version = "0.21.2", features = ["async-std-comp"] } -mobc = { version = "0.7.3" } -mobc-redis = { version = "0.7.0", default-features = false, features = ["async-std-comp"] } - # web rocket = { version = "0.5.0-rc.2", default-features = false, features = ["json"] } rocket_empty = { version = "0.1.1", features = ["schema"] } diff --git a/crates/quark/Cargo.toml b/crates/quark/Cargo.toml index 0467d89b..43a22a9d 100644 --- a/crates/quark/Cargo.toml +++ b/crates/quark/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-quark" -version = "0.5.18" +version = "0.5.19" license = "AGPL-3.0-or-later" edition = "2021" @@ -42,7 +42,7 @@ revolt_okapi = "0.9.1" revolt_rocket_okapi = { version = "0.9.1", features = [ "swagger" ] } # Databases -redis-kiss = { version = "0.1.3" } +redis-kiss = { version = "0.1.4" } mongodb = { optional = true, version = "2.1.0", default-features = false } # Async From f2bb388b3b38855d3856c4886a47fbbd09b35ad1 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 00:01:56 +0100 Subject: [PATCH 11/56] refactor: minimum implementation of revolt-models and revolt-database --- Cargo.lock | 133 ++++++---- Cargo.toml | 2 +- crates/core/database/Cargo.toml | 22 ++ crates/core/database/src/drivers/dummy.rs | 4 + crates/core/database/src/drivers/mod.rs | 5 + crates/core/database/src/drivers/mongodb.rs | 228 ++++++++++++++++++ crates/core/database/src/lib.rs | 66 +++++ crates/core/models/Cargo.toml | 33 +++ crates/core/models/examples/migrate.rs | 8 + .../core/models/src/admin_migrations/mod.rs | 9 + .../core/models/src/admin_migrations/model.rs | 10 + .../models/src/admin_migrations/ops/dummy.rs | 11 + .../models/src/admin_migrations/ops/mod.rs | 8 + .../src/admin_migrations/ops/mongodb.rs} | 7 +- .../src/admin_migrations/ops/mongodb}/init.rs | 7 +- .../admin_migrations/ops/mongodb}/scripts.rs | 18 +- crates/core/models/src/lib.rs | 40 +++ crates/delta/Cargo.toml | 4 + crates/delta/src/main.rs | 11 +- .../quark/src/impl/dummy/admin/migrations.rs | 11 - crates/quark/src/impl/dummy/mod.rs | 1 - .../src/impl/generic/admin/migrations.rs | 1 - crates/quark/src/impl/generic/mod.rs | 4 - crates/quark/src/impl/mongo/mod.rs | 1 - crates/quark/src/models/admin/migrations.rs | 11 - crates/quark/src/models/mod.rs | 2 - crates/quark/src/traits/admin/migrations.rs | 6 - crates/quark/src/traits/mod.rs | 3 - 28 files changed, 558 insertions(+), 108 deletions(-) create mode 100644 crates/core/database/Cargo.toml create mode 100644 crates/core/database/src/drivers/dummy.rs create mode 100644 crates/core/database/src/drivers/mod.rs create mode 100644 crates/core/database/src/drivers/mongodb.rs create mode 100644 crates/core/database/src/lib.rs create mode 100644 crates/core/models/Cargo.toml create mode 100644 crates/core/models/examples/migrate.rs create mode 100644 crates/core/models/src/admin_migrations/mod.rs create mode 100644 crates/core/models/src/admin_migrations/model.rs create mode 100644 crates/core/models/src/admin_migrations/ops/dummy.rs create mode 100644 crates/core/models/src/admin_migrations/ops/mod.rs rename crates/{quark/src/impl/mongo/admin/migrations.rs => core/models/src/admin_migrations/ops/mongodb.rs} (75%) rename crates/{quark/src/impl/mongo/admin/migrations => core/models/src/admin_migrations/ops/mongodb}/init.rs (97%) rename crates/{quark/src/impl/mongo/admin/migrations => core/models/src/admin_migrations/ops/mongodb}/scripts.rs (98%) create mode 100644 crates/core/models/src/lib.rs delete mode 100644 crates/quark/src/impl/dummy/admin/migrations.rs delete mode 100644 crates/quark/src/impl/generic/admin/migrations.rs delete mode 100644 crates/quark/src/models/admin/migrations.rs delete mode 100644 crates/quark/src/traits/admin/migrations.rs diff --git a/Cargo.lock b/Cargo.lock index 19e0577f..575a3ac8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,7 +105,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" dependencies = [ - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -199,13 +199,13 @@ dependencies = [ [[package]] name = "async-recursion" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cda8f4bcc10624c4e85bc66b3f452cca98cfa5ca002dc83a16aad2367641bea" +checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", - "quote 1.0.18", - "syn 1.0.107", + "quote 1.0.26", + "syn 2.0.15", ] [[package]] @@ -269,7 +269,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -286,7 +286,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -715,7 +715,7 @@ version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c" dependencies = [ - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -747,7 +747,7 @@ dependencies = [ "fnv", "ident_case", "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "strsim", "syn 1.0.107", ] @@ -759,7 +759,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -808,7 +808,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -829,7 +829,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "123c73e7a6e51b05c75fe1a1b2f4e241399ea5740ed810b0e3e6cacd9db5e7b2" dependencies = [ "devise_core", - "quote 1.0.18", + "quote 1.0.26", ] [[package]] @@ -841,7 +841,7 @@ dependencies = [ "bitflags", "proc-macro2", "proc-macro2-diagnostics", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -909,7 +909,7 @@ checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" dependencies = [ "heck", "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -929,7 +929,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "828de45d0ca18782232dfb8f3ea9cc428e8ced380eb26a520baaacfc70de39ce" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -1113,7 +1113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -1204,7 +1204,7 @@ checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" dependencies = [ "proc-macro-error", "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -1842,7 +1842,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49e30813093f757be5cf21e50389a24dc7dbb22c49f23b7e8f51d69b508a5ffa" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -2110,7 +2110,7 @@ checksum = "3b0498641e53dd6ac1a4f22547548caa6864cc4933784319cd1775271c5a46ce" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -2163,7 +2163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -2268,7 +2268,7 @@ checksum = "82a5ca643c2303ecb740d506539deba189e16f2754040a42901cd8105d0282d0" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -2306,7 +2306,7 @@ dependencies = [ "pest", "pest_meta", "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -2337,7 +2337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -2424,7 +2424,7 @@ checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", "version_check", ] @@ -2436,15 +2436,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "version_check", ] [[package]] name = "proc-macro2" -version = "1.0.50" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" dependencies = [ "unicode-ident", ] @@ -2456,7 +2456,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bf29726d67464d49fa6224a1d07936a8c08bb3fba727c7493f6cf1616fdaada" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", "version_check", "yansi", @@ -2482,9 +2482,9 @@ checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" [[package]] name = "quote" -version = "1.0.18" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -2722,7 +2722,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a043824e29c94169374ac5183ac0ed43f5724dc4556b19568007486bd840fa1f" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -2823,6 +2823,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "revolt-database" +version = "0.1.0" +dependencies = [ + "async-recursion", + "futures", + "mongodb", + "serde", + "serde_json", +] + [[package]] name = "revolt-delta" version = "0.5.19" @@ -2845,6 +2856,8 @@ dependencies = [ "once_cell", "regex", "reqwest", + "revolt-database", + "revolt-models", "revolt-quark", "revolt_rocket_okapi", "rocket", @@ -2859,6 +2872,19 @@ dependencies = [ "vergen", ] +[[package]] +name = "revolt-models" +version = "0.1.0" +dependencies = [ + "async-std", + "async-trait", + "authifier", + "futures", + "log", + "revolt-database", + "serde", +] + [[package]] name = "revolt-quark" version = "0.5.19" @@ -2943,7 +2969,7 @@ checksum = "cc6620569d8ac8f0a1690fcca13f488503807a60e96ebf729749b59aca1dbef9" dependencies = [ "darling", "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "rocket_http", "syn 1.0.107", ] @@ -3050,7 +3076,7 @@ dependencies = [ "glob", "indexmap", "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "rocket_http", "syn 1.0.107", "unicode-xid 0.2.3", @@ -3225,7 +3251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af4d7e1b012cb3d9129567661a63755ea4b8a7386d339dc945ae187e403c6743" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "serde_derive_internals", "syn 1.0.107", ] @@ -3401,7 +3427,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -3412,15 +3438,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "indexmap", "itoa 1.0.2", @@ -3458,7 +3484,7 @@ checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ "darling", "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -3633,7 +3659,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +dependencies = [ + "proc-macro2", + "quote 1.0.26", "unicode-ident", ] @@ -3705,7 +3742,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -3814,7 +3851,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -3950,7 +3987,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -4080,7 +4117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", ] @@ -4290,7 +4327,7 @@ dependencies = [ "lazy_static", "proc-macro-error", "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "regex", "syn 1.0.107", "validator_types", @@ -4400,7 +4437,7 @@ dependencies = [ "lazy_static", "log", "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", "wasm-bindgen-shared", ] @@ -4423,7 +4460,7 @@ version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" dependencies = [ - "quote 1.0.18", + "quote 1.0.26", "wasm-bindgen-macro-support", ] @@ -4434,7 +4471,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" dependencies = [ "proc-macro2", - "quote 1.0.18", + "quote 1.0.26", "syn 1.0.107", "wasm-bindgen-backend", "wasm-bindgen-shared", diff --git a/Cargo.toml b/Cargo.toml index 68903beb..89be2f2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/*"] +members = ["crates/delta", "crates/bonfire", "crates/quark", "crates/core/*"] [patch.crates-io] # mobc-redis = { git = "https://github.com/insertish/mobc", rev = "8b880bb59f2ba80b4c7bc40c649c113d8857a186" } diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml new file mode 100644 index 00000000..57b92202 --- /dev/null +++ b/crates/core/database/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "revolt-database" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +mongodb = [ "dep:mongodb" ] +default = [ "mongodb" ] + +[dependencies] +# Serialisation +serde_json = "1" +serde = { version = "1", features = ["derive"] } + +# Database +mongodb = { optional = true, version = "2.1.0", default-features = false } + +# Async Language Features +futures = "0.3.19" +async-recursion = "1.0.4" diff --git a/crates/core/database/src/drivers/dummy.rs b/crates/core/database/src/drivers/dummy.rs new file mode 100644 index 00000000..198bf59c --- /dev/null +++ b/crates/core/database/src/drivers/dummy.rs @@ -0,0 +1,4 @@ +database_derived!( + /// Dummy implementation + pub struct DummyDb {} +); diff --git a/crates/core/database/src/drivers/mod.rs b/crates/core/database/src/drivers/mod.rs new file mode 100644 index 00000000..19bd06c6 --- /dev/null +++ b/crates/core/database/src/drivers/mod.rs @@ -0,0 +1,5 @@ +mod dummy; +mod mongodb; + +pub use self::dummy::*; +pub use self::mongodb::*; diff --git a/crates/core/database/src/drivers/mongodb.rs b/crates/core/database/src/drivers/mongodb.rs new file mode 100644 index 00000000..6fb99135 --- /dev/null +++ b/crates/core/database/src/drivers/mongodb.rs @@ -0,0 +1,228 @@ +use std::collections::HashMap; +use std::ops::Deref; + +use futures::StreamExt; +use mongodb::bson::{doc, to_document, Document}; +use mongodb::error::Result; +use mongodb::options::{FindOneOptions, FindOptions}; +use mongodb::results::{DeleteResult, InsertOneResult, UpdateResult}; +use serde::de::DeserializeOwned; +use serde::Serialize; + +database_derived!( + #[cfg(feature = "mongodb")] + /// MongoDB implementation + pub struct MongoDb(pub ::mongodb::Client); +); + +impl Deref for MongoDb { + type Target = mongodb::Client; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[allow(dead_code)] +impl MongoDb { + /// Get the Revolt database + pub fn db(&self) -> mongodb::Database { + self.database("revolt") + } + + /// Get a collection by its name + pub fn col(&self, collection: &str) -> mongodb::Collection { + self.db().collection(collection) + } + + /// Insert one document into a collection + async fn insert_one( + &self, + collection: &'static str, + document: T, + ) -> Result { + self.col::(collection).insert_one(document, None).await + } + + /// Find multiple documents in a collection with options + async fn find_with_options( + &self, + collection: &'static str, + projection: Document, + options: O, + ) -> Result> + where + O: Into>, + { + Ok(self + .col::(collection) + .find(projection, options) + .await? + .filter_map(|s| async { + if cfg!(debug_assertions) { + // Hard fail on invalid documents + Some(s.unwrap()) + } else { + s.ok() + } + }) + .collect::>() + .await) + } + + /// Find multiple documents in a collection + async fn find( + &self, + collection: &'static str, + projection: Document, + ) -> Result> { + self.find_with_options(collection, projection, None).await + } + + /// Find one document with options + async fn find_one_with_options( + &self, + collection: &'static str, + projection: Document, + options: O, + ) -> Result> + where + O: Into>, + { + self.col::(collection) + .find_one(projection, options) + .await + } + + /// Find one document + async fn find_one( + &self, + collection: &'static str, + projection: Document, + ) -> Result> { + self.find_one_with_options(collection, projection, None) + .await + } + + /// Find one document by its ID + async fn find_one_by_id( + &self, + collection: &'static str, + id: &str, + ) -> Result> { + self.find_one( + collection, + doc! { + "_id": id + }, + ) + .await + } + + /// Update one document given a projection, partial document, and list of paths to unset + async fn update_one( + &self, + collection: &'static str, + projection: Document, + partial: T, + remove: Vec<&dyn IntoDocumentPath>, + prefix: P, + ) -> Result + where + P: Into>, + { + let prefix = prefix.into(); + + let mut unset = doc! {}; + for field in remove { + if let Some(path) = field.as_path() { + if let Some(prefix) = &prefix { + unset.insert(prefix.to_owned() + path, 1_i32); + } else { + unset.insert(path, 1_i32); + } + } + } + + let query = doc! { + "$unset": unset, + "$set": if let Some(prefix) = &prefix { + to_document(&prefix_keys(&partial, prefix)) + } else { + to_document(&partial) + }? + }; + + self.col::(collection) + .update_one(projection, query, None) + .await + } + + /// Update one document given an ID, partial document, and list of paths to unset + async fn update_one_by_id( + &self, + collection: &'static str, + id: &str, + partial: T, + remove: Vec<&dyn IntoDocumentPath>, + prefix: P, + ) -> Result + where + P: Into>, + { + self.update_one( + collection, + doc! { + "_id": id + }, + partial, + remove, + prefix, + ) + .await + } + + /// Delete one document by the given projection + async fn delete_one( + &self, + collection: &'static str, + projection: Document, + ) -> Result { + self.col::(collection) + .delete_one(projection, None) + .await + } + + /// Delete one document by the given ID + async fn delete_one_by_id(&self, collection: &'static str, id: &str) -> Result { + self.delete_one( + collection, + doc! { + "_id": id + }, + ) + .await + } +} + +/// Just a string ID struct +#[derive(Deserialize)] +pub struct DocumentId { + #[serde(rename = "_id")] + pub id: String, +} + +pub trait IntoDocumentPath: Send + Sync { + /// Create JSON key path + fn as_path(&self) -> Option<&'static str>; +} + +/// Prefix keys on an arbitrary object +pub fn prefix_keys(t: &T, prefix: &str) -> HashMap { + let v: String = serde_json::to_string(t).unwrap(); + let v: HashMap = serde_json::from_str(&v).unwrap(); + v.into_iter() + .filter(|(_k, v)| !v.is_null()) + .map(|(k, v)| (prefix.to_owned() + &k, v)) + .collect() +} diff --git a/crates/core/database/src/lib.rs b/crates/core/database/src/lib.rs new file mode 100644 index 00000000..3b926353 --- /dev/null +++ b/crates/core/database/src/lib.rs @@ -0,0 +1,66 @@ +#[macro_use] +extern crate serde; + +#[macro_use] +extern crate async_recursion; + +macro_rules! database_derived { + ( $( $item:item )+ ) => { + $( + #[derive(Clone)] + $item + )+ + }; +} + +#[cfg(feature = "mongodb")] +pub use mongodb; + +mod drivers; +pub use drivers::*; + +/// Database information to use to create a client +pub enum DatabaseInfo { + /// Auto-detect the database in use + Auto, + /// Use the mock database + Dummy, + /// Connect to MongoDB + MongoDb(String), + /// Use existing MongoDB connection + MongoDbFromClient(::mongodb::Client), +} + +/// Database +#[derive(Clone)] +pub enum Database { + /// Mock database + Dummy(DummyDb), + /// MongoDB database + MongoDb(MongoDb), +} + +impl DatabaseInfo { + /// Create a database client from the given database information + #[async_recursion] + pub async fn connect(self) -> Result { + Ok(match self { + DatabaseInfo::Auto => { + if let Ok(uri) = std::env::var("MONGODB") { + return DatabaseInfo::MongoDb(uri).connect().await; + } + + DatabaseInfo::Dummy.connect().await? + } + DatabaseInfo::Dummy => Database::Dummy(DummyDb {}), + DatabaseInfo::MongoDb(uri) => { + let client = mongodb::Client::with_uri_str(uri) + .await + .map_err(|_| "Failed to init db connection.".to_string())?; + + Database::MongoDb(MongoDb(client)) + } + DatabaseInfo::MongoDbFromClient(client) => Database::MongoDb(MongoDb(client)), + }) + } +} diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml new file mode 100644 index 00000000..13cf041a --- /dev/null +++ b/crates/core/models/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "revolt-models" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +serde = [ "dep:serde" ] +async-std-runtime = [ "async-std" ] +database = [ "revolt-database", "revolt-database/default", "authifier/database-mongodb" ] + +default = [ "serde", "async-std-runtime", "database" ] + +[dependencies] +# Utility +log = "*" + +# Serialisation +serde = { version = "1", features = ["derive"], optional = true } + +# Async Language Features +futures = "0.3.19" +async-trait = "0.1.51" + +# Async +async-std = { version = "1.8.0", features = ["attributes"], optional = true } + +# Peer dependencies +revolt-database = { path = "../database", optional = true, default-features = false } + +# Authifier +authifier = { version = "1.0", default-features = false } diff --git a/crates/core/models/examples/migrate.rs b/crates/core/models/examples/migrate.rs new file mode 100644 index 00000000..96f975b5 --- /dev/null +++ b/crates/core/models/examples/migrate.rs @@ -0,0 +1,8 @@ +use revolt_database::DatabaseInfo; +use revolt_models::*; + +#[async_std::main] +async fn main() { + let db = Database(DatabaseInfo::Auto.connect().await.unwrap()); + db.migrate_database().await.unwrap(); +} diff --git a/crates/core/models/src/admin_migrations/mod.rs b/crates/core/models/src/admin_migrations/mod.rs new file mode 100644 index 00000000..233a1d96 --- /dev/null +++ b/crates/core/models/src/admin_migrations/mod.rs @@ -0,0 +1,9 @@ +mod model; + +pub use model::*; + +#[cfg(feature = "database")] +mod ops; + +#[cfg(feature = "database")] +pub use ops::*; diff --git a/crates/core/models/src/admin_migrations/model.rs b/crates/core/models/src/admin_migrations/model.rs new file mode 100644 index 00000000..8f6dcbbc --- /dev/null +++ b/crates/core/models/src/admin_migrations/model.rs @@ -0,0 +1,10 @@ +auto_derived!( + /// Document representing migration information + pub struct MigrationInfo { + /// Unique Id + #[serde(rename = "_id")] + pub id: i32, + /// Current database revision + pub revision: i32, + } +); diff --git a/crates/core/models/src/admin_migrations/ops/dummy.rs b/crates/core/models/src/admin_migrations/ops/dummy.rs new file mode 100644 index 00000000..0db084c3 --- /dev/null +++ b/crates/core/models/src/admin_migrations/ops/dummy.rs @@ -0,0 +1,11 @@ +use revolt_database::DummyDb; + +use super::AbstractMigrations; + +#[async_trait] +impl AbstractMigrations for DummyDb { + /// Migrate the database + async fn migrate_database(&self) -> Result<(), ()> { + Ok(()) + } +} diff --git a/crates/core/models/src/admin_migrations/ops/mod.rs b/crates/core/models/src/admin_migrations/ops/mod.rs new file mode 100644 index 00000000..386434d8 --- /dev/null +++ b/crates/core/models/src/admin_migrations/ops/mod.rs @@ -0,0 +1,8 @@ +mod dummy; +mod mongodb; + +#[async_trait] +pub trait AbstractMigrations: Sync + Send { + /// Migrate the database + async fn migrate_database(&self) -> Result<(), ()>; +} diff --git a/crates/quark/src/impl/mongo/admin/migrations.rs b/crates/core/models/src/admin_migrations/ops/mongodb.rs similarity index 75% rename from crates/quark/src/impl/mongo/admin/migrations.rs rename to crates/core/models/src/admin_migrations/ops/mongodb.rs index 943ca78f..85086830 100644 --- a/crates/quark/src/impl/mongo/admin/migrations.rs +++ b/crates/core/models/src/admin_migrations/ops/mongodb.rs @@ -1,13 +1,14 @@ -use crate::{AbstractMigrations, Result}; +use revolt_database::MongoDb; -use super::super::MongoDb; +use super::AbstractMigrations; mod init; mod scripts; #[async_trait] impl AbstractMigrations for MongoDb { - async fn migrate_database(&self) -> Result<()> { + /// Migrate the database + async fn migrate_database(&self) -> Result<(), ()> { info!("Migrating the database."); let list = self diff --git a/crates/quark/src/impl/mongo/admin/migrations/init.rs b/crates/core/models/src/admin_migrations/ops/mongodb/init.rs similarity index 97% rename from crates/quark/src/impl/mongo/admin/migrations/init.rs rename to crates/core/models/src/admin_migrations/ops/mongodb/init.rs index 4501381c..37864805 100644 --- a/crates/quark/src/impl/mongo/admin/migrations/init.rs +++ b/crates/core/models/src/admin_migrations/ops/mongodb/init.rs @@ -1,9 +1,8 @@ -use crate::r#impl::mongo::MongoDb; - use super::scripts::LATEST_REVISION; -use mongodb::bson::doc; -use mongodb::options::CreateCollectionOptions; +use revolt_database::mongodb::bson::doc; +use revolt_database::mongodb::options::CreateCollectionOptions; +use revolt_database::MongoDb; pub async fn create_database(db: &MongoDb) { info!("Creating database."); diff --git a/crates/quark/src/impl/mongo/admin/migrations/scripts.rs b/crates/core/models/src/admin_migrations/ops/mongodb/scripts.rs similarity index 98% rename from crates/quark/src/impl/mongo/admin/migrations/scripts.rs rename to crates/core/models/src/admin_migrations/ops/mongodb/scripts.rs index 168029e6..1aa1c7c3 100644 --- a/crates/quark/src/impl/mongo/admin/migrations/scripts.rs +++ b/crates/core/models/src/admin_migrations/ops/mongodb/scripts.rs @@ -1,15 +1,15 @@ -use std::{time::Duration, ops::BitXor}; +use std::{ops::BitXor, time::Duration}; -use bson::{Bson, DateTime}; use futures::StreamExt; -use mongodb::{ - bson::{doc, from_bson, from_document, to_document, Document}, - options::FindOptions, +use revolt_database::{ + mongodb::{ + bson::{doc, from_bson, from_document, to_document, Bson, DateTime, Document}, + options::FindOptions, + }, + MongoDb, }; use serde::{Deserialize, Serialize}; -use crate::{r#impl::mongo::MongoDb, Permission, DEFAULT_PERMISSION_SERVER}; - #[derive(Serialize, Deserialize)] struct MigrationInfo { _id: i32, @@ -504,7 +504,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 { update.insert( "default_permissions", // Remove Send Message permission if it wasn't originally granted - DEFAULT_PERMISSION_SERVER.bitxor(if has_send { 0 } else { Permission::SendMessage as u64}) as i64, + (4000323584).bitxor(if has_send { 0 } else { (1 << 22) as u64 }) as i64, ); if let Some(Bson::Document(mut roles)) = document.remove("roles") { @@ -563,7 +563,7 @@ pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 { doc! { "default_permissions": { "a": 0_i64, - "d": Permission::SendMessage as i64 + "d": (1 << 22) as i64 } }, ); diff --git a/crates/core/models/src/lib.rs b/crates/core/models/src/lib.rs new file mode 100644 index 00000000..bffcadf4 --- /dev/null +++ b/crates/core/models/src/lib.rs @@ -0,0 +1,40 @@ +#[cfg(feature = "serde")] +#[macro_use] +extern crate serde; + +#[macro_use] +extern crate async_trait; + +#[macro_use] +extern crate log; + +macro_rules! auto_derived { + ( $( $item:item )+ ) => { + $( + #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] + #[derive(Debug, Clone)] + $item + )+ + }; +} + +mod admin_migrations; + +pub use admin_migrations::*; + +pub struct Database(pub revolt_database::Database); + +pub trait AbstractDatabase: Sync + Send + admin_migrations::AbstractMigrations {} +impl AbstractDatabase for revolt_database::DummyDb {} +impl AbstractDatabase for revolt_database::MongoDb {} + +impl std::ops::Deref for Database { + type Target = dyn AbstractDatabase; + + fn deref(&self) -> &Self::Target { + match &self.0 { + revolt_database::Database::Dummy(dummy) => dummy, + revolt_database::Database::MongoDb(mongo) => mongo, + } + } +} diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 84d4ffdd..556f301d 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -55,5 +55,9 @@ revolt_rocket_okapi = { version = "0.9.1", features = [ "swagger" ] } # quark revolt-quark = { path = "../quark" } +# core +revolt-database = { path = "../core/database" } +revolt-models = { path = "../core/models" } + [build-dependencies] vergen = "7.5.0" diff --git a/crates/delta/src/main.rs b/crates/delta/src/main.rs index 4dc3ba38..3c2b516a 100644 --- a/crates/delta/src/main.rs +++ b/crates/delta/src/main.rs @@ -22,15 +22,19 @@ async fn rocket() -> _ { revolt_quark::variables::delta::preflight_checks(); // Setup database - let db = DatabaseInfo::Auto.connect().await.unwrap(); + let db = revolt_database::DatabaseInfo::Auto.connect().await.unwrap(); + let db = revolt_models::Database(db); db.migrate_database().await.unwrap(); + // Legacy database setup from quark + let legacy_db = DatabaseInfo::Auto.connect().await.unwrap(); + // Setup Authifier event channel let (sender, receiver) = unbounded(); // Setup Authifier let authifier = Authifier { - database: db.clone().into(), + database: legacy_db.clone().into(), config: revolt_quark::util::authifier::config(), event_channel: Some(sender), }; @@ -52,7 +56,7 @@ async fn rocket() -> _ { }); // Launch background task workers - async_std::task::spawn(revolt_quark::tasks::start_workers(db.clone())); + async_std::task::spawn(revolt_quark::tasks::start_workers(legacy_db.clone())); // Configure CORS let cors = revolt_quark::web::cors::new(); @@ -65,6 +69,7 @@ async fn rocket() -> _ { .mount("/swagger/", revolt_quark::web::swagger::routes()) .manage(authifier) .manage(db) + .manage(legacy_db) .manage(cors.clone()) .attach(revolt_quark::web::ratelimiter::RatelimitFairing) .attach(cors) diff --git a/crates/quark/src/impl/dummy/admin/migrations.rs b/crates/quark/src/impl/dummy/admin/migrations.rs deleted file mode 100644 index d8694453..00000000 --- a/crates/quark/src/impl/dummy/admin/migrations.rs +++ /dev/null @@ -1,11 +0,0 @@ -use crate::{AbstractMigrations, Result}; - -use super::super::DummyDb; - -#[async_trait] -impl AbstractMigrations for DummyDb { - async fn migrate_database(&self) -> Result<()> { - info!("Migrating the database."); - Ok(()) - } -} diff --git a/crates/quark/src/impl/dummy/mod.rs b/crates/quark/src/impl/dummy/mod.rs index fe26641b..194f2edb 100644 --- a/crates/quark/src/impl/dummy/mod.rs +++ b/crates/quark/src/impl/dummy/mod.rs @@ -1,7 +1,6 @@ use crate::AbstractDatabase; pub mod admin { - pub mod migrations; pub mod stats; } diff --git a/crates/quark/src/impl/generic/admin/migrations.rs b/crates/quark/src/impl/generic/admin/migrations.rs deleted file mode 100644 index 8b137891..00000000 --- a/crates/quark/src/impl/generic/admin/migrations.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/quark/src/impl/generic/mod.rs b/crates/quark/src/impl/generic/mod.rs index 5dd5dff6..015bc9dd 100644 --- a/crates/quark/src/impl/generic/mod.rs +++ b/crates/quark/src/impl/generic/mod.rs @@ -1,9 +1,5 @@ //! Database agnostic implementations. -pub mod admin { - pub mod migrations; -} - pub mod media { pub mod attachment; pub mod emoji; diff --git a/crates/quark/src/impl/mongo/mod.rs b/crates/quark/src/impl/mongo/mod.rs index 582d3d6e..7711ab0f 100644 --- a/crates/quark/src/impl/mongo/mod.rs +++ b/crates/quark/src/impl/mongo/mod.rs @@ -12,7 +12,6 @@ use serde::{Deserialize, Serialize}; use crate::{util::manipulation::prefix_keys, AbstractDatabase, Error, Result}; pub mod admin { - pub mod migrations; pub mod stats; } diff --git a/crates/quark/src/models/admin/migrations.rs b/crates/quark/src/models/admin/migrations.rs deleted file mode 100644 index 0a3e620f..00000000 --- a/crates/quark/src/models/admin/migrations.rs +++ /dev/null @@ -1,11 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// Document representing migration information -#[derive(Serialize, Deserialize)] -pub struct MigrationInfo { - /// Unique Id - #[serde(rename = "_id")] - id: i32, - /// Current database revision - revision: i32, -} diff --git a/crates/quark/src/models/mod.rs b/crates/quark/src/models/mod.rs index c0fdcf28..370c72a1 100644 --- a/crates/quark/src/models/mod.rs +++ b/crates/quark/src/models/mod.rs @@ -1,5 +1,4 @@ mod admin { - pub mod migrations; pub mod simple; pub mod stats; } @@ -47,7 +46,6 @@ pub use channel_invite::Invite; pub use channel_unread::ChannelUnread; pub use emoji::Emoji; pub use message::Message; -pub use migrations::MigrationInfo; pub use report::Report; pub use server::Server; pub use server_ban::ServerBan; diff --git a/crates/quark/src/traits/admin/migrations.rs b/crates/quark/src/traits/admin/migrations.rs deleted file mode 100644 index bb46db7a..00000000 --- a/crates/quark/src/traits/admin/migrations.rs +++ /dev/null @@ -1,6 +0,0 @@ -use crate::Result; - -#[async_trait] -pub trait AbstractMigrations: Sync + Send { - async fn migrate_database(&self) -> Result<()>; -} diff --git a/crates/quark/src/traits/mod.rs b/crates/quark/src/traits/mod.rs index b7c2f6c9..341565a1 100644 --- a/crates/quark/src/traits/mod.rs +++ b/crates/quark/src/traits/mod.rs @@ -1,5 +1,4 @@ mod admin { - pub mod migrations; pub mod stats; } @@ -32,7 +31,6 @@ mod safety { pub mod snapshot; } -pub use admin::migrations::AbstractMigrations; pub use admin::stats::AbstractStats; pub use media::attachment::AbstractAttachment; @@ -57,7 +55,6 @@ pub use safety::snapshot::AbstractSnapshot; pub trait AbstractDatabase: Sync + Send - + AbstractMigrations + AbstractStats + AbstractAttachment + AbstractEmoji From 7f86337cb2b26e4c09680185162d305e7a22c588 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 10:41:12 +0100 Subject: [PATCH 12/56] refactor: split `SISMEMBER` into two commands --- Cargo.lock | 2 +- Cargo.toml | 2 +- crates/quark/src/presence/mod.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19e0577f..9aa64794 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2644,7 +2644,7 @@ dependencies = [ [[package]] name = "redis" version = "0.22.3" -source = "git+https://github.com/insertish/redis-rs?rev=6575a6b1c09eb8c9cc7f0082d95fe6b8f903c4d7#6575a6b1c09eb8c9cc7f0082d95fe6b8f903c4d7" +source = "git+https://github.com/insertish/redis-rs?rev=1a41faf356fd21aebba71cea7eb7eb2653e5f0ef#1a41faf356fd21aebba71cea7eb7eb2653e5f0ef" dependencies = [ "async-std", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 68903beb..b51fd959 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,4 +3,4 @@ members = ["crates/*"] [patch.crates-io] # mobc-redis = { git = "https://github.com/insertish/mobc", rev = "8b880bb59f2ba80b4c7bc40c649c113d8857a186" } -redis = { git = "https://github.com/insertish/redis-rs", rev = "6575a6b1c09eb8c9cc7f0082d95fe6b8f903c4d7" } +redis = { git = "https://github.com/insertish/redis-rs", rev = "1a41faf356fd21aebba71cea7eb7eb2653e5f0ef" } diff --git a/crates/quark/src/presence/mod.rs b/crates/quark/src/presence/mod.rs index 6505ee3c..fe50537b 100644 --- a/crates/quark/src/presence/mod.rs +++ b/crates/quark/src/presence/mod.rs @@ -114,7 +114,7 @@ pub async fn presence_filter_online(user_ids: &'_ [String]) -> HashSet { // Ok so, if this breaks, that means we've lost the Redis patch which adds SMISMEMBER // Currently it's patched in through a forked repository, investigate what happen to it let data: Vec = conn - .sismember("online", user_ids) + .smismember("online", user_ids) .await .expect("this shouldn't happen, please read this code! presence/mod.rs"); if data.is_empty() { From e43833c0ea49fd2cf472eac7ee39edd509613881 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 12:12:02 +0100 Subject: [PATCH 13/56] refactor: combine models and db crate together --- Cargo.lock | 20 ++--- crates/core/database/Cargo.toml | 20 ++++- .../{models => database}/examples/migrate.rs | 0 crates/core/database/src/drivers/dummy.rs | 4 - crates/core/database/src/drivers/mod.rs | 50 ++++++++++- crates/core/database/src/drivers/reference.rs | 13 +++ crates/core/database/src/lib.rs | 83 +++++++++---------- .../src/models/admin_migrations/mod.rs | 5 ++ .../src/models}/admin_migrations/model.rs | 0 .../src/models/admin_migrations/ops.rs} | 2 +- .../models}/admin_migrations/ops/mongodb.rs | 2 +- .../admin_migrations/ops/mongodb/init.rs | 6 +- .../admin_migrations/ops/mongodb/scripts.rs | 4 +- .../models/admin_migrations/ops/reference.rs} | 5 +- crates/core/database/src/models/bots/mod.rs | 5 ++ crates/core/database/src/models/bots/model.rs | 74 +++++++++++++++++ crates/core/database/src/models/bots/ops.rs | 26 ++++++ .../database/src/models/bots/ops/mongodb.rs | 9 ++ .../database/src/models/bots/ops/reference.rs | 6 ++ crates/core/database/src/models/mod.rs | 22 +++++ crates/core/models/Cargo.toml | 25 ------ .../core/models/src/admin_migrations/mod.rs | 9 -- crates/core/models/src/lib.rs | 39 --------- crates/core/result/Cargo.toml | 8 ++ crates/core/result/src/lib.rs | 14 ++++ crates/delta/Cargo.toml | 1 - crates/delta/src/main.rs | 1 - 27 files changed, 306 insertions(+), 147 deletions(-) rename crates/core/{models => database}/examples/migrate.rs (100%) delete mode 100644 crates/core/database/src/drivers/dummy.rs create mode 100644 crates/core/database/src/drivers/reference.rs create mode 100644 crates/core/database/src/models/admin_migrations/mod.rs rename crates/core/{models/src => database/src/models}/admin_migrations/model.rs (100%) rename crates/core/{models/src/admin_migrations/ops/mod.rs => database/src/models/admin_migrations/ops.rs} (91%) rename crates/core/{models/src => database/src/models}/admin_migrations/ops/mongodb.rs (95%) rename crates/core/{models/src => database/src/models}/admin_migrations/ops/mongodb/init.rs (97%) rename crates/core/{models/src => database/src/models}/admin_migrations/ops/mongodb/scripts.rs (99%) rename crates/core/{models/src/admin_migrations/ops/dummy.rs => database/src/models/admin_migrations/ops/reference.rs} (51%) create mode 100644 crates/core/database/src/models/bots/mod.rs create mode 100644 crates/core/database/src/models/bots/model.rs create mode 100644 crates/core/database/src/models/bots/ops.rs create mode 100644 crates/core/database/src/models/bots/ops/mongodb.rs create mode 100644 crates/core/database/src/models/bots/ops/reference.rs create mode 100644 crates/core/database/src/models/mod.rs delete mode 100644 crates/core/models/src/admin_migrations/mod.rs create mode 100644 crates/core/result/Cargo.toml create mode 100644 crates/core/result/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 575a3ac8..e0770de2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2828,8 +2828,14 @@ name = "revolt-database" version = "0.1.0" dependencies = [ "async-recursion", + "async-std", + "async-trait", + "authifier", "futures", + "log", "mongodb", + "nanoid", + "optional_struct", "serde", "serde_json", ] @@ -2857,7 +2863,6 @@ dependencies = [ "regex", "reqwest", "revolt-database", - "revolt-models", "revolt-quark", "revolt_rocket_okapi", "rocket", @@ -2875,15 +2880,6 @@ dependencies = [ [[package]] name = "revolt-models" version = "0.1.0" -dependencies = [ - "async-std", - "async-trait", - "authifier", - "futures", - "log", - "revolt-database", - "serde", -] [[package]] name = "revolt-quark" @@ -2933,6 +2929,10 @@ dependencies = [ "web-push", ] +[[package]] +name = "revolt-result" +version = "0.1.0" + [[package]] name = "revolt_okapi" version = "0.9.1" diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 57b92202..f60154e8 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -6,17 +6,35 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] +# Databases mongodb = [ "dep:mongodb" ] -default = [ "mongodb" ] + +# ... Other +async-std-runtime = [ "async-std" ] + +# Default Features +default = [ "mongodb", "async-std-runtime" ] [dependencies] +# Utility +log = "*" +nanoid = "0.4.0" + # Serialisation serde_json = "1" serde = { version = "1", features = ["derive"] } +optional_struct = { git = "https://github.com/insertish/OptionalStruct", rev = "ee56427cee1f007839825d93d07fffd5a5e038c7" } # Database mongodb = { optional = true, version = "2.1.0", default-features = false } # Async Language Features futures = "0.3.19" +async-trait = "0.1.51" async-recursion = "1.0.4" + +# Async +async-std = { version = "1.8.0", features = ["attributes"], optional = true } + +# Authifier +authifier = { version = "1.0", default-features = false } diff --git a/crates/core/models/examples/migrate.rs b/crates/core/database/examples/migrate.rs similarity index 100% rename from crates/core/models/examples/migrate.rs rename to crates/core/database/examples/migrate.rs diff --git a/crates/core/database/src/drivers/dummy.rs b/crates/core/database/src/drivers/dummy.rs deleted file mode 100644 index 198bf59c..00000000 --- a/crates/core/database/src/drivers/dummy.rs +++ /dev/null @@ -1,4 +0,0 @@ -database_derived!( - /// Dummy implementation - pub struct DummyDb {} -); diff --git a/crates/core/database/src/drivers/mod.rs b/crates/core/database/src/drivers/mod.rs index 19bd06c6..3fb174fc 100644 --- a/crates/core/database/src/drivers/mod.rs +++ b/crates/core/database/src/drivers/mod.rs @@ -1,5 +1,51 @@ -mod dummy; mod mongodb; +mod reference; -pub use self::dummy::*; pub use self::mongodb::*; +pub use self::reference::*; + +/// Database information to use to create a client +pub enum DatabaseInfo { + /// Auto-detect the database in use + Auto, + /// Use the mock database + Reference, + /// Connect to MongoDB + MongoDb(String), + /// Use existing MongoDB connection + MongoDbFromClient(::mongodb::Client), +} + +/// Database +#[derive(Clone)] +pub enum Database { + /// Mock database + Reference(ReferenceDb), + /// MongoDB database + MongoDb(MongoDb), +} + +impl DatabaseInfo { + /// Create a database client from the given database information + #[async_recursion] + pub async fn connect(self) -> Result { + Ok(match self { + DatabaseInfo::Auto => { + if let Ok(uri) = std::env::var("MONGODB") { + return DatabaseInfo::MongoDb(uri).connect().await; + } + + DatabaseInfo::Reference.connect().await? + } + DatabaseInfo::Reference => Database::Reference(Default::default()), + DatabaseInfo::MongoDb(uri) => { + let client = ::mongodb::Client::with_uri_str(uri) + .await + .map_err(|_| "Failed to init db connection.".to_string())?; + + Database::MongoDb(MongoDb(client)) + } + DatabaseInfo::MongoDbFromClient(client) => Database::MongoDb(MongoDb(client)), + }) + } +} diff --git a/crates/core/database/src/drivers/reference.rs b/crates/core/database/src/drivers/reference.rs new file mode 100644 index 00000000..e14ff6f3 --- /dev/null +++ b/crates/core/database/src/drivers/reference.rs @@ -0,0 +1,13 @@ +use std::{collections::HashMap, sync::Arc}; + +use futures::lock::Mutex; + +use crate::Bot; + +database_derived!( + /// Reference implementation + #[derive(Default)] + pub struct ReferenceDb { + pub bots: Arc>>, + } +); diff --git a/crates/core/database/src/lib.rs b/crates/core/database/src/lib.rs index 3b926353..4a3aa36f 100644 --- a/crates/core/database/src/lib.rs +++ b/crates/core/database/src/lib.rs @@ -4,6 +4,18 @@ extern crate serde; #[macro_use] extern crate async_recursion; +#[macro_use] +extern crate async_trait; + +#[macro_use] +extern crate log; + +#[macro_use] +extern crate optional_struct; + +#[cfg(feature = "mongodb")] +pub use mongodb; + macro_rules! database_derived { ( $( $item:item )+ ) => { $( @@ -13,54 +25,33 @@ macro_rules! database_derived { }; } -#[cfg(feature = "mongodb")] -pub use mongodb; +macro_rules! auto_derived { + ( $( $item:item )+ ) => { + $( + #[derive(Serialize, Deserialize, Debug, Clone)] + $item + )+ + }; +} + +macro_rules! auto_derived_partial { + ( $item:item, $name:expr ) => { + #[derive(OptionalStruct, Serialize, Deserialize, Debug, Clone)] + #[optional_derive(Serialize, Deserialize, Debug, Clone)] + #[optional_name = $name] + #[opt_skip_serializing_none] + #[opt_some_priority] + $item + }; +} mod drivers; pub use drivers::*; -/// Database information to use to create a client -pub enum DatabaseInfo { - /// Auto-detect the database in use - Auto, - /// Use the mock database - Dummy, - /// Connect to MongoDB - MongoDb(String), - /// Use existing MongoDB connection - MongoDbFromClient(::mongodb::Client), -} - -/// Database -#[derive(Clone)] -pub enum Database { - /// Mock database - Dummy(DummyDb), - /// MongoDB database - MongoDb(MongoDb), -} - -impl DatabaseInfo { - /// Create a database client from the given database information - #[async_recursion] - pub async fn connect(self) -> Result { - Ok(match self { - DatabaseInfo::Auto => { - if let Ok(uri) = std::env::var("MONGODB") { - return DatabaseInfo::MongoDb(uri).connect().await; - } - - DatabaseInfo::Dummy.connect().await? - } - DatabaseInfo::Dummy => Database::Dummy(DummyDb {}), - DatabaseInfo::MongoDb(uri) => { - let client = mongodb::Client::with_uri_str(uri) - .await - .map_err(|_| "Failed to init db connection.".to_string())?; - - Database::MongoDb(MongoDb(client)) - } - DatabaseInfo::MongoDbFromClient(client) => Database::MongoDb(MongoDb(client)), - }) - } +mod models; +pub use models::*; + +/// Utility function to check if a boolean value is false +pub fn if_false(t: &bool) -> bool { + !t } diff --git a/crates/core/database/src/models/admin_migrations/mod.rs b/crates/core/database/src/models/admin_migrations/mod.rs new file mode 100644 index 00000000..4d801b73 --- /dev/null +++ b/crates/core/database/src/models/admin_migrations/mod.rs @@ -0,0 +1,5 @@ +mod model; +mod ops; + +pub use model::*; +pub use ops::*; diff --git a/crates/core/models/src/admin_migrations/model.rs b/crates/core/database/src/models/admin_migrations/model.rs similarity index 100% rename from crates/core/models/src/admin_migrations/model.rs rename to crates/core/database/src/models/admin_migrations/model.rs diff --git a/crates/core/models/src/admin_migrations/ops/mod.rs b/crates/core/database/src/models/admin_migrations/ops.rs similarity index 91% rename from crates/core/models/src/admin_migrations/ops/mod.rs rename to crates/core/database/src/models/admin_migrations/ops.rs index 386434d8..f5de7a88 100644 --- a/crates/core/models/src/admin_migrations/ops/mod.rs +++ b/crates/core/database/src/models/admin_migrations/ops.rs @@ -1,5 +1,5 @@ -mod dummy; mod mongodb; +mod reference; #[async_trait] pub trait AbstractMigrations: Sync + Send { diff --git a/crates/core/models/src/admin_migrations/ops/mongodb.rs b/crates/core/database/src/models/admin_migrations/ops/mongodb.rs similarity index 95% rename from crates/core/models/src/admin_migrations/ops/mongodb.rs rename to crates/core/database/src/models/admin_migrations/ops/mongodb.rs index 85086830..193e0973 100644 --- a/crates/core/models/src/admin_migrations/ops/mongodb.rs +++ b/crates/core/database/src/models/admin_migrations/ops/mongodb.rs @@ -1,4 +1,4 @@ -use revolt_database::MongoDb; +use crate::MongoDb; use super::AbstractMigrations; diff --git a/crates/core/models/src/admin_migrations/ops/mongodb/init.rs b/crates/core/database/src/models/admin_migrations/ops/mongodb/init.rs similarity index 97% rename from crates/core/models/src/admin_migrations/ops/mongodb/init.rs rename to crates/core/database/src/models/admin_migrations/ops/mongodb/init.rs index 37864805..2649635d 100644 --- a/crates/core/models/src/admin_migrations/ops/mongodb/init.rs +++ b/crates/core/database/src/models/admin_migrations/ops/mongodb/init.rs @@ -1,8 +1,8 @@ use super::scripts::LATEST_REVISION; -use revolt_database::mongodb::bson::doc; -use revolt_database::mongodb::options::CreateCollectionOptions; -use revolt_database::MongoDb; +use crate::mongodb::bson::doc; +use crate::mongodb::options::CreateCollectionOptions; +use crate::MongoDb; pub async fn create_database(db: &MongoDb) { info!("Creating database."); diff --git a/crates/core/models/src/admin_migrations/ops/mongodb/scripts.rs b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs similarity index 99% rename from crates/core/models/src/admin_migrations/ops/mongodb/scripts.rs rename to crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs index 1aa1c7c3..8c5aa874 100644 --- a/crates/core/models/src/admin_migrations/ops/mongodb/scripts.rs +++ b/crates/core/database/src/models/admin_migrations/ops/mongodb/scripts.rs @@ -1,13 +1,13 @@ use std::{ops::BitXor, time::Duration}; -use futures::StreamExt; -use revolt_database::{ +use crate::{ mongodb::{ bson::{doc, from_bson, from_document, to_document, Bson, DateTime, Document}, options::FindOptions, }, MongoDb, }; +use futures::StreamExt; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] diff --git a/crates/core/models/src/admin_migrations/ops/dummy.rs b/crates/core/database/src/models/admin_migrations/ops/reference.rs similarity index 51% rename from crates/core/models/src/admin_migrations/ops/dummy.rs rename to crates/core/database/src/models/admin_migrations/ops/reference.rs index 0db084c3..19f3b99e 100644 --- a/crates/core/models/src/admin_migrations/ops/dummy.rs +++ b/crates/core/database/src/models/admin_migrations/ops/reference.rs @@ -1,11 +1,12 @@ -use revolt_database::DummyDb; +use crate::ReferenceDb; use super::AbstractMigrations; #[async_trait] -impl AbstractMigrations for DummyDb { +impl AbstractMigrations for ReferenceDb { /// Migrate the database async fn migrate_database(&self) -> Result<(), ()> { + // Here you would do your typical migrations if this was a real database. Ok(()) } } diff --git a/crates/core/database/src/models/bots/mod.rs b/crates/core/database/src/models/bots/mod.rs new file mode 100644 index 00000000..3037c2f0 --- /dev/null +++ b/crates/core/database/src/models/bots/mod.rs @@ -0,0 +1,5 @@ +mod model; +// mod ops; + +pub use model::*; +// pub use ops::*; diff --git a/crates/core/database/src/models/bots/model.rs b/crates/core/database/src/models/bots/model.rs new file mode 100644 index 00000000..a143340a --- /dev/null +++ b/crates/core/database/src/models/bots/model.rs @@ -0,0 +1,74 @@ +use crate::Database; + +auto_derived_partial!( + /// Bot + pub struct Bot { + /// Bot Id + /// + /// This equals the associated bot user's id. + #[serde(rename = "_id")] + pub id: String, + /// User Id of the bot owner + pub owner: String, + /// Token used to authenticate requests for this bot + pub token: String, + /// Whether the bot is public + /// (may be invited by anyone) + pub public: bool, + + /// Whether to enable analytics + #[serde(skip_serializing_if = "crate::if_false", default)] + pub analytics: bool, + /// Whether this bot should be publicly discoverable + #[serde(skip_serializing_if = "crate::if_false", default)] + pub discoverable: bool, + /// Reserved; URL for handling interactions + #[serde(skip_serializing_if = "Option::is_none")] + pub interactions_url: Option, + /// URL for terms of service + #[serde(skip_serializing_if = "Option::is_none")] + pub terms_of_service_url: Option, + /// URL for privacy policy + #[serde(skip_serializing_if = "Option::is_none")] + pub privacy_policy_url: Option, + + /// Enum of bot flags + #[serde(skip_serializing_if = "Option::is_none")] + pub flags: Option, + }, + "PartialBot" +); + +auto_derived!( + /// Flags that may be attributed to a bot + #[repr(i32)] + pub enum BotFlags { + Verified = 1, + Official = 2, + } + + /// Optional fields on bot object + pub enum FieldsBot { + Token, + InteractionsURL, + } +); + +impl Bot { + /// Remove a field from this object + pub fn remove(&mut self, field: &FieldsBot) { + match field { + FieldsBot::Token => self.token = nanoid::nanoid!(64), + FieldsBot::InteractionsURL => { + self.interactions_url.take(); + } + } + } + + /// Delete this bot + pub async fn delete(&self, db: &Database) -> Result<(), ()> { + // db.fetch_user(&self.id).await?.mark_deleted(db).await?; + // db.delete_bot(&self.id).await + Ok(()) + } +} diff --git a/crates/core/database/src/models/bots/ops.rs b/crates/core/database/src/models/bots/ops.rs new file mode 100644 index 00000000..96056f90 --- /dev/null +++ b/crates/core/database/src/models/bots/ops.rs @@ -0,0 +1,26 @@ +mod dummy; +mod mongodb; + +#[async_trait] +pub trait AbstractBots: Sync + Send { + /// Fetch a bot by its id + async fn fetch_bot(&self, id: &str) -> Result; + + /// Fetch a bot by its token + async fn fetch_bot_by_token(&self, token: &str) -> Result; + + /// Insert new bot into the database + async fn insert_bot(&self, bot: &Bot) -> Result<()>; + + /// Update bot with new information + async fn update_bot(&self, id: &str, bot: &PartialBot, remove: Vec) -> Result<()>; + + /// Delete a bot from the database + async fn delete_bot(&self, id: &str) -> Result<()>; + + /// Fetch bots owned by a user + async fn fetch_bots_by_user(&self, user_id: &str) -> Result>; + + /// Get the number of bots owned by a user + async fn get_number_of_bots_by_user(&self, user_id: &str) -> Result; +} diff --git a/crates/core/database/src/models/bots/ops/mongodb.rs b/crates/core/database/src/models/bots/ops/mongodb.rs new file mode 100644 index 00000000..631e6bc7 --- /dev/null +++ b/crates/core/database/src/models/bots/ops/mongodb.rs @@ -0,0 +1,9 @@ +use revolt_database::MongoDb; + +use super::AbstractBots; + +mod init; +mod scripts; + +#[async_trait] +impl AbstractBots for MongoDb {} diff --git a/crates/core/database/src/models/bots/ops/reference.rs b/crates/core/database/src/models/bots/ops/reference.rs new file mode 100644 index 00000000..30a56a45 --- /dev/null +++ b/crates/core/database/src/models/bots/ops/reference.rs @@ -0,0 +1,6 @@ +use revolt_database::DummyDb; + +use super::AbstractBots; + +#[async_trait] +impl AbstractBots for DummyDb {} diff --git a/crates/core/database/src/models/mod.rs b/crates/core/database/src/models/mod.rs new file mode 100644 index 00000000..4ee16f0e --- /dev/null +++ b/crates/core/database/src/models/mod.rs @@ -0,0 +1,22 @@ +mod admin_migrations; +mod bots; + +pub use admin_migrations::*; +pub use bots::*; + +use crate::{Database, MongoDb, ReferenceDb}; + +pub trait AbstractDatabase: Sync + Send + admin_migrations::AbstractMigrations {} +impl AbstractDatabase for ReferenceDb {} +impl AbstractDatabase for MongoDb {} + +impl std::ops::Deref for Database { + type Target = dyn AbstractDatabase; + + fn deref(&self) -> &Self::Target { + match &self { + Database::Reference(dummy) => dummy, + Database::MongoDb(mongo) => mongo, + } + } +} diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 13cf041a..812e083f 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -5,29 +5,4 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[features] -serde = [ "dep:serde" ] -async-std-runtime = [ "async-std" ] -database = [ "revolt-database", "revolt-database/default", "authifier/database-mongodb" ] - -default = [ "serde", "async-std-runtime", "database" ] - [dependencies] -# Utility -log = "*" - -# Serialisation -serde = { version = "1", features = ["derive"], optional = true } - -# Async Language Features -futures = "0.3.19" -async-trait = "0.1.51" - -# Async -async-std = { version = "1.8.0", features = ["attributes"], optional = true } - -# Peer dependencies -revolt-database = { path = "../database", optional = true, default-features = false } - -# Authifier -authifier = { version = "1.0", default-features = false } diff --git a/crates/core/models/src/admin_migrations/mod.rs b/crates/core/models/src/admin_migrations/mod.rs deleted file mode 100644 index 233a1d96..00000000 --- a/crates/core/models/src/admin_migrations/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod model; - -pub use model::*; - -#[cfg(feature = "database")] -mod ops; - -#[cfg(feature = "database")] -pub use ops::*; diff --git a/crates/core/models/src/lib.rs b/crates/core/models/src/lib.rs index bffcadf4..8b137891 100644 --- a/crates/core/models/src/lib.rs +++ b/crates/core/models/src/lib.rs @@ -1,40 +1 @@ -#[cfg(feature = "serde")] -#[macro_use] -extern crate serde; -#[macro_use] -extern crate async_trait; - -#[macro_use] -extern crate log; - -macro_rules! auto_derived { - ( $( $item:item )+ ) => { - $( - #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] - #[derive(Debug, Clone)] - $item - )+ - }; -} - -mod admin_migrations; - -pub use admin_migrations::*; - -pub struct Database(pub revolt_database::Database); - -pub trait AbstractDatabase: Sync + Send + admin_migrations::AbstractMigrations {} -impl AbstractDatabase for revolt_database::DummyDb {} -impl AbstractDatabase for revolt_database::MongoDb {} - -impl std::ops::Deref for Database { - type Target = dyn AbstractDatabase; - - fn deref(&self) -> &Self::Target { - match &self.0 { - revolt_database::Database::Dummy(dummy) => dummy, - revolt_database::Database::MongoDb(mongo) => mongo, - } - } -} diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml new file mode 100644 index 00000000..839aceec --- /dev/null +++ b/crates/core/result/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "revolt-result" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crates/core/result/src/lib.rs b/crates/core/result/src/lib.rs new file mode 100644 index 00000000..7d12d9af --- /dev/null +++ b/crates/core/result/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: usize, right: usize) -> usize { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 556f301d..6eea2d4a 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -57,7 +57,6 @@ revolt-quark = { path = "../quark" } # core revolt-database = { path = "../core/database" } -revolt-models = { path = "../core/models" } [build-dependencies] vergen = "7.5.0" diff --git a/crates/delta/src/main.rs b/crates/delta/src/main.rs index 3c2b516a..e88897d1 100644 --- a/crates/delta/src/main.rs +++ b/crates/delta/src/main.rs @@ -23,7 +23,6 @@ async fn rocket() -> _ { // Setup database let db = revolt_database::DatabaseInfo::Auto.connect().await.unwrap(); - let db = revolt_models::Database(db); db.migrate_database().await.unwrap(); // Legacy database setup from quark From d633cba630cba39fe15ce083946a816d67a9419a Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 12:12:23 +0100 Subject: [PATCH 14/56] fix: migration example --- crates/core/database/examples/migrate.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/core/database/examples/migrate.rs b/crates/core/database/examples/migrate.rs index 96f975b5..f276d773 100644 --- a/crates/core/database/examples/migrate.rs +++ b/crates/core/database/examples/migrate.rs @@ -1,8 +1,7 @@ use revolt_database::DatabaseInfo; -use revolt_models::*; #[async_std::main] async fn main() { - let db = Database(DatabaseInfo::Auto.connect().await.unwrap()); + let db = DatabaseInfo::Auto.connect().await.unwrap(); db.migrate_database().await.unwrap(); } From 56ead0f894104331c5eb224e6c7164cae89bd95a Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 12:48:09 +0100 Subject: [PATCH 15/56] feat: write result crate --- Cargo.lock | 3 + crates/core/result/Cargo.toml | 6 ++ crates/core/result/src/lib.rs | 137 ++++++++++++++++++++++++++++++++-- 3 files changed, 140 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0770de2..0d7f8209 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2932,6 +2932,9 @@ dependencies = [ [[package]] name = "revolt-result" version = "0.1.0" +dependencies = [ + "serde", +] [[package]] name = "revolt_okapi" diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index 839aceec..12947827 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -5,4 +5,10 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +serde = [ "dep:serde" ] +default = [ "serde" ] + [dependencies] +# Serialisation +serde = { version = "1", features = ["derive"], optional = true } diff --git a/crates/core/result/src/lib.rs b/crates/core/result/src/lib.rs index 7d12d9af..7f0eea5d 100644 --- a/crates/core/result/src/lib.rs +++ b/crates/core/result/src/lib.rs @@ -1,14 +1,139 @@ -pub fn add(left: usize, right: usize) -> usize { - left + right +#[cfg(feature = "serde")] +#[macro_use] +extern crate serde; + +/// Result type with custom Error +pub type Result = std::result::Result; + +/// Error information +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[derive(Debug, Clone)] +pub struct Error { + /// Type of error and additional information + #[cfg_attr(feature = "serde", serde(flatten))] + pub error_type: ErrorType, + + /// Where this error occurred + pub location: String, +} + +/// Possible error types +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(tag = "type"))] +#[derive(Debug, Clone)] +pub enum ErrorType { + /// This error was not labeled :( + LabelMe, + + // ? Onboarding related errors + AlreadyOnboarded, + + // ? User related errors + UsernameTaken, + InvalidUsername, + UnknownUser, + AlreadyFriends, + AlreadySentRequest, + Blocked, + BlockedByOther, + NotFriends, + + // ? Channel related errors + UnknownChannel, + UnknownAttachment, + UnknownMessage, + CannotEditMessage, + CannotJoinCall, + TooManyAttachments { + max: usize, + }, + TooManyReplies { + max: usize, + }, + TooManyChannels { + max: usize, + }, + EmptyMessage, + PayloadTooLarge, + CannotRemoveYourself, + GroupTooLarge { + max: usize, + }, + AlreadyInGroup, + NotInGroup, + + // ? Server related errors + UnknownServer, + InvalidRole, + Banned, + TooManyServers { + max: usize, + }, + TooManyEmoji { + max: usize, + }, + TooManyRoles { + max: usize, + }, + + // ? Bot related errors + ReachedMaximumBots, + IsBot, + BotIsPrivate, + + // ? User safety related errors + CannotReportYourself, + + // ? Permission errors + MissingPermission { + permission: String, + }, + MissingUserPermission { + permission: String, + }, + NotElevated, + NotPrivileged, + CannotGiveMissingPermissions, + NotOwner, + + // ? General errors + DatabaseError { + operation: String, + with: String, + }, + InternalError, + InvalidOperation, + InvalidCredentials, + InvalidProperty, + InvalidSession, + DuplicateNonce, + NotFound, + NoEffect, + FailedValidation { + error: String, + }, + + // ? Legacy errors + VosoUnavailable, +} + +#[macro_export] +macro_rules! create_error { + ( $error:ident ) => { + $crate::Error { + error_type: $crate::ErrorType::$error, + location: format!("{}:{}:{}", file!(), line!(), column!()), + } + }; } #[cfg(test)] mod tests { - use super::*; + use crate::ErrorType; #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); + fn use_macro_to_construct_error() { + let error = create_error!(LabelMe); + assert!(matches!(error.error_type, ErrorType::LabelMe)); } } From 6b10385c0d7723f5480a71975c1fe4ba79375aa6 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 14:25:02 +0100 Subject: [PATCH 16/56] feat: implement bots including tests this starts work on #78 --- Cargo.lock | 1 + crates/core/database/Cargo.toml | 5 +- crates/core/database/src/drivers/mongodb.rs | 35 ++++-- crates/core/database/src/lib.rs | 18 +++- crates/core/database/src/models/bots/mod.rs | 4 +- crates/core/database/src/models/bots/model.rs | 58 +++++++++- crates/core/database/src/models/bots/ops.rs | 13 ++- .../database/src/models/bots/ops/mongodb.rs | 101 +++++++++++++++++- .../database/src/models/bots/ops/reference.rs | 79 +++++++++++++- crates/core/database/src/models/mod.rs | 6 +- crates/core/result/src/lib.rs | 22 +++- 11 files changed, 311 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d7f8209..24e4e775 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2836,6 +2836,7 @@ dependencies = [ "mongodb", "nanoid", "optional_struct", + "revolt-result", "serde", "serde_json", ] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index f60154e8..b17ddb45 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -16,6 +16,9 @@ async-std-runtime = [ "async-std" ] default = [ "mongodb", "async-std-runtime" ] [dependencies] +# Repo +revolt-result = { path = "../result" } + # Utility log = "*" nanoid = "0.4.0" @@ -37,4 +40,4 @@ async-recursion = "1.0.4" async-std = { version = "1.8.0", features = ["attributes"], optional = true } # Authifier -authifier = { version = "1.0", default-features = false } +authifier = { version = "1.0" } diff --git a/crates/core/database/src/drivers/mongodb.rs b/crates/core/database/src/drivers/mongodb.rs index 6fb99135..afe22ac7 100644 --- a/crates/core/database/src/drivers/mongodb.rs +++ b/crates/core/database/src/drivers/mongodb.rs @@ -36,7 +36,7 @@ impl MongoDb { } /// Insert one document into a collection - async fn insert_one( + pub async fn insert_one( &self, collection: &'static str, document: T, @@ -44,8 +44,19 @@ impl MongoDb { self.col::(collection).insert_one(document, None).await } + /// Count documents by projection + pub async fn count_documents( + &self, + collection: &'static str, + projection: Document, + ) -> Result { + self.col::(collection) + .count_documents(projection, None) + .await + } + /// Find multiple documents in a collection with options - async fn find_with_options( + pub async fn find_with_options( &self, collection: &'static str, projection: Document, @@ -71,7 +82,7 @@ impl MongoDb { } /// Find multiple documents in a collection - async fn find( + pub async fn find( &self, collection: &'static str, projection: Document, @@ -80,7 +91,7 @@ impl MongoDb { } /// Find one document with options - async fn find_one_with_options( + pub async fn find_one_with_options( &self, collection: &'static str, projection: Document, @@ -95,7 +106,7 @@ impl MongoDb { } /// Find one document - async fn find_one( + pub async fn find_one( &self, collection: &'static str, projection: Document, @@ -105,7 +116,7 @@ impl MongoDb { } /// Find one document by its ID - async fn find_one_by_id( + pub async fn find_one_by_id( &self, collection: &'static str, id: &str, @@ -120,7 +131,7 @@ impl MongoDb { } /// Update one document given a projection, partial document, and list of paths to unset - async fn update_one( + pub async fn update_one( &self, collection: &'static str, projection: Document, @@ -159,7 +170,7 @@ impl MongoDb { } /// Update one document given an ID, partial document, and list of paths to unset - async fn update_one_by_id( + pub async fn update_one_by_id( &self, collection: &'static str, id: &str, @@ -183,7 +194,7 @@ impl MongoDb { } /// Delete one document by the given projection - async fn delete_one( + pub async fn delete_one( &self, collection: &'static str, projection: Document, @@ -194,7 +205,11 @@ impl MongoDb { } /// Delete one document by the given ID - async fn delete_one_by_id(&self, collection: &'static str, id: &str) -> Result { + pub async fn delete_one_by_id( + &self, + collection: &'static str, + id: &str, + ) -> Result { self.delete_one( collection, doc! { diff --git a/crates/core/database/src/lib.rs b/crates/core/database/src/lib.rs index 4a3aa36f..257abe96 100644 --- a/crates/core/database/src/lib.rs +++ b/crates/core/database/src/lib.rs @@ -13,6 +13,9 @@ extern crate log; #[macro_use] extern crate optional_struct; +#[macro_use] +extern crate revolt_result; + #[cfg(feature = "mongodb")] pub use mongodb; @@ -28,7 +31,7 @@ macro_rules! database_derived { macro_rules! auto_derived { ( $( $item:item )+ ) => { $( - #[derive(Serialize, Deserialize, Debug, Clone)] + #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] $item )+ }; @@ -36,8 +39,8 @@ macro_rules! auto_derived { macro_rules! auto_derived_partial { ( $item:item, $name:expr ) => { - #[derive(OptionalStruct, Serialize, Deserialize, Debug, Clone)] - #[optional_derive(Serialize, Deserialize, Debug, Clone)] + #[derive(OptionalStruct, Serialize, Deserialize, Debug, Clone, Default, Eq, PartialEq)] + #[optional_derive(Serialize, Deserialize, Debug, Clone, Default, Eq, PartialEq)] #[optional_name = $name] #[opt_skip_serializing_none] #[opt_some_priority] @@ -48,6 +51,15 @@ macro_rules! auto_derived_partial { mod drivers; pub use drivers::*; +macro_rules! database_test { + ( $name: expr ) => { + $crate::DatabaseInfo::Reference + .connect() + .await + .expect("Database connection failed.") + }; +} + mod models; pub use models::*; diff --git a/crates/core/database/src/models/bots/mod.rs b/crates/core/database/src/models/bots/mod.rs index 3037c2f0..4d801b73 100644 --- a/crates/core/database/src/models/bots/mod.rs +++ b/crates/core/database/src/models/bots/mod.rs @@ -1,5 +1,5 @@ mod model; -// mod ops; +mod ops; pub use model::*; -// pub use ops::*; +pub use ops::*; diff --git a/crates/core/database/src/models/bots/model.rs b/crates/core/database/src/models/bots/model.rs index a143340a..05fed343 100644 --- a/crates/core/database/src/models/bots/model.rs +++ b/crates/core/database/src/models/bots/model.rs @@ -1,3 +1,5 @@ +use revolt_result::Result; + use crate::Database; auto_derived_partial!( @@ -66,9 +68,59 @@ impl Bot { } /// Delete this bot - pub async fn delete(&self, db: &Database) -> Result<(), ()> { + pub async fn delete(&self, db: &Database) -> Result<()> { // db.fetch_user(&self.id).await?.mark_deleted(db).await?; - // db.delete_bot(&self.id).await - Ok(()) + db.delete_bot(&self.id).await + } +} + +#[cfg(test)] +mod tests { + use crate::{Bot, FieldsBot}; + + #[async_std::test] + async fn crud() { + let db = database_test!("bot_crud"); + + let bot_id = "bot"; + let user_id = "user"; + let token = "my_token"; + + let bot = Bot { + id: bot_id.to_string(), + owner: user_id.to_string(), + token: token.to_string(), + interactions_url: Some("some url".to_string()), + ..Default::default() + }; + + db.insert_bot(&bot).await.unwrap(); + db.update_bot( + bot_id, + &crate::PartialBot { + public: Some(true), + ..Default::default() + }, + vec![FieldsBot::Token, FieldsBot::InteractionsURL], + ) + .await + .unwrap(); + + let fetched_bot1 = db.fetch_bot(bot_id).await.unwrap(); + let fetched_bot2 = db.fetch_bot_by_token(&fetched_bot1.token).await.unwrap(); + let fetched_bots = db.fetch_bots_by_user(user_id).await.unwrap(); + + assert!(!bot.public); + assert!(fetched_bot1.public); + assert!(bot.interactions_url.is_some()); + assert!(fetched_bot1.interactions_url.is_none()); + assert_ne!(bot.token, fetched_bot1.token); + assert_eq!(fetched_bot1, fetched_bot2); + assert_eq!(fetched_bot1, fetched_bots[0]); + assert_eq!(1, db.get_number_of_bots_by_user(user_id).await.unwrap()); + + bot.delete(&db).await.unwrap(); + assert!(db.fetch_bot(bot_id).await.is_err()); + assert_eq!(0, db.get_number_of_bots_by_user(user_id).await.unwrap()); } } diff --git a/crates/core/database/src/models/bots/ops.rs b/crates/core/database/src/models/bots/ops.rs index 96056f90..3a618253 100644 --- a/crates/core/database/src/models/bots/ops.rs +++ b/crates/core/database/src/models/bots/ops.rs @@ -1,5 +1,9 @@ -mod dummy; +use revolt_result::Result; + +use crate::{Bot, FieldsBot, PartialBot}; + mod mongodb; +mod reference; #[async_trait] pub trait AbstractBots: Sync + Send { @@ -13,7 +17,12 @@ pub trait AbstractBots: Sync + Send { async fn insert_bot(&self, bot: &Bot) -> Result<()>; /// Update bot with new information - async fn update_bot(&self, id: &str, bot: &PartialBot, remove: Vec) -> Result<()>; + async fn update_bot( + &self, + id: &str, + partial: &PartialBot, + remove: Vec, + ) -> Result<()>; /// Delete a bot from the database async fn delete_bot(&self, id: &str) -> Result<()>; diff --git a/crates/core/database/src/models/bots/ops/mongodb.rs b/crates/core/database/src/models/bots/ops/mongodb.rs index 631e6bc7..7b4c7eee 100644 --- a/crates/core/database/src/models/bots/ops/mongodb.rs +++ b/crates/core/database/src/models/bots/ops/mongodb.rs @@ -1,9 +1,102 @@ -use revolt_database::MongoDb; +use ::mongodb::bson::doc; +use revolt_result::Result; + +use crate::{Bot, FieldsBot, PartialBot}; +use crate::{IntoDocumentPath, MongoDb}; use super::AbstractBots; -mod init; -mod scripts; +static COL: &str = "bots"; #[async_trait] -impl AbstractBots for MongoDb {} +impl AbstractBots for MongoDb { + /// Fetch a bot by its id + async fn fetch_bot(&self, id: &str) -> Result { + self.find_one_by_id(COL, id) + .await + .map_err(|_| create_database_error!("find_one", COL))? + .ok_or_else(|| create_error!(NotFound)) + } + + /// Fetch a bot by its token + async fn fetch_bot_by_token(&self, token: &str) -> Result { + self.find_one( + COL, + doc! { + "token": token + }, + ) + .await + .map_err(|_| create_database_error!("find_one", COL))? + .ok_or_else(|| create_error!(NotFound)) + } + + /// Insert new bot into the database + async fn insert_bot(&self, bot: &Bot) -> Result<()> { + self.insert_one(COL, &bot) + .await + .map(|_| ()) + .map_err(|_| create_database_error!("insert_one", COL)) + } + + /// Update bot with new information + async fn update_bot( + &self, + id: &str, + partial: &PartialBot, + remove: Vec, + ) -> Result<()> { + self.update_one_by_id( + COL, + id, + partial, + remove.iter().map(|x| x as &dyn IntoDocumentPath).collect(), + None, + ) + .await + .map(|_| ()) + .map_err(|_| create_database_error!("update_one", COL)) + } + + /// Delete a bot from the database + async fn delete_bot(&self, id: &str) -> Result<()> { + self.delete_one_by_id(COL, id) + .await + .map(|_| ()) + .map_err(|_| create_database_error!("delete_one", COL)) + } + + /// Fetch bots owned by a user + async fn fetch_bots_by_user(&self, user_id: &str) -> Result> { + self.find( + COL, + doc! { + "owner": user_id + }, + ) + .await + .map_err(|_| create_database_error!("find", COL)) + } + + /// Get the number of bots owned by a user + async fn get_number_of_bots_by_user(&self, user_id: &str) -> Result { + self.count_documents( + COL, + doc! { + "owner": user_id + }, + ) + .await + .map(|v| v as usize) + .map_err(|_| create_database_error!("count", COL)) + } +} + +impl IntoDocumentPath for FieldsBot { + fn as_path(&self) -> Option<&'static str> { + match self { + FieldsBot::InteractionsURL => Some("interactions_url"), + FieldsBot::Token => None, + } + } +} diff --git a/crates/core/database/src/models/bots/ops/reference.rs b/crates/core/database/src/models/bots/ops/reference.rs index 30a56a45..26ce98c6 100644 --- a/crates/core/database/src/models/bots/ops/reference.rs +++ b/crates/core/database/src/models/bots/ops/reference.rs @@ -1,6 +1,81 @@ -use revolt_database::DummyDb; +use revolt_result::Result; + +use crate::ReferenceDb; +use crate::{Bot, FieldsBot, PartialBot}; use super::AbstractBots; #[async_trait] -impl AbstractBots for DummyDb {} +impl AbstractBots for ReferenceDb { + /// Fetch a bot by its id + async fn fetch_bot(&self, id: &str) -> Result { + let bots = self.bots.lock().await; + bots.get(id).cloned().ok_or_else(|| create_error!(NotFound)) + } + + /// Fetch a bot by its token + async fn fetch_bot_by_token(&self, token: &str) -> Result { + let bots = self.bots.lock().await; + bots.values() + .find(|bot| bot.token == token) + .cloned() + .ok_or_else(|| create_error!(NotFound)) + } + + /// Insert new bot into the database + async fn insert_bot(&self, bot: &Bot) -> Result<()> { + let mut bots = self.bots.lock().await; + if bots.contains_key(&bot.id) { + Err(create_database_error!("insert", "bot")) + } else { + bots.insert(bot.id.to_string(), bot.clone()); + Ok(()) + } + } + + /// Update bot with new information + async fn update_bot( + &self, + id: &str, + partial: &PartialBot, + remove: Vec, + ) -> Result<()> { + let mut bots = self.bots.lock().await; + if let Some(bot) = bots.get_mut(id) { + for field in remove { + bot.remove(&field); + } + + bot.apply_options(partial.clone()); + Ok(()) + } else { + Err(create_error!(NotFound)) + } + } + + /// Delete a bot from the database + async fn delete_bot(&self, id: &str) -> Result<()> { + let mut bots = self.bots.lock().await; + if bots.remove(id).is_some() { + Ok(()) + } else { + Err(create_error!(NotFound)) + } + } + + /// Fetch bots owned by a user + async fn fetch_bots_by_user(&self, user_id: &str) -> Result> { + let bots = self.bots.lock().await; + Ok(bots + .values() + .filter(|bot| bot.owner == user_id) + .cloned() + .collect()) + } + + /// Get the number of bots owned by a user + async fn get_number_of_bots_by_user(&self, user_id: &str) -> Result { + let bots = self.bots.lock().await; + Ok(bots.values().filter(|bot| bot.owner == user_id).count()) + } +} diff --git a/crates/core/database/src/models/mod.rs b/crates/core/database/src/models/mod.rs index 4ee16f0e..d6b2a21c 100644 --- a/crates/core/database/src/models/mod.rs +++ b/crates/core/database/src/models/mod.rs @@ -6,7 +6,11 @@ pub use bots::*; use crate::{Database, MongoDb, ReferenceDb}; -pub trait AbstractDatabase: Sync + Send + admin_migrations::AbstractMigrations {} +pub trait AbstractDatabase: + Sync + Send + admin_migrations::AbstractMigrations + bots::AbstractBots +{ +} + impl AbstractDatabase for ReferenceDb {} impl AbstractDatabase for MongoDb {} diff --git a/crates/core/result/src/lib.rs b/crates/core/result/src/lib.rs index 7f0eea5d..ada4d55f 100644 --- a/crates/core/result/src/lib.rs +++ b/crates/core/result/src/lib.rs @@ -99,7 +99,7 @@ pub enum ErrorType { // ? General errors DatabaseError { operation: String, - with: String, + collection: String, }, InternalError, InvalidOperation, @@ -119,14 +119,24 @@ pub enum ErrorType { #[macro_export] macro_rules! create_error { - ( $error:ident ) => { + ( $error:ident $( $tt:tt )? ) => { $crate::Error { - error_type: $crate::ErrorType::$error, + error_type: $crate::ErrorType::$error $( $tt )?, location: format!("{}:{}:{}", file!(), line!(), column!()), } }; } +#[macro_export] +macro_rules! create_database_error { + ( $operation:expr, $collection:expr ) => { + create_error!(DatabaseError { + operation: $operation.to_string(), + collection: $collection.to_string() + }) + }; +} + #[cfg(test)] mod tests { use crate::ErrorType; @@ -136,4 +146,10 @@ mod tests { let error = create_error!(LabelMe); assert!(matches!(error.error_type, ErrorType::LabelMe)); } + + #[test] + fn use_macro_to_construct_complex_error() { + let error = create_error!(LabelMe); + assert!(matches!(error.error_type, ErrorType::LabelMe)); + } } From 750037b5d23f70b5cbf8310c1f88fd871d37eac5 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 14:58:55 +0100 Subject: [PATCH 17/56] test: use custom database name and add clean up --- crates/core/database/src/drivers/mod.rs | 28 +++++-- crates/core/database/src/drivers/mongodb.rs | 4 +- crates/core/database/src/lib.rs | 15 +++- crates/core/database/src/models/bots/model.rs | 74 +++++++++---------- 4 files changed, 73 insertions(+), 48 deletions(-) diff --git a/crates/core/database/src/drivers/mod.rs b/crates/core/database/src/drivers/mod.rs index 3fb174fc..79cc03f1 100644 --- a/crates/core/database/src/drivers/mod.rs +++ b/crates/core/database/src/drivers/mod.rs @@ -8,12 +8,14 @@ pub use self::reference::*; pub enum DatabaseInfo { /// Auto-detect the database in use Auto, + /// Auto-detect the database in use and create an empty testing database + Test(String), /// Use the mock database Reference, /// Connect to MongoDB - MongoDb(String), + MongoDb { uri: String, database_name: String }, /// Use existing MongoDB connection - MongoDbFromClient(::mongodb::Client), + MongoDbFromClient(::mongodb::Client, String), } /// Database @@ -32,20 +34,34 @@ impl DatabaseInfo { Ok(match self { DatabaseInfo::Auto => { if let Ok(uri) = std::env::var("MONGODB") { - return DatabaseInfo::MongoDb(uri).connect().await; + return DatabaseInfo::MongoDb { + uri, + database_name: "revolt".to_string(), + } + .connect() + .await; + } + + DatabaseInfo::Reference.connect().await? + } + DatabaseInfo::Test(database_name) => { + if let Ok(uri) = std::env::var("MONGODB") { + return DatabaseInfo::MongoDb { uri, database_name }.connect().await; } DatabaseInfo::Reference.connect().await? } DatabaseInfo::Reference => Database::Reference(Default::default()), - DatabaseInfo::MongoDb(uri) => { + DatabaseInfo::MongoDb { uri, database_name } => { let client = ::mongodb::Client::with_uri_str(uri) .await .map_err(|_| "Failed to init db connection.".to_string())?; - Database::MongoDb(MongoDb(client)) + Database::MongoDb(MongoDb(client, database_name)) + } + DatabaseInfo::MongoDbFromClient(client, database_name) => { + Database::MongoDb(MongoDb(client, database_name)) } - DatabaseInfo::MongoDbFromClient(client) => Database::MongoDb(MongoDb(client)), }) } } diff --git a/crates/core/database/src/drivers/mongodb.rs b/crates/core/database/src/drivers/mongodb.rs index afe22ac7..74e5c6ef 100644 --- a/crates/core/database/src/drivers/mongodb.rs +++ b/crates/core/database/src/drivers/mongodb.rs @@ -12,7 +12,7 @@ use serde::Serialize; database_derived!( #[cfg(feature = "mongodb")] /// MongoDB implementation - pub struct MongoDb(pub ::mongodb::Client); + pub struct MongoDb(pub ::mongodb::Client, pub String); ); impl Deref for MongoDb { @@ -27,7 +27,7 @@ impl Deref for MongoDb { impl MongoDb { /// Get the Revolt database pub fn db(&self) -> mongodb::Database { - self.database("revolt") + self.database(&self.1) } /// Get a collection by its name diff --git a/crates/core/database/src/lib.rs b/crates/core/database/src/lib.rs index 257abe96..fcc19a31 100644 --- a/crates/core/database/src/lib.rs +++ b/crates/core/database/src/lib.rs @@ -51,12 +51,21 @@ macro_rules! auto_derived_partial { mod drivers; pub use drivers::*; +#[cfg(test)] macro_rules! database_test { - ( $name: expr ) => { - $crate::DatabaseInfo::Reference + ( | $db: ident | $test:expr ) => { + let db = $crate::DatabaseInfo::Test(format!("{}:{}", file!().replace('/', "_"), line!())) .connect() .await - .expect("Database connection failed.") + .expect("Database connection failed."); + + #[allow(clippy::redundant_closure_call)] + (|$db: $crate::Database| $test)(db.clone()).await; + + match db { + $crate::Database::Reference(_) => {} + $crate::Database::MongoDb(db) => db.0.database(&db.1).drop(None).await.unwrap(), + } }; } diff --git a/crates/core/database/src/models/bots/model.rs b/crates/core/database/src/models/bots/model.rs index 05fed343..384d4829 100644 --- a/crates/core/database/src/models/bots/model.rs +++ b/crates/core/database/src/models/bots/model.rs @@ -80,47 +80,47 @@ mod tests { #[async_std::test] async fn crud() { - let db = database_test!("bot_crud"); + database_test!(|db| async move { + let bot_id = "bot"; + let user_id = "user"; + let token = "my_token"; - let bot_id = "bot"; - let user_id = "user"; - let token = "my_token"; - - let bot = Bot { - id: bot_id.to_string(), - owner: user_id.to_string(), - token: token.to_string(), - interactions_url: Some("some url".to_string()), - ..Default::default() - }; - - db.insert_bot(&bot).await.unwrap(); - db.update_bot( - bot_id, - &crate::PartialBot { - public: Some(true), + let bot = Bot { + id: bot_id.to_string(), + owner: user_id.to_string(), + token: token.to_string(), + interactions_url: Some("some url".to_string()), ..Default::default() - }, - vec![FieldsBot::Token, FieldsBot::InteractionsURL], - ) - .await - .unwrap(); + }; - let fetched_bot1 = db.fetch_bot(bot_id).await.unwrap(); - let fetched_bot2 = db.fetch_bot_by_token(&fetched_bot1.token).await.unwrap(); - let fetched_bots = db.fetch_bots_by_user(user_id).await.unwrap(); + db.insert_bot(&bot).await.unwrap(); + db.update_bot( + bot_id, + &crate::PartialBot { + public: Some(true), + ..Default::default() + }, + vec![FieldsBot::Token, FieldsBot::InteractionsURL], + ) + .await + .unwrap(); - assert!(!bot.public); - assert!(fetched_bot1.public); - assert!(bot.interactions_url.is_some()); - assert!(fetched_bot1.interactions_url.is_none()); - assert_ne!(bot.token, fetched_bot1.token); - assert_eq!(fetched_bot1, fetched_bot2); - assert_eq!(fetched_bot1, fetched_bots[0]); - assert_eq!(1, db.get_number_of_bots_by_user(user_id).await.unwrap()); + let fetched_bot1 = db.fetch_bot(bot_id).await.unwrap(); + let fetched_bot2 = db.fetch_bot_by_token(&fetched_bot1.token).await.unwrap(); + let fetched_bots = db.fetch_bots_by_user(user_id).await.unwrap(); - bot.delete(&db).await.unwrap(); - assert!(db.fetch_bot(bot_id).await.is_err()); - assert_eq!(0, db.get_number_of_bots_by_user(user_id).await.unwrap()); + assert!(!bot.public); + assert!(fetched_bot1.public); + assert!(bot.interactions_url.is_some()); + assert!(fetched_bot1.interactions_url.is_none()); + assert_ne!(bot.token, fetched_bot1.token); + assert_eq!(fetched_bot1, fetched_bot2); + assert_eq!(fetched_bot1, fetched_bots[0]); + assert_eq!(1, db.get_number_of_bots_by_user(user_id).await.unwrap()); + + bot.delete(&db).await.unwrap(); + assert!(db.fetch_bot(bot_id).await.is_err()); + assert_eq!(0, db.get_number_of_bots_by_user(user_id).await.unwrap()) + }); } } From 40790de90971e70bb202abab45c9550eabb69d47 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 15:23:20 +0100 Subject: [PATCH 18/56] feat: macros for reducing error boilerplate --- .../database/src/models/bots/ops/mongodb.rs | 49 ++++++++----------- crates/core/result/src/lib.rs | 21 +++++++- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/crates/core/database/src/models/bots/ops/mongodb.rs b/crates/core/database/src/models/bots/ops/mongodb.rs index 7b4c7eee..bda3ae02 100644 --- a/crates/core/database/src/models/bots/ops/mongodb.rs +++ b/crates/core/database/src/models/bots/ops/mongodb.rs @@ -12,31 +12,25 @@ static COL: &str = "bots"; impl AbstractBots for MongoDb { /// Fetch a bot by its id async fn fetch_bot(&self, id: &str) -> Result { - self.find_one_by_id(COL, id) - .await - .map_err(|_| create_database_error!("find_one", COL))? - .ok_or_else(|| create_error!(NotFound)) + query!(self, find_one_by_id, COL, id)?.ok_or_else(|| create_error!(NotFound)) } /// Fetch a bot by its token async fn fetch_bot_by_token(&self, token: &str) -> Result { - self.find_one( + query!( + self, + find_one, COL, doc! { "token": token - }, - ) - .await - .map_err(|_| create_database_error!("find_one", COL))? + } + )? .ok_or_else(|| create_error!(NotFound)) } /// Insert new bot into the database async fn insert_bot(&self, bot: &Bot) -> Result<()> { - self.insert_one(COL, &bot) - .await - .map(|_| ()) - .map_err(|_| create_database_error!("insert_one", COL)) + query!(self, insert_one, COL, &bot).map(|_| ()) } /// Update bot with new information @@ -46,49 +40,46 @@ impl AbstractBots for MongoDb { partial: &PartialBot, remove: Vec, ) -> Result<()> { - self.update_one_by_id( + query!( + self, + update_one_by_id, COL, id, partial, remove.iter().map(|x| x as &dyn IntoDocumentPath).collect(), - None, + None ) - .await .map(|_| ()) - .map_err(|_| create_database_error!("update_one", COL)) } /// Delete a bot from the database async fn delete_bot(&self, id: &str) -> Result<()> { - self.delete_one_by_id(COL, id) - .await - .map(|_| ()) - .map_err(|_| create_database_error!("delete_one", COL)) + query!(self, delete_one_by_id, COL, id).map(|_| ()) } /// Fetch bots owned by a user async fn fetch_bots_by_user(&self, user_id: &str) -> Result> { - self.find( + query!( + self, + find, COL, doc! { "owner": user_id - }, + } ) - .await - .map_err(|_| create_database_error!("find", COL)) } /// Get the number of bots owned by a user async fn get_number_of_bots_by_user(&self, user_id: &str) -> Result { - self.count_documents( + query!( + self, + count_documents, COL, doc! { "owner": user_id - }, + } ) - .await .map(|v| v as usize) - .map_err(|_| create_database_error!("count", COL)) } } diff --git a/crates/core/result/src/lib.rs b/crates/core/result/src/lib.rs index ada4d55f..9a476b3f 100644 --- a/crates/core/result/src/lib.rs +++ b/crates/core/result/src/lib.rs @@ -119,7 +119,7 @@ pub enum ErrorType { #[macro_export] macro_rules! create_error { - ( $error:ident $( $tt:tt )? ) => { + ( $error: ident $( $tt:tt )? ) => { $crate::Error { error_type: $crate::ErrorType::$error $( $tt )?, location: format!("{}:{}:{}", file!(), line!(), column!()), @@ -129,7 +129,7 @@ macro_rules! create_error { #[macro_export] macro_rules! create_database_error { - ( $operation:expr, $collection:expr ) => { + ( $operation: expr, $collection: expr ) => { create_error!(DatabaseError { operation: $operation.to_string(), collection: $collection.to_string() @@ -137,6 +137,23 @@ macro_rules! create_database_error { }; } +#[macro_export] +#[cfg(debug_assertions)] +macro_rules! query { + ( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => { + Ok($self.$type($collection, $($rest),+).await.unwrap()) + }; +} + +#[macro_export] +#[cfg(not(debug_assertions))] +macro_rules! query { + ( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => { + $self.$type($collection, $($rest),+).await + .map_err(|_| create_database_error!(stringify!($type), $collection)) + }; +} + #[cfg(test)] mod tests { use crate::ErrorType; From ace6431cb88a5b84437ee8464b09277f54ed4886 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 15:24:46 +0100 Subject: [PATCH 19/56] fix: ensure database namespace is valid --- crates/core/database/src/lib.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/core/database/src/lib.rs b/crates/core/database/src/lib.rs index fcc19a31..625563f6 100644 --- a/crates/core/database/src/lib.rs +++ b/crates/core/database/src/lib.rs @@ -54,10 +54,14 @@ pub use drivers::*; #[cfg(test)] macro_rules! database_test { ( | $db: ident | $test:expr ) => { - let db = $crate::DatabaseInfo::Test(format!("{}:{}", file!().replace('/', "_"), line!())) - .connect() - .await - .expect("Database connection failed."); + let db = $crate::DatabaseInfo::Test(format!( + "{}:{}", + file!().replace('/', "_").replace(".rs", ""), + line!() + )) + .connect() + .await + .expect("Database connection failed."); #[allow(clippy::redundant_closure_call)] (|$db: $crate::Database| $test)(db.clone()).await; From 69ab7e031ba831b8cff27e59d6d17386be98d29a Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 16:00:30 +0100 Subject: [PATCH 20/56] chore: add linting rules for disallowed methods --- clippy.toml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 clippy.toml diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 00000000..bb3ea02e --- /dev/null +++ b/clippy.toml @@ -0,0 +1,7 @@ +disallowed-methods = [ + # Shouldn't need to access these directly + "revolt_database::models::bots::model::Bot::remove_field", + + # Prefer to use Object::delete() + "revolt_database::models::bots::ops::AbstractBots::update_bot", +] From b8cda2ec74d96ea0aab9a5d48d432b88420a9022 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 16:00:57 +0100 Subject: [PATCH 21/56] feat: add drop_database for tests --- crates/core/database/src/models/admin_migrations/ops.rs | 4 ++++ .../database/src/models/admin_migrations/ops/mongodb.rs | 6 ++++++ .../database/src/models/admin_migrations/ops/reference.rs | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/crates/core/database/src/models/admin_migrations/ops.rs b/crates/core/database/src/models/admin_migrations/ops.rs index f5de7a88..ff8ae25d 100644 --- a/crates/core/database/src/models/admin_migrations/ops.rs +++ b/crates/core/database/src/models/admin_migrations/ops.rs @@ -3,6 +3,10 @@ mod reference; #[async_trait] pub trait AbstractMigrations: Sync + Send { + #[cfg(test)] + /// Drop the database + async fn drop_database(&self); + /// Migrate the database async fn migrate_database(&self) -> Result<(), ()>; } diff --git a/crates/core/database/src/models/admin_migrations/ops/mongodb.rs b/crates/core/database/src/models/admin_migrations/ops/mongodb.rs index 193e0973..784f59fa 100644 --- a/crates/core/database/src/models/admin_migrations/ops/mongodb.rs +++ b/crates/core/database/src/models/admin_migrations/ops/mongodb.rs @@ -7,6 +7,12 @@ mod scripts; #[async_trait] impl AbstractMigrations for MongoDb { + #[cfg(test)] + /// Drop the database + async fn drop_database(&self) { + self.db().drop(None).await.ok(); + } + /// Migrate the database async fn migrate_database(&self) -> Result<(), ()> { info!("Migrating the database."); diff --git a/crates/core/database/src/models/admin_migrations/ops/reference.rs b/crates/core/database/src/models/admin_migrations/ops/reference.rs index 19f3b99e..a0913ae0 100644 --- a/crates/core/database/src/models/admin_migrations/ops/reference.rs +++ b/crates/core/database/src/models/admin_migrations/ops/reference.rs @@ -4,6 +4,10 @@ use super::AbstractMigrations; #[async_trait] impl AbstractMigrations for ReferenceDb { + #[cfg(test)] + /// Drop the database + async fn drop_database(&self) {} + /// Migrate the database async fn migrate_database(&self) -> Result<(), ()> { // Here you would do your typical migrations if this was a real database. From eacf4decaba8867da5a3b9a34e48f721c1fd4037 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 16:01:07 +0100 Subject: [PATCH 22/56] refactor: clean up database before and after tests --- crates/core/database/src/lib.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/core/database/src/lib.rs b/crates/core/database/src/lib.rs index 625563f6..e53c8231 100644 --- a/crates/core/database/src/lib.rs +++ b/crates/core/database/src/lib.rs @@ -63,13 +63,12 @@ macro_rules! database_test { .await .expect("Database connection failed."); + db.drop_database().await; + #[allow(clippy::redundant_closure_call)] (|$db: $crate::Database| $test)(db.clone()).await; - match db { - $crate::Database::Reference(_) => {} - $crate::Database::MongoDb(db) => db.0.database(&db.1).drop(None).await.unwrap(), - } + db.drop_database().await }; } From a9c82791b31e4f5ab516b5112b15cb2af5650613 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 16:01:27 +0100 Subject: [PATCH 23/56] fix: force OOP bot updates to avoid inconsistencies --- crates/core/database/src/models/bots/model.rs | 50 ++++++++++++++----- .../database/src/models/bots/ops/reference.rs | 3 +- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/crates/core/database/src/models/bots/model.rs b/crates/core/database/src/models/bots/model.rs index 384d4829..6e284125 100644 --- a/crates/core/database/src/models/bots/model.rs +++ b/crates/core/database/src/models/bots/model.rs @@ -56,9 +56,10 @@ auto_derived!( } ); +#[allow(clippy::disallowed_methods)] impl Bot { /// Remove a field from this object - pub fn remove(&mut self, field: &FieldsBot) { + pub fn remove_field(&mut self, field: &FieldsBot) { match field { FieldsBot::Token => self.token = nanoid::nanoid!(64), FieldsBot::InteractionsURL => { @@ -67,6 +68,27 @@ impl Bot { } } + /// Update this bot + pub async fn update( + &mut self, + db: &Database, + mut partial: PartialBot, + remove: Vec, + ) -> Result<()> { + if remove.contains(&FieldsBot::Token) { + partial.token = Some(nanoid::nanoid!(64)); + } + + for field in &remove { + self.remove_field(field); + } + + db.update_bot(&self.id, &partial, remove).await?; + + self.apply_options(partial); + Ok(()) + } + /// Delete this bot pub async fn delete(&self, db: &Database) -> Result<()> { // db.fetch_user(&self.id).await?.mark_deleted(db).await?; @@ -76,7 +98,7 @@ impl Bot { #[cfg(test)] mod tests { - use crate::{Bot, FieldsBot}; + use crate::{Bot, FieldsBot, PartialBot}; #[async_std::test] async fn crud() { @@ -94,16 +116,19 @@ mod tests { }; db.insert_bot(&bot).await.unwrap(); - db.update_bot( - bot_id, - &crate::PartialBot { - public: Some(true), - ..Default::default() - }, - vec![FieldsBot::Token, FieldsBot::InteractionsURL], - ) - .await - .unwrap(); + + let mut updated_bot = bot.clone(); + updated_bot + .update( + &db, + PartialBot { + public: Some(true), + ..Default::default() + }, + vec![FieldsBot::Token, FieldsBot::InteractionsURL], + ) + .await + .unwrap(); let fetched_bot1 = db.fetch_bot(bot_id).await.unwrap(); let fetched_bot2 = db.fetch_bot_by_token(&fetched_bot1.token).await.unwrap(); @@ -114,6 +139,7 @@ mod tests { assert!(bot.interactions_url.is_some()); assert!(fetched_bot1.interactions_url.is_none()); assert_ne!(bot.token, fetched_bot1.token); + assert_eq!(updated_bot, fetched_bot1); assert_eq!(fetched_bot1, fetched_bot2); assert_eq!(fetched_bot1, fetched_bots[0]); assert_eq!(1, db.get_number_of_bots_by_user(user_id).await.unwrap()); diff --git a/crates/core/database/src/models/bots/ops/reference.rs b/crates/core/database/src/models/bots/ops/reference.rs index 26ce98c6..3a8caff2 100644 --- a/crates/core/database/src/models/bots/ops/reference.rs +++ b/crates/core/database/src/models/bots/ops/reference.rs @@ -43,7 +43,8 @@ impl AbstractBots for ReferenceDb { let mut bots = self.bots.lock().await; if let Some(bot) = bots.get_mut(id) { for field in remove { - bot.remove(&field); + #[allow(clippy::disallowed_methods)] + bot.remove_field(&field); } bot.apply_options(partial.clone()); From 736220a94e93b11eea5100fbd59f80266c906a19 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 16:03:08 +0100 Subject: [PATCH 24/56] ci: run tests with and without MongoDB --- .github/workflows/rust.yaml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index c7f57608..cad2a73f 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -32,16 +32,23 @@ jobs: with: command: test - - name: Copy .env.example - if: github.event_name != 'pull_request' - run: | - cp .env.example .env - - name: Run services in background if: github.event_name != 'pull_request' run: | docker-compose -f docker-compose.db.yml up -d + - name: Run cargo test (with MongoDB) + uses: actions-rs/cargo@v1 + env: + MONGODB: mongodb://localhost + with: + command: test + + - name: Copy .env.example + if: github.event_name != 'pull_request' + run: | + cp .env.example .env + - name: Start API in background if: github.event_name != 'pull_request' run: | From 0054019f825cda5e7fdb8e5bbd704e0a022f745e Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 16:05:40 +0100 Subject: [PATCH 25/56] ci: run test on all pushes --- .github/workflows/rust.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index cad2a73f..ca9d9d56 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -2,7 +2,6 @@ name: Rust build, test, and generate specification on: push: - branches: [master] pull_request: branches: [master] @@ -33,7 +32,6 @@ jobs: command: test - name: Run services in background - if: github.event_name != 'pull_request' run: | docker-compose -f docker-compose.db.yml up -d @@ -45,23 +43,23 @@ jobs: command: test - name: Copy .env.example - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.ref_name == 'master' run: | cp .env.example .env - name: Start API in background - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.ref_name == 'master' run: | cargo run --bin revolt-delta & - name: Wait for API to go up - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.ref_name == 'master' uses: nev7n/wait_for_response@v1 with: url: "http://localhost:8000/" - name: Checkout API repository - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.ref_name == 'master' uses: actions/checkout@v3 with: repository: revoltchat/api @@ -69,11 +67,11 @@ jobs: token: ${{ secrets.PAT }} - name: Download OpenAPI specification - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.ref_name == 'master' run: curl http://localhost:8000/openapi.json -o api/OpenAPI.json - name: Commit changes - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.ref_name == 'master' uses: EndBug/add-and-commit@v4 with: cwd: "api" From b93dd90caf01a3f4c7843a3f59dc2724df6b8854 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 17:54:46 +0100 Subject: [PATCH 26/56] refactor(core/database): use empty strings instead of options --- crates/core/database/src/models/bots/model.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/core/database/src/models/bots/model.rs b/crates/core/database/src/models/bots/model.rs index 6e284125..a8c551e0 100644 --- a/crates/core/database/src/models/bots/model.rs +++ b/crates/core/database/src/models/bots/model.rs @@ -25,14 +25,14 @@ auto_derived_partial!( #[serde(skip_serializing_if = "crate::if_false", default)] pub discoverable: bool, /// Reserved; URL for handling interactions - #[serde(skip_serializing_if = "Option::is_none")] - pub interactions_url: Option, + #[serde(skip_serializing_if = "String::is_empty", default)] + pub interactions_url: String, /// URL for terms of service - #[serde(skip_serializing_if = "Option::is_none")] - pub terms_of_service_url: Option, + #[serde(skip_serializing_if = "String::is_empty", default)] + pub terms_of_service_url: String, /// URL for privacy policy - #[serde(skip_serializing_if = "Option::is_none")] - pub privacy_policy_url: Option, + #[serde(skip_serializing_if = "String::is_empty", default)] + pub privacy_policy_url: String, /// Enum of bot flags #[serde(skip_serializing_if = "Option::is_none")] @@ -63,7 +63,7 @@ impl Bot { match field { FieldsBot::Token => self.token = nanoid::nanoid!(64), FieldsBot::InteractionsURL => { - self.interactions_url.take(); + self.interactions_url = String::new(); } } } @@ -111,7 +111,7 @@ mod tests { id: bot_id.to_string(), owner: user_id.to_string(), token: token.to_string(), - interactions_url: Some("some url".to_string()), + interactions_url: "some url".to_string(), ..Default::default() }; @@ -136,8 +136,8 @@ mod tests { assert!(!bot.public); assert!(fetched_bot1.public); - assert!(bot.interactions_url.is_some()); - assert!(fetched_bot1.interactions_url.is_none()); + assert!(!bot.interactions_url.is_empty()); + assert!(fetched_bot1.interactions_url.is_empty()); assert_ne!(bot.token, fetched_bot1.token); assert_eq!(updated_bot, fetched_bot1); assert_eq!(fetched_bot1, fetched_bot2); From 8b30dddc067497f4ec958dbfaa56979773a0a6c8 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 17:55:12 +0100 Subject: [PATCH 27/56] chore(core/result): add schema feature --- crates/core/result/Cargo.toml | 5 +++++ crates/core/result/src/lib.rs | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index 12947827..ce96fe85 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -7,8 +7,13 @@ edition = "2021" [features] serde = [ "dep:serde" ] +schemas = [ "dep:schemars" ] + default = [ "serde" ] [dependencies] # Serialisation serde = { version = "1", features = ["derive"], optional = true } + +# Spec Generation +schemars = { version = "0.8.8", optional = true } diff --git a/crates/core/result/src/lib.rs b/crates/core/result/src/lib.rs index 9a476b3f..7e0122da 100644 --- a/crates/core/result/src/lib.rs +++ b/crates/core/result/src/lib.rs @@ -2,11 +2,16 @@ #[macro_use] extern crate serde; +#[cfg(feature = "schemas")] +#[macro_use] +extern crate schemars; + /// Result type with custom Error pub type Result = std::result::Result; /// Error information #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "schemas", derive(JsonSchema))] #[derive(Debug, Clone)] pub struct Error { /// Type of error and additional information @@ -20,6 +25,7 @@ pub struct Error { /// Possible error types #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", serde(tag = "type"))] +#[cfg_attr(feature = "schemas", derive(JsonSchema))] #[derive(Debug, Clone)] pub enum ErrorType { /// This error was not labeled :( From 11a87263be99fc785654aad810f15b2de0f162b5 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 17:55:30 +0100 Subject: [PATCH 28/56] feat(core/models): implement Bot and PublicBot --- crates/core/models/Cargo.toml | 15 +++++ crates/core/models/src/bots.rs | 103 +++++++++++++++++++++++++++++++++ crates/core/models/src/lib.rs | 26 +++++++++ 3 files changed, 144 insertions(+) create mode 100644 crates/core/models/src/bots.rs diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 812e083f..5f6f64e3 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -5,4 +5,19 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +serde = [ "dep:serde" ] +schemas = [ "dep:schemars" ] +from_database = [ "revolt-database" ] + +default = [ "serde", "from_database" ] + [dependencies] +# Repo +revolt-database = { path = "../database", optional = true } + +# Serialisation +serde = { version = "1", features = ["derive"], optional = true } + +# Spec Generation +schemars = { version = "0.8.8", optional = true } diff --git a/crates/core/models/src/bots.rs b/crates/core/models/src/bots.rs new file mode 100644 index 00000000..7c201e68 --- /dev/null +++ b/crates/core/models/src/bots.rs @@ -0,0 +1,103 @@ +auto_derived!( + /// # Bot + pub struct Bot { + /// Bot Id + #[serde(rename = "_id")] + pub id: String, + + /// User Id of the bot owner + #[serde(rename = "owner")] + pub owner_id: String, + /// Token used to authenticate requests for this bot + pub token: String, + /// Whether the bot is public + /// (may be invited by anyone) + pub public: bool, + + /// Whether to enable analytics + #[cfg_attr( + feature = "serde", + serde(skip_serializing_if = "crate::if_false", default) + )] + pub analytics: bool, + /// Whether this bot should be publicly discoverable + #[cfg_attr( + feature = "serde", + serde(skip_serializing_if = "crate::if_false", default) + )] + pub discoverable: bool, + /// Reserved; URL for handling interactions + #[cfg_attr( + feature = "serde", + serde(skip_serializing_if = "String::is_empty", default) + )] + pub interactions_url: String, + /// URL for terms of service + #[cfg_attr( + feature = "serde", + serde(skip_serializing_if = "String::is_empty", default) + )] + pub terms_of_service_url: String, + /// URL for privacy policy + #[cfg_attr( + feature = "serde", + serde(skip_serializing_if = "String::is_empty", default) + )] + pub privacy_policy_url: String, + + /// Enum of bot flags + #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] + pub flags: Option, + } + + /// # Public Bot + pub struct PublicBot { + /// Bot Id + #[serde(rename = "_id")] + id: String, + + /// Bot Username + username: String, + /// Profile Avatar + #[serde(skip_serializing_if = "String::is_empty")] + avatar: String, + /// Profile Description + #[serde(skip_serializing_if = "String::is_empty")] + description: String, + } +); + +#[cfg(feature = "from_database")] +impl PublicBot { + pub fn from( + bot: revolt_database::Bot, + username: String, + avatar: Option, + description: Option, + ) -> Self { + PublicBot { + id: bot.id, + username, + avatar: avatar.unwrap_or_default(), + description: description.unwrap_or_default(), + } + } +} + +#[cfg(feature = "from_database")] +impl From for Bot { + fn from(value: revolt_database::Bot) -> Self { + Bot { + id: value.id, + owner_id: value.owner, + token: value.token, + public: value.public, + analytics: value.analytics, + discoverable: value.discoverable, + interactions_url: value.interactions_url, + terms_of_service_url: value.terms_of_service_url, + privacy_policy_url: value.privacy_policy_url, + flags: value.flags, + } + } +} diff --git a/crates/core/models/src/lib.rs b/crates/core/models/src/lib.rs index 8b137891..21f199c8 100644 --- a/crates/core/models/src/lib.rs +++ b/crates/core/models/src/lib.rs @@ -1 +1,27 @@ +#[cfg(feature = "serde")] +#[macro_use] +extern crate serde; +#[cfg(feature = "schemas")] +#[macro_use] +extern crate schemars; + +macro_rules! auto_derived { + ( $( $item:item )+ ) => { + $( + #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] + #[cfg_attr(feature = "schemas", derive(JsonSchema))] + #[derive(Debug, Clone, Eq, PartialEq)] + $item + )+ + }; +} + +mod bots; + +pub use bots::*; + +/// Utility function to check if a boolean value is false +pub fn if_false(t: &bool) -> bool { + !t +} From bbe1f4936c8cc6860910f3fd3f72724ee32a8064 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 17:56:09 +0100 Subject: [PATCH 29/56] chore: bridge gap between core/result and quark --- crates/quark/Cargo.toml | 3 +++ crates/quark/src/util/result.rs | 22 +++++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/quark/Cargo.toml b/crates/quark/Cargo.toml index 43a22a9d..5b37b88e 100644 --- a/crates/quark/Cargo.toml +++ b/crates/quark/Cargo.toml @@ -89,3 +89,6 @@ authifier = { version = "1.0.7", features = [ "async-std-runtime" ] } # Sentry sentry = "0.25.0" + +# Core +revolt-result = { path = "../core/result", features = [ "serde", "schemas" ] } diff --git a/crates/quark/src/util/result.rs b/crates/quark/src/util/result.rs index 8fb449d0..a717d999 100644 --- a/crates/quark/src/util/result.rs +++ b/crates/quark/src/util/result.rs @@ -19,6 +19,12 @@ pub enum Error { /// This error was not labeled :( LabelMe, + /// Core crate error + Core { + #[serde(flatten)] + error: revolt_result::Error, + }, + // ? Onboarding related errors AlreadyOnboarded, @@ -39,13 +45,13 @@ pub enum Error { CannotEditMessage, CannotJoinCall, TooManyAttachments { - max: usize + max: usize, }, TooManyReplies { - max: usize + max: usize, }, TooManyChannels { - max: usize + max: usize, }, EmptyMessage, PayloadTooLarge, @@ -64,10 +70,10 @@ pub enum Error { max: usize, }, TooManyEmoji { - max: usize + max: usize, }, TooManyRoles { - max: usize + max: usize, }, // ? Bot related errors @@ -135,6 +141,11 @@ impl Error { error: validation_error, }) } + + /// Create a error from core error + pub fn from_core(error: revolt_result::Error) -> Error { + Error::Core { error } + } } /// Result type with custom Error @@ -145,6 +156,7 @@ impl<'r> Responder<'r, 'static> for Error { fn respond_to(self, _: &'r Request<'_>) -> response::Result<'static> { let status = match self { Error::LabelMe => Status::InternalServerError, + Error::Core { .. } => Status::InternalServerError, Error::AlreadyOnboarded => Status::Forbidden, From 633eb786308364af738a1c5b1ef3fade7f20518d Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 17:56:24 +0100 Subject: [PATCH 30/56] chore: migrate to revolt_optional_struct --- crates/core/database/Cargo.toml | 2 +- crates/core/database/src/lib.rs | 2 +- crates/quark/Cargo.toml | 2 +- crates/quark/src/lib.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index b17ddb45..a0cc695e 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -25,8 +25,8 @@ nanoid = "0.4.0" # Serialisation serde_json = "1" +revolt_optional_struct = "0.2.0" serde = { version = "1", features = ["derive"] } -optional_struct = { git = "https://github.com/insertish/OptionalStruct", rev = "ee56427cee1f007839825d93d07fffd5a5e038c7" } # Database mongodb = { optional = true, version = "2.1.0", default-features = false } diff --git a/crates/core/database/src/lib.rs b/crates/core/database/src/lib.rs index e53c8231..4796b684 100644 --- a/crates/core/database/src/lib.rs +++ b/crates/core/database/src/lib.rs @@ -11,7 +11,7 @@ extern crate async_trait; extern crate log; #[macro_use] -extern crate optional_struct; +extern crate revolt_optional_struct; #[macro_use] extern crate revolt_result; diff --git a/crates/quark/Cargo.toml b/crates/quark/Cargo.toml index 5b37b88e..eeffa630 100644 --- a/crates/quark/Cargo.toml +++ b/crates/quark/Cargo.toml @@ -26,10 +26,10 @@ default = [ "test" ] [dependencies] # Serialisation +revolt_optional_struct = "0.2.0" serde = { version = "1", features = ["derive"] } validator = { version = "0.14", features = ["derive"] } iso8601-timestamp = { version = "0.1.8", features = ["schema", "bson"] } -optional_struct = { git = "https://github.com/insertish/OptionalStruct", rev = "ee56427cee1f007839825d93d07fffd5a5e038c7" } # Formats bincode = "1.3.3" diff --git a/crates/quark/src/lib.rs b/crates/quark/src/lib.rs index 13b1ce82..f57d9127 100644 --- a/crates/quark/src/lib.rs +++ b/crates/quark/src/lib.rs @@ -9,7 +9,7 @@ extern crate log; #[macro_use] extern crate impl_ops; #[macro_use] -extern crate optional_struct; +extern crate revolt_optional_struct; #[macro_use] extern crate bitfield; #[macro_use] From 403a94f70c858c914af5b9fde98d2feec5ec8c87 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 17:56:48 +0100 Subject: [PATCH 31/56] feat(delta): example implementation of new core models --- Cargo.lock | 41 ++++++++++------- crates/delta/Cargo.toml | 1 + crates/delta/src/routes/bots/fetch.rs | 23 ++++++---- crates/delta/src/routes/bots/fetch_public.rs | 48 ++++++++------------ 4 files changed, 58 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 24e4e775..bf73f49c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2186,15 +2186,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "optional_struct" -version = "0.2.0" -source = "git+https://github.com/insertish/OptionalStruct?rev=ee56427cee1f007839825d93d07fffd5a5e038c7#ee56427cee1f007839825d93d07fffd5a5e038c7" -dependencies = [ - "quote 0.3.15", - "syn 0.11.11", -] - [[package]] name = "os_info" version = "3.4.0" @@ -2835,8 +2826,8 @@ dependencies = [ "log", "mongodb", "nanoid", - "optional_struct", "revolt-result", + "revolt_optional_struct", "serde", "serde_json", ] @@ -2864,6 +2855,7 @@ dependencies = [ "regex", "reqwest", "revolt-database", + "revolt-models", "revolt-quark", "revolt_rocket_okapi", "rocket", @@ -2881,6 +2873,11 @@ dependencies = [ [[package]] name = "revolt-models" version = "0.1.0" +dependencies = [ + "revolt-database", + "schemars", + "serde", +] [[package]] name = "revolt-quark" @@ -2909,13 +2906,14 @@ dependencies = [ "nanoid", "num_enum", "once_cell", - "optional_struct", "pretty_env_logger", "rand 0.8.5", "redis-kiss", "regex", "reqwest", + "revolt-result", "revolt_okapi", + "revolt_optional_struct", "revolt_rocket_okapi", "rocket", "rocket_cors", @@ -2934,6 +2932,7 @@ dependencies = [ name = "revolt-result" version = "0.1.0" dependencies = [ + "schemars", "serde", ] @@ -2949,6 +2948,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "revolt_optional_struct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d342739658623fe9d72b42f7a36ba19094d1d9bf9b423c3e337ab203c15d53e" +dependencies = [ + "quote 0.3.15", + "syn 0.11.11", +] + [[package]] name = "revolt_rocket_okapi" version = "0.9.1" @@ -3408,9 +3417,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" dependencies = [ "serde_derive", ] @@ -3426,13 +3435,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" dependencies = [ "proc-macro2", "quote 1.0.26", - "syn 1.0.107", + "syn 2.0.15", ] [[package]] diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index 6eea2d4a..b4c7cde6 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -57,6 +57,7 @@ revolt-quark = { path = "../quark" } # core revolt-database = { path = "../core/database" } +revolt-models = { path = "../core/models", features = [ "schemas" ] } [build-dependencies] vergen = "7.5.0" diff --git a/crates/delta/src/routes/bots/fetch.rs b/crates/delta/src/routes/bots/fetch.rs index a6f94a4f..19a16ee0 100644 --- a/crates/delta/src/routes/bots/fetch.rs +++ b/crates/delta/src/routes/bots/fetch.rs @@ -1,11 +1,11 @@ -use revolt_quark::{ - models::{Bot, User}, - Db, Error, Ref, Result, -}; -use rocket::serde::json::Json; +use revolt_database::Database; +use revolt_models::Bot; +use revolt_quark::{models::User, Db, Error, Ref, Result}; +use rocket::{serde::json::Json, State}; use serde::Serialize; /// # Bot Response +/// TODO: move to revolt-models #[derive(Serialize, JsonSchema)] pub struct BotResponse { /// Bot object @@ -19,18 +19,23 @@ pub struct BotResponse { /// Fetch details of a bot you own by its id. #[openapi(tag = "Bots")] #[get("/")] -pub async fn fetch_bot(db: &Db, user: User, target: Ref) -> Result> { +pub async fn fetch_bot( + legacy_db: &Db, + db: &State, + user: User, + target: Ref, +) -> Result> { if user.bot.is_some() { return Err(Error::IsBot); } - let bot = target.as_bot(db).await?; + let bot = db.fetch_bot(&target.id).await.map_err(Error::from_core)?; if bot.owner != user.id { return Err(Error::NotFound); } Ok(Json(BotResponse { - user: db.fetch_user(&bot.id).await?.foreign(), - bot, + user: legacy_db.fetch_user(&bot.id).await?.foreign(), + bot: bot.into(), })) } diff --git a/crates/delta/src/routes/bots/fetch_public.rs b/crates/delta/src/routes/bots/fetch_public.rs index 03e80a65..1473eb65 100644 --- a/crates/delta/src/routes/bots/fetch_public.rs +++ b/crates/delta/src/routes/bots/fetch_public.rs @@ -1,44 +1,32 @@ -use revolt_quark::{ - models::{File, User}, - Db, Error, Ref, Result, -}; +use revolt_database::Database; +use revolt_models::PublicBot; +use revolt_quark::{models::User, Db, Error, Ref, Result}; use rocket::serde::json::Json; -use serde::{Deserialize, Serialize}; - -/// # Public Bot -#[derive(Serialize, Deserialize, JsonSchema)] -pub struct PublicBot { - /// Bot Id - #[serde(rename = "_id")] - id: String, - /// Bot Username - username: String, - /// Profile Avatar - #[serde(skip_serializing_if = "Option::is_none")] - avatar: Option, - /// Profile Description - #[serde(skip_serializing_if = "Option::is_none")] - description: Option, -} +use rocket::State; /// # Fetch Public Bot /// /// Fetch details of a public (or owned) bot by its id. #[openapi(tag = "Bots")] #[get("//invite")] -pub async fn fetch_public_bot(db: &Db, user: Option, target: Ref) -> Result> { - let bot = target.as_bot(db).await?; +pub async fn fetch_public_bot( + legacy_db: &Db, + db: &State, + user: Option, + target: Ref, +) -> Result> { + let bot = db.fetch_bot(&target.id).await.map_err(Error::from_core)?; if !bot.public && user.map_or(true, |x| x.id != bot.owner) { return Err(Error::NotFound); } - let user = db.fetch_user(&bot.id).await?; + let user = legacy_db.fetch_user(&bot.id).await?; - Ok(Json(PublicBot { - id: bot.id, - username: user.username, - avatar: user.avatar, - description: user.profile.and_then(|p| p.content), - })) + Ok(Json(PublicBot::from( + bot, + user.username, + user.avatar.map(|f| f.id), + user.profile.and_then(|p| p.content), + ))) } From e84d55a69752a2c3435d676f1ff3ec679b82a7ec Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 18:01:06 +0100 Subject: [PATCH 32/56] chore: update Cargo.toml for core crates --- Cargo.lock | 6 +++--- crates/core/database/Cargo.toml | 5 ++++- crates/core/models/Cargo.toml | 7 +++++-- crates/core/result/Cargo.toml | 5 ++++- crates/quark/Cargo.toml | 2 +- justfile | 4 ++++ 6 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 justfile diff --git a/Cargo.lock b/Cargo.lock index bf73f49c..02235890 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2816,7 +2816,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.1.0" +version = "0.0.1" dependencies = [ "async-recursion", "async-std", @@ -2872,7 +2872,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.1.0" +version = "0.0.1" dependencies = [ "revolt-database", "schemars", @@ -2930,7 +2930,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.1.0" +version = "0.0.1" dependencies = [ "schemars", "serde", diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index a0cc695e..029d4d1c 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "revolt-database" -version = "0.1.0" +version = "0.0.1" edition = "2021" +license = "AGPL-3.0-or-later" +authors = [ "Paul Makles " ] +description = "Revolt Backend: Database Implementation" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 5f6f64e3..152363ad 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "revolt-models" -version = "0.1.0" +version = "0.0.1" edition = "2021" +license = "AGPL-3.0-or-later" +authors = [ "Paul Makles " ] +description = "Revolt Backend: API Models" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -14,7 +17,7 @@ default = [ "serde", "from_database" ] [dependencies] # Repo -revolt-database = { path = "../database", optional = true } +revolt-database = { version = "0.0.1", path = "../database", optional = true } # Serialisation serde = { version = "1", features = ["derive"], optional = true } diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index ce96fe85..5f3820db 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "revolt-result" -version = "0.1.0" +version = "0.0.1" edition = "2021" +license = "AGPL-3.0-or-later" +authors = [ "Paul Makles " ] +description = "Revolt Backend: Result and Error types" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/crates/quark/Cargo.toml b/crates/quark/Cargo.toml index eeffa630..ef686897 100644 --- a/crates/quark/Cargo.toml +++ b/crates/quark/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "revolt-quark" version = "0.5.19" -license = "AGPL-3.0-or-later" edition = "2021" +license = "AGPL-3.0-or-later" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/justfile b/justfile new file mode 100644 index 00000000..0b633151 --- /dev/null +++ b/justfile @@ -0,0 +1,4 @@ +publish: + cargo publish --package revolt-database + cargo publish --package revolt-models + cargo publish --package revolt-result From 2a9cc3190cc37a369acaac2bb0509e3a64dee982 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 18:01:44 +0100 Subject: [PATCH 33/56] fix: forgot to specify version for result crate --- crates/core/database/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 029d4d1c..153c0636 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -19,8 +19,8 @@ async-std-runtime = [ "async-std" ] default = [ "mongodb", "async-std-runtime" ] [dependencies] -# Repo -revolt-result = { path = "../result" } +# Core +revolt-result = { version = "0.0.1", path = "../result" } # Utility log = "*" From 8a230ba9894a981fda5ce08fcf5785aaed9381aa Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 18:02:01 +0100 Subject: [PATCH 34/56] chore: change publishing order --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 0b633151..778aeaa4 100644 --- a/justfile +++ b/justfile @@ -1,4 +1,4 @@ publish: + cargo publish --package revolt-result cargo publish --package revolt-database cargo publish --package revolt-models - cargo publish --package revolt-result From c817c2dd4069393c132c9baf1b3a1f58deb4f8be Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 18:03:19 +0100 Subject: [PATCH 35/56] fix: don't use wildcard for log dependency --- crates/core/database/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 153c0636..336c01cf 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -23,7 +23,7 @@ default = [ "mongodb", "async-std-runtime" ] revolt-result = { version = "0.0.1", path = "../result" } # Utility -log = "*" +log = "0.4" nanoid = "0.4.0" # Serialisation From 12d963d2bd1a576775128c26e77af84b77c7576e Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 20:01:23 +0100 Subject: [PATCH 36/56] refactor: create presence crate and add tests --- Cargo.lock | 18 ++- crates/bonfire/Cargo.toml | 3 + crates/bonfire/src/main.rs | 4 +- crates/bonfire/src/websocket.rs | 7 +- crates/core/presence/Cargo.toml | 19 ++++ .../mod.rs => core/presence/src/lib.rs} | 103 ++++++++++++++---- .../presence/src}/operations.rs | 0 crates/quark/Cargo.toml | 1 + crates/quark/examples/test.rs | 34 ------ crates/quark/src/events/impl.rs | 9 +- .../src/impl/generic/channels/message.rs | 4 +- crates/quark/src/impl/generic/users/user.rs | 4 +- crates/quark/src/lib.rs | 1 - crates/quark/src/presence/entry.rs | 13 --- crates/quark/src/util/ref.rs | 4 +- 15 files changed, 136 insertions(+), 88 deletions(-) create mode 100644 crates/core/presence/Cargo.toml rename crates/{quark/src/presence/mod.rs => core/presence/src/lib.rs} (60%) rename crates/{quark/src/presence => core/presence/src}/operations.rs (100%) delete mode 100644 crates/quark/examples/test.rs delete mode 100644 crates/quark/src/presence/entry.rs diff --git a/Cargo.lock b/Cargo.lock index 02235890..e1b7cd1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -210,9 +210,9 @@ dependencies = [ [[package]] name = "async-std" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52580991739c5cdb36cde8b2a516371c0a3b70dda36d916cc08b82372916808c" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" dependencies = [ "async-attributes", "async-channel", @@ -229,7 +229,6 @@ dependencies = [ "kv-log-macro", "log", "memchr", - "num_cpus", "once_cell", "pin-project-lite 0.2.9", "pin-utils", @@ -2808,6 +2807,7 @@ dependencies = [ "log", "once_cell", "querystring", + "revolt-presence", "revolt-quark", "rmp-serde", "serde", @@ -2879,6 +2879,17 @@ dependencies = [ "serde", ] +[[package]] +name = "revolt-presence" +version = "0.0.1" +dependencies = [ + "async-std", + "log", + "once_cell", + "rand 0.8.5", + "redis-kiss", +] + [[package]] name = "revolt-quark" version = "0.5.19" @@ -2911,6 +2922,7 @@ dependencies = [ "redis-kiss", "regex", "reqwest", + "revolt-presence", "revolt-result", "revolt_okapi", "revolt_optional_struct", diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index 14dca258..c7d4604d 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -26,3 +26,6 @@ serde = "1.0.136" futures = "0.3.21" async-tungstenite = { version = "0.17.0", features = ["async-std-runtime"] } async-std = { version = "1.8.0", features = ["tokio1", "tokio02", "attributes"] } + +# core +revolt-presence = { path = "../core/presence" } diff --git a/crates/bonfire/src/main.rs b/crates/bonfire/src/main.rs index 8fe3b2f3..e9bcef88 100644 --- a/crates/bonfire/src/main.rs +++ b/crates/bonfire/src/main.rs @@ -1,7 +1,7 @@ use std::env; use async_std::net::TcpListener; -use revolt_quark::presence::presence_clear_region; +use revolt_presence::clear_region; #[macro_use] extern crate log; @@ -18,7 +18,7 @@ async fn main() { database::connect().await; // Clean up the current region information. - presence_clear_region(None).await; + clear_region(None).await; // Setup a TCP listener to accept WebSocket connections on. // By default, we bind to port 9000 on all interfaces. diff --git a/crates/bonfire/src/websocket.rs b/crates/bonfire/src/websocket.rs index e298db3e..c2bbead6 100644 --- a/crates/bonfire/src/websocket.rs +++ b/crates/bonfire/src/websocket.rs @@ -1,6 +1,7 @@ use std::net::SocketAddr; use futures::{channel::oneshot, pin_mut, select, FutureExt, SinkExt, StreamExt, TryStreamExt}; +use revolt_presence::{create_session, delete_session}; use revolt_quark::{ events::{ client::EventV1, @@ -8,7 +9,6 @@ use revolt_quark::{ state::{State, SubscriptionStateChange}, }, models::{user::UserHint, User}, - presence::{presence_create_session, presence_delete_session}, redis_kiss, Database, }; @@ -69,8 +69,7 @@ pub fn spawn_client(db: &'static Database, stream: TcpStream, addr: SocketAddr) let user_id = state.cache.user_id.clone(); // Create presence session. - let (first_session, session_id) = - presence_create_session(&user_id, 0).await; + let (first_session, session_id) = create_session(&user_id, 0).await; // Notify socket we have authenticated. write @@ -225,7 +224,7 @@ pub fn spawn_client(db: &'static Database, stream: TcpStream, addr: SocketAddr) } // Clean up presence session. - let last_session = presence_delete_session(&user_id, session_id).await; + let last_session = delete_session(&user_id, session_id).await; // If this was the last session, notify other users that we just went offline. if last_session { diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml new file mode 100644 index 00000000..7f904775 --- /dev/null +++ b/crates/core/presence/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "revolt-presence" +version = "0.0.1" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dev-dependencies] +# Async +async-std = { version = "1.8.0", features = ["attributes"] } + +[dependencies] +# Utility +log = "0.4.17" +rand = "0.8.5" +once_cell = "1.17.1" + +# Redis +redis-kiss = "0.1.4" diff --git a/crates/quark/src/presence/mod.rs b/crates/core/presence/src/lib.rs similarity index 60% rename from crates/quark/src/presence/mod.rs rename to crates/core/presence/src/lib.rs index 6505ee3c..67c0722e 100644 --- a/crates/quark/src/presence/mod.rs +++ b/crates/core/presence/src/lib.rs @@ -1,20 +1,31 @@ +#[macro_use] +extern crate log; + +use once_cell::sync::Lazy; +use rand::Rng; +use redis_kiss::{get_connection, AsyncCommands}; use std::collections::HashSet; -use redis_kiss::{get_connection, AsyncCommands}; - -use rand::Rng; -mod entry; mod operations; - use operations::{ __add_to_set_string, __add_to_set_u32, __delete_key, __get_set_members_as_string, __get_set_size, __remove_from_set_string, __remove_from_set_u32, }; -use self::entry::{ONLINE_SET, REGION_KEY}; +pub static REGION_ID: Lazy = Lazy::new(|| { + std::env::var("REGION_ID") + .unwrap_or_else(|_| "0".to_string()) + .parse() + .unwrap() +}); + +pub static REGION_KEY: Lazy = Lazy::new(|| format!("region{}", &*REGION_ID)); +pub static ONLINE_SET: &str = "online"; + +pub static FLAG_BITS: u32 = 0b1; /// Create a new presence session, returns the ID of this session -pub async fn presence_create_session(user_id: &str, flags: u8) -> (bool, u32) { +pub async fn create_session(user_id: &str, flags: u8) -> (bool, u32) { info!("Creating a presence session for {user_id} with flags {flags}"); if let Ok(mut conn) = get_connection().await { @@ -24,7 +35,7 @@ pub async fn presence_create_session(user_id: &str, flags: u8) -> (bool, u32) { // A session ID is comprised of random data and any flags ORed to the end let session_id = { let mut rng = rand::thread_rng(); - (rng.gen::() ^ 1) | (flags as u32 & 1) + (rng.gen::() & !FLAG_BITS) | (flags as u32 & FLAG_BITS) }; // Add session to user's sessions and to the region @@ -41,16 +52,12 @@ pub async fn presence_create_session(user_id: &str, flags: u8) -> (bool, u32) { } /// Delete existing presence session -pub async fn presence_delete_session(user_id: &str, session_id: u32) -> bool { - presence_delete_session_internal(user_id, session_id, false).await +pub async fn delete_session(user_id: &str, session_id: u32) -> bool { + delete_session_internal(user_id, session_id, false).await } /// Delete existing presence session (but also choose whether to skip region) -async fn presence_delete_session_internal( - user_id: &str, - session_id: u32, - skip_region: bool, -) -> bool { +async fn delete_session_internal(user_id: &str, session_id: u32, skip_region: bool) -> bool { info!("Deleting presence session for {user_id} with id {session_id}"); if let Ok(mut conn) = get_connection().await { @@ -78,7 +85,7 @@ async fn presence_delete_session_internal( } /// Check whether a given user ID is online -pub async fn presence_is_online(user_id: &str) -> bool { +pub async fn is_online(user_id: &str) -> bool { if let Ok(mut conn) = get_connection().await { conn.exists(user_id).await.unwrap_or(false) } else { @@ -87,7 +94,7 @@ pub async fn presence_is_online(user_id: &str) -> bool { } /// Check whether a set of users is online, returns a set of the online user IDs -pub async fn presence_filter_online(user_ids: &'_ [String]) -> HashSet { +pub async fn filter_online(user_ids: &'_ [String]) -> HashSet { // Ignore empty list immediately, to save time. let mut set = HashSet::new(); if user_ids.is_empty() { @@ -102,7 +109,7 @@ pub async fn presence_filter_online(user_ids: &'_ [String]) -> HashSet { // as for some reason or another, Redis does not like us sending // a list of just one ID to the server. if user_ids.len() == 1 { - if presence_is_online(&user_ids[0]).await { + if is_online(&user_ids[0]).await { set.insert(user_ids[0].to_string()); } @@ -133,7 +140,7 @@ pub async fn presence_filter_online(user_ids: &'_ [String]) -> HashSet { } /// Reset any stale presence data -pub async fn presence_clear_region(region_id: Option<&str>) { +pub async fn clear_region(region_id: Option<&str>) { let region_id = region_id.unwrap_or(&*REGION_KEY); let mut conn = get_connection().await.expect("Redis connection"); @@ -150,7 +157,7 @@ pub async fn presence_clear_region(region_id: Option<&str>) { let parts = session.split(':').collect::>(); if let (Some(user_id), Some(session_id)) = (parts.first(), parts.get(1)) { if let Ok(session_id) = session_id.parse() { - presence_delete_session_internal(user_id, session_id, true).await; + delete_session_internal(user_id, session_id, true).await; } } } @@ -161,3 +168,59 @@ pub async fn presence_clear_region(region_id: Option<&str>) { info!("Clean up complete."); } } + +#[cfg(test)] +mod tests { + use crate::{clear_region, create_session, delete_session, filter_online, is_online}; + use rand::Rng; + + #[async_std::test] + async fn it_works() { + // Clear the region before we start the tests: + clear_region(None).await; + + // Generate some data we'll use: + let user_id = rand::thread_rng().gen::().to_string(); + let other_id = rand::thread_rng().gen::().to_string(); + let flags = 1; + + // Create a session + let (first_session, session_id) = create_session(&user_id, flags).await; + assert!(first_session); + assert_ne!(session_id, 0); + assert_eq!(session_id as u8 & flags, flags); + + // Check if the user is online + assert!(is_online(&user_id).await); + + let user_ids = filter_online(&[user_id.to_string()]).await; + assert_eq!(user_ids.len(), 1); + assert!(user_ids.contains(&user_id)); + + // Create a few more sessions + let (first_session, second_session_id) = create_session(&user_id, 0).await; + assert!(!first_session); + dbg!(second_session_id); + assert_eq!(second_session_id as u8 & 1, 0); + + let (first_session, other_session_id) = create_session(&other_id, 0).await; + assert!(first_session); + + let user_ids = filter_online(&[user_id.to_string(), other_id.to_string()]).await; + assert_eq!(user_ids.len(), 2); + assert!(user_ids.contains(&user_id)); + assert!(user_ids.contains(&other_id)); + + // Remove sessions + delete_session(&user_id, session_id).await; + delete_session(&other_id, other_session_id).await; + assert!(!is_online(&other_id).await); + + // Check if we can wipe everything too + clear_region(None).await; + assert!(!is_online(&user_id).await); + + let user_ids = filter_online(&[user_id.to_string(), other_id.to_string()]).await; + assert!(user_ids.is_empty()) + } +} diff --git a/crates/quark/src/presence/operations.rs b/crates/core/presence/src/operations.rs similarity index 100% rename from crates/quark/src/presence/operations.rs rename to crates/core/presence/src/operations.rs diff --git a/crates/quark/Cargo.toml b/crates/quark/Cargo.toml index ef686897..06f14a96 100644 --- a/crates/quark/Cargo.toml +++ b/crates/quark/Cargo.toml @@ -92,3 +92,4 @@ sentry = "0.25.0" # Core revolt-result = { path = "../core/result", features = [ "serde", "schemas" ] } +revolt-presence = { path = "../core/presence" } diff --git a/crates/quark/examples/test.rs b/crates/quark/examples/test.rs deleted file mode 100644 index d09220c3..00000000 --- a/crates/quark/examples/test.rs +++ /dev/null @@ -1,34 +0,0 @@ -use revolt_quark::models::user::PartialUser; -use revolt_quark::presence::{ - presence_create_session, presence_delete_session, presence_filter_online, presence_is_online, -}; -use revolt_quark::*; - -#[async_std::main] -async fn main() { - let db = DatabaseInfo::Dummy.connect().await.unwrap(); - - let sus = PartialUser { - username: Some("neat".into()), - ..Default::default() - }; - - db.update_user("user id", &sus, vec![]).await.unwrap(); - - dbg!(presence_create_session("entry", 0).await); - dbg!(presence_is_online("entry").await); - dbg!(presence_filter_online(&["a".into(), "b".into(), "entry".into()]).await); - - dbg!(presence_delete_session("entry", 0).await); - dbg!(presence_is_online("entry").await); - - dbg!(presence_create_session("entry", 0).await); - dbg!(presence_create_session("entry", 0).await); - dbg!(presence_delete_session("entry", 0).await); - dbg!(presence_is_online("entry").await); - dbg!(presence_delete_session("entry", 1).await); - dbg!(presence_is_online("entry").await); - - // __set_key("dietz", vec![0xFF]).await; - // dbg!(presence_filter_online(&["dietz".into(), "nuts".into()]).await); -} diff --git a/crates/quark/src/events/impl.rs b/crates/quark/src/events/impl.rs index ffac793e..0ea6e05b 100644 --- a/crates/quark/src/events/impl.rs +++ b/crates/quark/src/events/impl.rs @@ -7,11 +7,11 @@ use crate::{ user::{PartialUser, Presence, RelationshipStatus}, Channel, Member, User, }, - perms, - presence::presence_filter_online, - Database, Permission, Result, + perms, Database, Permission, Result, }; +use revolt_presence::filter_online; + use super::{ client::EventV1, state::{Cache, State}, @@ -135,8 +135,7 @@ impl State { } // Fetch presence data for known users. - let online_ids = - presence_filter_online(&user_ids.iter().cloned().collect::>()).await; + let online_ids = filter_online(&user_ids.iter().cloned().collect::>()).await; user.online = Some(true); // Fetch user data. diff --git a/crates/quark/src/impl/generic/channels/message.rs b/crates/quark/src/impl/generic/channels/message.rs index aaee8e2d..308285a1 100644 --- a/crates/quark/src/impl/generic/channels/message.rs +++ b/crates/quark/src/impl/generic/channels/message.rs @@ -1,5 +1,6 @@ use std::collections::HashSet; +use revolt_presence::filter_online; use serde_json::json; use ulid::Ulid; use validator::Validate; @@ -14,7 +15,6 @@ use crate::{ Channel, Emoji, Message, User, }, permissions::PermissionCalculator, - presence::presence_filter_online, tasks::ack::AckEvent, types::{ january::{Embed, Text}, @@ -79,7 +79,7 @@ impl Message { Channel::DirectMessage { recipients, .. } | Channel::Group { recipients, .. } => { target_ids = (&recipients.iter().cloned().collect::>() - - &presence_filter_online(recipients).await) + - &filter_online(recipients).await) .into_iter() .collect::>(); } diff --git a/crates/quark/src/impl/generic/users/user.rs b/crates/quark/src/impl/generic/users/user.rs index 76d2603b..2707723b 100644 --- a/crates/quark/src/impl/generic/users/user.rs +++ b/crates/quark/src/impl/generic/users/user.rs @@ -4,11 +4,11 @@ use crate::models::user::{ }; use crate::permissions::defn::UserPerms; use crate::permissions::r#impl::user::get_relationship; -use crate::presence::presence_filter_online; use crate::{perms, Database, Error, Result}; use futures::try_join; use impl_ops::impl_op_ex_commutative; +use revolt_presence::filter_online; use std::ops; impl_op_ex_commutative!(+ |a: &i32, b: &Badges| -> i32 { *a | *b as i32 }); @@ -98,7 +98,7 @@ impl User { /// Fetch foreign users by a list of IDs pub async fn fetch_foreign_users(db: &Database, user_ids: &[String]) -> Result> { - let online_ids = presence_filter_online(user_ids).await; + let online_ids = filter_online(user_ids).await; Ok(db .fetch_users(user_ids) diff --git a/crates/quark/src/lib.rs b/crates/quark/src/lib.rs index f57d9127..b95c6583 100644 --- a/crates/quark/src/lib.rs +++ b/crates/quark/src/lib.rs @@ -22,7 +22,6 @@ pub use redis_kiss; pub mod events; pub mod r#impl; pub mod models; -pub mod presence; pub mod tasks; pub mod types; pub mod util; diff --git a/crates/quark/src/presence/entry.rs b/crates/quark/src/presence/entry.rs deleted file mode 100644 index c177a5ab..00000000 --- a/crates/quark/src/presence/entry.rs +++ /dev/null @@ -1,13 +0,0 @@ -use std::env; - -use once_cell::sync::Lazy; - -pub static REGION_ID: Lazy = Lazy::new(|| { - env::var("REGION_ID") - .unwrap_or_else(|_| "0".to_string()) - .parse() - .unwrap() -}); - -pub static REGION_KEY: Lazy = Lazy::new(|| format!("region{}", &*REGION_ID)); -pub static ONLINE_SET: &str = "online"; diff --git a/crates/quark/src/util/ref.rs b/crates/quark/src/util/ref.rs index 4d59bc06..1b3ff148 100644 --- a/crates/quark/src/util/ref.rs +++ b/crates/quark/src/util/ref.rs @@ -1,4 +1,5 @@ use futures::future::join; +use revolt_presence::is_online; use rocket::request::FromParam; use schemars::schema::{InstanceType, Schema, SchemaObject, SingleOrVec}; use schemars::JsonSchema; @@ -7,7 +8,6 @@ use serde::{Deserialize, Serialize}; use crate::models::{ Bot, Channel, Emoji, Invite, Member, Message, Report, Server, ServerBan, User, }; -use crate::presence::presence_is_online; use crate::{Database, Error, Result}; /// Reference to some object in the database @@ -25,7 +25,7 @@ impl Ref { /// Fetch user from Ref pub async fn as_user(&self, db: &Database) -> Result { - let (user, online) = join(db.fetch_user(&self.id), presence_is_online(&self.id)).await; + let (user, online) = join(db.fetch_user(&self.id), is_online(&self.id)).await; let mut user = user?; user.online = Some(online); Ok(user) From 8bfb48dff3cdac66fcdeb9866ab3734ab30307ac Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 20:02:23 +0100 Subject: [PATCH 37/56] test(core/database): try migrating the database --- .../database/src/models/admin_migrations/model.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/core/database/src/models/admin_migrations/model.rs b/crates/core/database/src/models/admin_migrations/model.rs index 8f6dcbbc..70bbc62c 100644 --- a/crates/core/database/src/models/admin_migrations/model.rs +++ b/crates/core/database/src/models/admin_migrations/model.rs @@ -8,3 +8,17 @@ auto_derived!( pub revision: i32, } ); + +#[cfg(test)] +mod tests { + #[async_std::test] + async fn migrate() { + database_test!(|db| async move { + // Initialise the database + db.migrate_database().await.unwrap(); + + // Migrate the existing database + db.migrate_database().await.unwrap() + }); + } +} From dbb66edd9fa96727c59698155dfc42ba342405a5 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 20:07:56 +0100 Subject: [PATCH 38/56] chore(core/database): delete useless example binary --- crates/core/database/examples/migrate.rs | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 crates/core/database/examples/migrate.rs diff --git a/crates/core/database/examples/migrate.rs b/crates/core/database/examples/migrate.rs deleted file mode 100644 index f276d773..00000000 --- a/crates/core/database/examples/migrate.rs +++ /dev/null @@ -1,7 +0,0 @@ -use revolt_database::DatabaseInfo; - -#[async_std::main] -async fn main() { - let db = DatabaseInfo::Auto.connect().await.unwrap(); - db.migrate_database().await.unwrap(); -} From dd3d7e9c491a394ef3423671dbe1cb5519d2d48d Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 20:15:10 +0100 Subject: [PATCH 39/56] fix(core/database): use configured database name for migrations --- crates/core/database/src/models/admin_migrations/ops/mongodb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core/database/src/models/admin_migrations/ops/mongodb.rs b/crates/core/database/src/models/admin_migrations/ops/mongodb.rs index 784f59fa..cdc684e9 100644 --- a/crates/core/database/src/models/admin_migrations/ops/mongodb.rs +++ b/crates/core/database/src/models/admin_migrations/ops/mongodb.rs @@ -22,7 +22,7 @@ impl AbstractMigrations for MongoDb { .await .expect("Failed to fetch database names."); - if list.iter().any(|x| x == "revolt") { + if list.iter().any(|x| x == &self.1) { scripts::migrate_database(self).await; } else { init::create_database(self).await; From 63f56aec0c494f4ed2b02c6b6bbd4b51baf04300 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 20:30:02 +0100 Subject: [PATCH 40/56] ci: re-order services and testing --- .github/workflows/rust.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index ca9d9d56..d41ff722 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -26,15 +26,15 @@ jobs: with: command: build + - name: Run services in background + run: | + docker-compose -f docker-compose.db.yml up -d + - name: Run cargo test uses: actions-rs/cargo@v1 with: command: test - - name: Run services in background - run: | - docker-compose -f docker-compose.db.yml up -d - - name: Run cargo test (with MongoDB) uses: actions-rs/cargo@v1 env: From 22bfd720b5362105372e48b989eaec8948c5fe30 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 21:09:51 +0100 Subject: [PATCH 41/56] merge: remote-tracking branch 'origin/master' into refactor/split-project-into-core-crates --- Cargo.lock | 2 +- Cargo.toml | 2 +- crates/core/presence/src/lib.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1b7cd1d..5129b4cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2634,7 +2634,7 @@ dependencies = [ [[package]] name = "redis" version = "0.22.3" -source = "git+https://github.com/insertish/redis-rs?rev=6575a6b1c09eb8c9cc7f0082d95fe6b8f903c4d7#6575a6b1c09eb8c9cc7f0082d95fe6b8f903c4d7" +source = "git+https://github.com/insertish/redis-rs?rev=1a41faf356fd21aebba71cea7eb7eb2653e5f0ef#1a41faf356fd21aebba71cea7eb7eb2653e5f0ef" dependencies = [ "async-std", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 89be2f2a..a5a13117 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,4 +3,4 @@ members = ["crates/delta", "crates/bonfire", "crates/quark", "crates/core/*"] [patch.crates-io] # mobc-redis = { git = "https://github.com/insertish/mobc", rev = "8b880bb59f2ba80b4c7bc40c649c113d8857a186" } -redis = { git = "https://github.com/insertish/redis-rs", rev = "6575a6b1c09eb8c9cc7f0082d95fe6b8f903c4d7" } +redis = { git = "https://github.com/insertish/redis-rs", rev = "1a41faf356fd21aebba71cea7eb7eb2653e5f0ef" } diff --git a/crates/core/presence/src/lib.rs b/crates/core/presence/src/lib.rs index 67c0722e..72418734 100644 --- a/crates/core/presence/src/lib.rs +++ b/crates/core/presence/src/lib.rs @@ -121,7 +121,7 @@ pub async fn filter_online(user_ids: &'_ [String]) -> HashSet { // Ok so, if this breaks, that means we've lost the Redis patch which adds SMISMEMBER // Currently it's patched in through a forked repository, investigate what happen to it let data: Vec = conn - .sismember("online", user_ids) + .smismember("online", user_ids) .await .expect("this shouldn't happen, please read this code! presence/mod.rs"); if data.is_empty() { From 9124c9f1e3d0fa23c8389ac8e887783c9418281d Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 22 Apr 2023 23:42:17 +0100 Subject: [PATCH 42/56] ci: include new core crates in build --- scripts/build-image-layer.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/build-image-layer.sh b/scripts/build-image-layer.sh index 69fca820..b8a600e6 100644 --- a/scripts/build-image-layer.sh +++ b/scripts/build-image-layer.sh @@ -27,7 +27,11 @@ deps() { tee crates/bonfire/src/main.rs | tee crates/delta/src/main.rs echo '' | - tee crates/quark/src/lib.rs + tee crates/quark/src/lib.rs | + tee crates/core/database/src/lib.rs | + tee crates/core/models/src/lib.rs | + tee crates/core/presence/src/lib.rs | + tee crates/core/result/src/lib.rs cargo build --locked --release --target "${BUILD_TARGET}" } @@ -35,7 +39,11 @@ apps() { touch -am \ crates/bonfire/src/main.rs \ crates/delta/src/main.rs \ - crates/quark/src/lib.rs + crates/quark/src/lib.rs \ + crates/core/database/src/lib.rs \ + crates/core/models/src/lib.rs \ + crates/core/presence/src/lib.rs \ + crates/core/result/src/lib.rs cargo build --locked --release --target "${BUILD_TARGET}" mv target _target && mv _target/"${BUILD_TARGET}" target } From bf71b45fbb225be3eccaff2397221148a56116fe Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 08:51:43 +0100 Subject: [PATCH 43/56] ci: copy core crates into Dockerfile --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 748757cb..f86f3e3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,10 @@ COPY Cargo.toml Cargo.lock ./ COPY crates/bonfire/Cargo.toml ./crates/bonfire/ COPY crates/delta/Cargo.toml ./crates/delta/ COPY crates/quark/Cargo.toml ./crates/quark/ +COPY crates/core/database/Cargo.toml ./crates/core/database/ +COPY crates/core/models/Cargo.toml ./crates/core/models/ +COPY crates/core/presence/Cargo.toml ./crates/core/presence/ +COPY crates/core/result/Cargo.toml ./crates/core/result/ RUN sh /tmp/build-image-layer.sh deps # Build all apps From 8a695b4bb50f6dfb233177089192ca2f43975c9e Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 09:03:37 +0100 Subject: [PATCH 44/56] ci: actually make the source folders for building deps --- scripts/build-image-layer.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/build-image-layer.sh b/scripts/build-image-layer.sh index b8a600e6..82dab680 100644 --- a/scripts/build-image-layer.sh +++ b/scripts/build-image-layer.sh @@ -22,7 +22,11 @@ deps() { mkdir -p \ crates/bonfire/src \ crates/delta/src \ - crates/quark/src + crates/quark/src \ + crates/core/database/src \ + crates/core/models/src \ + crates/core/presence/src \ + crates/core/result/src echo 'fn main() { panic!("stub"); }' | tee crates/bonfire/src/main.rs | tee crates/delta/src/main.rs From 1933c9ea3d4530b4cb04c7393852163364731fe8 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 14:59:44 +0100 Subject: [PATCH 45/56] feat(core/database): add Reference struct --- Cargo.lock | 2 + crates/core/database/Cargo.toml | 5 +++ crates/core/database/src/lib.rs | 1 + crates/core/database/src/util/mod.rs | 1 + crates/core/database/src/util/reference.rs | 52 ++++++++++++++++++++++ crates/delta/Cargo.toml | 2 +- crates/delta/src/routes/bots/fetch.rs | 10 ++--- 7 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 crates/core/database/src/util/mod.rs create mode 100644 crates/core/database/src/util/reference.rs diff --git a/Cargo.lock b/Cargo.lock index 5129b4cb..bc2cba4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2828,6 +2828,8 @@ dependencies = [ "nanoid", "revolt-result", "revolt_optional_struct", + "rocket", + "schemars", "serde", "serde_json", ] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 336c01cf..ec998c16 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -14,6 +14,7 @@ mongodb = [ "dep:mongodb" ] # ... Other async-std-runtime = [ "async-std" ] +rocket-impl = [ "rocket", "schemars" ] # Default Features default = [ "mongodb", "async-std-runtime" ] @@ -42,5 +43,9 @@ async-recursion = "1.0.4" # Async async-std = { version = "1.8.0", features = ["attributes"], optional = true } +# Rocket Impl +schemars = { version = "0.8.8", optional = true } +rocket = { version = "0.5.0-rc.2", default-features = false, features = ["json"], optional = true } + # Authifier authifier = { version = "1.0" } diff --git a/crates/core/database/src/lib.rs b/crates/core/database/src/lib.rs index 4796b684..5a5c999b 100644 --- a/crates/core/database/src/lib.rs +++ b/crates/core/database/src/lib.rs @@ -73,6 +73,7 @@ macro_rules! database_test { } mod models; +pub mod util; pub use models::*; /// Utility function to check if a boolean value is false diff --git a/crates/core/database/src/util/mod.rs b/crates/core/database/src/util/mod.rs new file mode 100644 index 00000000..7b6d3fe1 --- /dev/null +++ b/crates/core/database/src/util/mod.rs @@ -0,0 +1 @@ +pub mod reference; diff --git a/crates/core/database/src/util/reference.rs b/crates/core/database/src/util/reference.rs new file mode 100644 index 00000000..5a6149a3 --- /dev/null +++ b/crates/core/database/src/util/reference.rs @@ -0,0 +1,52 @@ +use revolt_result::Result; +#[cfg(feature = "rocket-impl")] +use rocket::request::FromParam; +#[cfg(feature = "rocket-impl")] +use schemars::{ + schema::{InstanceType, Schema, SchemaObject, SingleOrVec}, + JsonSchema, +}; + +use crate::{Bot, Database}; + +/// Reference to some object in the database +#[derive(Serialize, Deserialize)] +pub struct Reference { + /// Id of object + pub id: String, +} + +impl Reference { + /// Create a Ref from an unchecked string + pub fn from_unchecked(id: String) -> Reference { + Reference { id } + } + + /// Fetch bot from Ref + pub async fn as_bot(&self, db: &Database) -> Result { + db.fetch_bot(&self.id).await + } +} + +#[cfg(feature = "rocket-impl")] +impl<'r> FromParam<'r> for Reference { + type Error = &'r str; + + fn from_param(param: &'r str) -> Result { + Ok(Reference::from_unchecked(param.into())) + } +} + +#[cfg(feature = "rocket-impl")] +impl JsonSchema for Reference { + fn schema_name() -> String { + "Id".to_string() + } + + fn json_schema(_gen: &mut schemars::gen::SchemaGenerator) -> Schema { + Schema::Object(SchemaObject { + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::String))), + ..Default::default() + }) + } +} diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index b4c7cde6..c85f4a70 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -56,7 +56,7 @@ revolt_rocket_okapi = { version = "0.9.1", features = [ "swagger" ] } revolt-quark = { path = "../quark" } # core -revolt-database = { path = "../core/database" } +revolt-database = { path = "../core/database", features = [ "rocket-impl" ] } revolt-models = { path = "../core/models", features = [ "schemas" ] } [build-dependencies] diff --git a/crates/delta/src/routes/bots/fetch.rs b/crates/delta/src/routes/bots/fetch.rs index 19a16ee0..ef4cb214 100644 --- a/crates/delta/src/routes/bots/fetch.rs +++ b/crates/delta/src/routes/bots/fetch.rs @@ -1,6 +1,6 @@ -use revolt_database::Database; +use revolt_database::{util::reference::Reference, Database}; use revolt_models::Bot; -use revolt_quark::{models::User, Db, Error, Ref, Result}; +use revolt_quark::{models::User, Db, Error, Result}; use rocket::{serde::json::Json, State}; use serde::Serialize; @@ -18,18 +18,18 @@ pub struct BotResponse { /// /// Fetch details of a bot you own by its id. #[openapi(tag = "Bots")] -#[get("/")] +#[get("/")] pub async fn fetch_bot( legacy_db: &Db, db: &State, user: User, - target: Ref, + bot: Reference, ) -> Result> { if user.bot.is_some() { return Err(Error::IsBot); } - let bot = db.fetch_bot(&target.id).await.map_err(Error::from_core)?; + let bot = bot.as_bot(db).await.map_err(Error::from_core)?; if bot.owner != user.id { return Err(Error::NotFound); } From e6d0d44c5a7e898f6df4fa5cd4d9a09ba66fafce Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 17:44:23 +0100 Subject: [PATCH 46/56] feat(core/database): basic implementation of User and File models --- Cargo.lock | 2 + crates/core/database/Cargo.toml | 3 +- crates/core/database/src/drivers/reference.rs | 3 +- crates/core/database/src/lib.rs | 4 + crates/core/database/src/models/files/mod.rs | 3 + .../core/database/src/models/files/model.rs | 59 +++++++++++ crates/core/database/src/models/mod.rs | 6 +- crates/core/database/src/models/users/mod.rs | 8 ++ .../core/database/src/models/users/model.rs | 99 +++++++++++++++++++ crates/core/database/src/models/users/ops.rs | 12 +++ .../database/src/models/users/ops/mongodb.rs | 16 +++ .../src/models/users/ops/reference.rs | 18 ++++ .../core/database/src/models/users/rocket.rs | 44 +++++++++ 13 files changed, 274 insertions(+), 3 deletions(-) create mode 100644 crates/core/database/src/models/files/mod.rs create mode 100644 crates/core/database/src/models/files/model.rs create mode 100644 crates/core/database/src/models/users/mod.rs create mode 100644 crates/core/database/src/models/users/model.rs create mode 100644 crates/core/database/src/models/users/ops.rs create mode 100644 crates/core/database/src/models/users/ops/mongodb.rs create mode 100644 crates/core/database/src/models/users/ops/reference.rs create mode 100644 crates/core/database/src/models/users/rocket.rs diff --git a/Cargo.lock b/Cargo.lock index bc2cba4c..558f63b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2822,6 +2822,7 @@ dependencies = [ "async-std", "async-trait", "authifier", + "bson", "futures", "log", "mongodb", @@ -2877,6 +2878,7 @@ name = "revolt-models" version = "0.0.1" dependencies = [ "revolt-database", + "revolt-presence", "schemars", "serde", ] diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index ec998c16..104fa271 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -10,7 +10,7 @@ description = "Revolt Backend: Database Implementation" [features] # Databases -mongodb = [ "dep:mongodb" ] +mongodb = [ "dep:mongodb", "bson" ] # ... Other async-std-runtime = [ "async-std" ] @@ -33,6 +33,7 @@ revolt_optional_struct = "0.2.0" serde = { version = "1", features = ["derive"] } # Database +bson = { optional = true, version = "2.1.0" } mongodb = { optional = true, version = "2.1.0", default-features = false } # Async Language Features diff --git a/crates/core/database/src/drivers/reference.rs b/crates/core/database/src/drivers/reference.rs index e14ff6f3..cd4bc1a1 100644 --- a/crates/core/database/src/drivers/reference.rs +++ b/crates/core/database/src/drivers/reference.rs @@ -2,12 +2,13 @@ use std::{collections::HashMap, sync::Arc}; use futures::lock::Mutex; -use crate::Bot; +use crate::{Bot, User}; database_derived!( /// Reference implementation #[derive(Default)] pub struct ReferenceDb { pub bots: Arc>>, + pub users: Arc>>, } ); diff --git a/crates/core/database/src/lib.rs b/crates/core/database/src/lib.rs index 5a5c999b..18d5db63 100644 --- a/crates/core/database/src/lib.rs +++ b/crates/core/database/src/lib.rs @@ -19,6 +19,10 @@ extern crate revolt_result; #[cfg(feature = "mongodb")] pub use mongodb; +#[cfg(feature = "mongodb")] +#[macro_use] +extern crate bson; + macro_rules! database_derived { ( $( $item:item )+ ) => { $( diff --git a/crates/core/database/src/models/files/mod.rs b/crates/core/database/src/models/files/mod.rs new file mode 100644 index 00000000..4a7ebf60 --- /dev/null +++ b/crates/core/database/src/models/files/mod.rs @@ -0,0 +1,3 @@ +mod model; + +pub use model::*; diff --git a/crates/core/database/src/models/files/model.rs b/crates/core/database/src/models/files/model.rs new file mode 100644 index 00000000..d39e1eb3 --- /dev/null +++ b/crates/core/database/src/models/files/model.rs @@ -0,0 +1,59 @@ +auto_derived_partial!( + /// File + pub struct File { + /// Unique Id + #[serde(rename = "_id")] + pub id: String, + /// Tag / bucket this file was uploaded to + pub tag: String, + /// Original filename + pub filename: String, + /// Parsed metadata of this file + pub metadata: Metadata, + /// Raw content type of this file + pub content_type: String, + /// Size of this file (in bytes) + pub size: isize, + + /// Whether this file was deleted + #[serde(skip_serializing_if = "Option::is_none")] + pub deleted: Option, + /// Whether this file was reported + #[serde(skip_serializing_if = "Option::is_none")] + pub reported: Option, + + // TODO: migrate this mess to having: + // - author_id + // - parent: Parent { Message(id), User(id), etc } + #[serde(skip_serializing_if = "Option::is_none")] + pub message_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub user_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_id: Option, + + /// Id of the object this file is associated with + #[serde(skip_serializing_if = "Option::is_none")] + pub object_id: Option, + }, + "PartialFile" +); + +auto_derived!( + /// Metadata associated with a file + #[serde(tag = "type")] + #[derive(Default)] + pub enum Metadata { + /// File is just a generic uncategorised file + #[default] + File, + /// File contains textual data and should be displayed as such + Text, + /// File is an image with specific dimensions + Image { width: isize, height: isize }, + /// File is a video with specific dimensions + Video { width: isize, height: isize }, + /// File is audio + Audio, + } +); diff --git a/crates/core/database/src/models/mod.rs b/crates/core/database/src/models/mod.rs index d6b2a21c..9fb3ad3d 100644 --- a/crates/core/database/src/models/mod.rs +++ b/crates/core/database/src/models/mod.rs @@ -1,13 +1,17 @@ mod admin_migrations; mod bots; +mod files; +mod users; pub use admin_migrations::*; pub use bots::*; +pub use files::*; +pub use users::*; use crate::{Database, MongoDb, ReferenceDb}; pub trait AbstractDatabase: - Sync + Send + admin_migrations::AbstractMigrations + bots::AbstractBots + Sync + Send + admin_migrations::AbstractMigrations + bots::AbstractBots + users::AbstractUsers { } diff --git a/crates/core/database/src/models/users/mod.rs b/crates/core/database/src/models/users/mod.rs new file mode 100644 index 00000000..b56592f6 --- /dev/null +++ b/crates/core/database/src/models/users/mod.rs @@ -0,0 +1,8 @@ +mod model; +mod ops; +#[cfg(feature = "rocket-impl")] +mod rocket; + +pub use self::rocket::*; +pub use model::*; +pub use ops::*; diff --git a/crates/core/database/src/models/users/model.rs b/crates/core/database/src/models/users/model.rs new file mode 100644 index 00000000..ebdc2ecb --- /dev/null +++ b/crates/core/database/src/models/users/model.rs @@ -0,0 +1,99 @@ +use crate::File; + +auto_derived_partial!( + /// # User + pub struct User { + /// Unique Id + #[serde(rename = "_id")] + pub id: String, + /// Username + pub username: String, + #[serde(skip_serializing_if = "Option::is_none")] + /// Avatar attachment + pub avatar: Option, + /// Relationships with other users + #[serde(skip_serializing_if = "Option::is_none")] + pub relations: Option>, + + /// Bitfield of user badges + #[serde(skip_serializing_if = "Option::is_none")] + pub badges: Option, + /// User's current status + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + /// User's profile page + #[serde(skip_serializing_if = "Option::is_none")] + pub profile: Option, + + /// Enum of user flags + #[serde(skip_serializing_if = "Option::is_none")] + pub flags: Option, + /// Whether this user is privileged + #[serde(skip_serializing_if = "crate::if_false", default)] + pub privileged: bool, + /// Bot information + #[serde(skip_serializing_if = "Option::is_none")] + pub bot: Option, + }, + "PartialUser" +); + +auto_derived!( + /// User's relationship with another user (or themselves) + pub enum RelationshipStatus { + None, + User, + Friend, + Outgoing, + Incoming, + Blocked, + BlockedOther, + } + + /// Relationship entry indicating current status with other user + pub struct Relationship { + #[serde(rename = "_id")] + pub id: String, + pub status: RelationshipStatus, + } + + /// Presence status + pub enum Presence { + /// User is online + Online, + /// User is not currently available + Idle, + /// User is focusing / will only receive mentions + Focus, + /// User is busy / will not receive any notifications + Busy, + /// User appears to be offline + Invisible, + } + + /// User's active status + pub struct UserStatus { + /// Custom status text + #[serde(skip_serializing_if = "String::is_empty")] + pub text: String, + /// Current presence option + #[serde(skip_serializing_if = "Option::is_none")] + pub presence: Option, + } + + /// User's profile + pub struct UserProfile { + /// Text content on user's profile + #[serde(skip_serializing_if = "String::is_empty")] + pub content: String, + /// Background visible on user's profile + #[serde(skip_serializing_if = "Option::is_none")] + pub background: Option, + } + + /// Bot information for if the user is a bot + pub struct BotInformation { + /// Id of the owner of this bot + pub owner: String, + } +); diff --git a/crates/core/database/src/models/users/ops.rs b/crates/core/database/src/models/users/ops.rs new file mode 100644 index 00000000..ca2662ed --- /dev/null +++ b/crates/core/database/src/models/users/ops.rs @@ -0,0 +1,12 @@ +use revolt_result::Result; + +use crate::User; + +mod mongodb; +mod reference; + +#[async_trait] +pub trait AbstractUsers: Sync + Send { + /// Fetch a user from the database + async fn fetch_user(&self, id: &str) -> Result; +} diff --git a/crates/core/database/src/models/users/ops/mongodb.rs b/crates/core/database/src/models/users/ops/mongodb.rs new file mode 100644 index 00000000..c6e43c98 --- /dev/null +++ b/crates/core/database/src/models/users/ops/mongodb.rs @@ -0,0 +1,16 @@ +use revolt_result::Result; + +use crate::MongoDb; +use crate::User; + +use super::AbstractUsers; + +static COL: &str = "bots"; + +#[async_trait] +impl AbstractUsers for MongoDb { + /// Fetch a user from the database + async fn fetch_user(&self, id: &str) -> Result { + query!(self, find_one_by_id, COL, id)?.ok_or_else(|| create_error!(NotFound)) + } +} diff --git a/crates/core/database/src/models/users/ops/reference.rs b/crates/core/database/src/models/users/ops/reference.rs new file mode 100644 index 00000000..034f4867 --- /dev/null +++ b/crates/core/database/src/models/users/ops/reference.rs @@ -0,0 +1,18 @@ +use revolt_result::Result; + +use crate::ReferenceDb; +use crate::User; + +use super::AbstractUsers; + +#[async_trait] +impl AbstractUsers for ReferenceDb { + /// Fetch a user from the database + async fn fetch_user(&self, id: &str) -> Result { + let users = self.users.lock().await; + users + .get(id) + .cloned() + .ok_or_else(|| create_error!(NotFound)) + } +} diff --git a/crates/core/database/src/models/users/rocket.rs b/crates/core/database/src/models/users/rocket.rs new file mode 100644 index 00000000..4cc8cf00 --- /dev/null +++ b/crates/core/database/src/models/users/rocket.rs @@ -0,0 +1,44 @@ +use authifier::models::Session; +use rocket::http::Status; +use rocket::request::{self, FromRequest, Outcome, Request}; + +use crate::{Database, User}; + +#[rocket::async_trait] +impl<'r> FromRequest<'r> for User { + type Error = authifier::Error; + + async fn from_request(request: &'r Request<'_>) -> request::Outcome { + let user: &Option = request + .local_cache_async(async { + let db = request.rocket().state::().expect("`Database`"); + + let _header_bot_token = request + .headers() + .get("x-bot-token") + .next() + .map(|x| x.to_string()); + + /* if let Some(bot_token) = header_bot_token { + if let Ok(user) = User::from_token(db, &bot_token, UserHint::Bot).await { + return Some(user); + } + } else */ + if let Outcome::Success(session) = request.guard::().await { + // This uses a guard so can't really easily be refactored into from_token at this stage. + if let Ok(user) = db.fetch_user(&session.user_id).await { + return Some(user); + } + } + + None + }) + .await; + + if let Some(user) = user { + Outcome::Success(user.clone()) + } else { + Outcome::Failure((Status::Unauthorized, authifier::Error::InvalidSession)) + } + } +} From f8c8407af392fc35bad9ccd6241fb3a04c8e4198 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 17:44:36 +0100 Subject: [PATCH 47/56] feat(core/models): basic implementation of User and File models --- crates/core/database/src/models/bots/model.rs | 7 - .../database/src/models/bots/ops/mongodb.rs | 1 - crates/core/models/Cargo.toml | 3 +- crates/core/models/src/bots.rs | 49 ++-- crates/core/models/src/files.rs | 95 +++++++ crates/core/models/src/lib.rs | 9 + crates/core/models/src/users.rs | 233 ++++++++++++++++++ crates/delta/src/routes/bots/fetch.rs | 26 +- crates/delta/src/routes/bots/fetch_public.rs | 13 +- justfile | 1 + 10 files changed, 387 insertions(+), 50 deletions(-) create mode 100644 crates/core/models/src/files.rs create mode 100644 crates/core/models/src/users.rs diff --git a/crates/core/database/src/models/bots/model.rs b/crates/core/database/src/models/bots/model.rs index a8c551e0..1a9fa17e 100644 --- a/crates/core/database/src/models/bots/model.rs +++ b/crates/core/database/src/models/bots/model.rs @@ -42,13 +42,6 @@ auto_derived_partial!( ); auto_derived!( - /// Flags that may be attributed to a bot - #[repr(i32)] - pub enum BotFlags { - Verified = 1, - Official = 2, - } - /// Optional fields on bot object pub enum FieldsBot { Token, diff --git a/crates/core/database/src/models/bots/ops/mongodb.rs b/crates/core/database/src/models/bots/ops/mongodb.rs index bda3ae02..69da814f 100644 --- a/crates/core/database/src/models/bots/ops/mongodb.rs +++ b/crates/core/database/src/models/bots/ops/mongodb.rs @@ -1,4 +1,3 @@ -use ::mongodb::bson::doc; use revolt_result::Result; use crate::{Bot, FieldsBot, PartialBot}; diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 152363ad..409f2278 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -11,13 +11,14 @@ description = "Revolt Backend: API Models" [features] serde = [ "dep:serde" ] schemas = [ "dep:schemars" ] -from_database = [ "revolt-database" ] +from_database = [ "revolt-database", "revolt-presence" ] default = [ "serde", "from_database" ] [dependencies] # Repo revolt-database = { version = "0.0.1", path = "../database", optional = true } +revolt-presence = { version = "0.0.1", path = "../presence", optional = true } # Serialisation serde = { version = "1", features = ["derive"], optional = true } diff --git a/crates/core/models/src/bots.rs b/crates/core/models/src/bots.rs index 7c201e68..f558087f 100644 --- a/crates/core/models/src/bots.rs +++ b/crates/core/models/src/bots.rs @@ -1,5 +1,7 @@ +use crate::User; + auto_derived!( - /// # Bot + /// Bot pub struct Bot { /// Bot Id #[serde(rename = "_id")] @@ -46,11 +48,21 @@ auto_derived!( pub privacy_policy_url: String, /// Enum of bot flags - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub flags: Option, + #[cfg_attr( + feature = "serde", + serde(skip_serializing_if = "crate::if_zero_u32", default) + )] + pub flags: u32, } - /// # Public Bot + /// Flags that may be attributed to a bot + #[repr(u32)] + pub enum BotFlags { + Verified = 1, + Official = 2, + } + + /// Public Bot pub struct PublicBot { /// Bot Id #[serde(rename = "_id")] @@ -65,21 +77,30 @@ auto_derived!( #[serde(skip_serializing_if = "String::is_empty")] description: String, } + + /// Bot Response + pub struct FetchBotResponse { + /// Bot object + pub bot: Bot, + /// User object + pub user: User, + } ); #[cfg(feature = "from_database")] impl PublicBot { - pub fn from( - bot: revolt_database::Bot, - username: String, - avatar: Option, - description: Option, - ) -> Self { + pub fn from(bot: revolt_database::Bot, user: revolt_database::User) -> Self { + #[cfg(debug_assertions)] + assert_eq!(bot.id, user.id); + PublicBot { id: bot.id, - username, - avatar: avatar.unwrap_or_default(), - description: description.unwrap_or_default(), + username: user.username, + avatar: user.avatar.map(|x| x.id).unwrap_or_default(), + description: user + .profile + .map(|profile| profile.content) + .unwrap_or_default(), } } } @@ -97,7 +118,7 @@ impl From for Bot { interactions_url: value.interactions_url, terms_of_service_url: value.terms_of_service_url, privacy_policy_url: value.privacy_policy_url, - flags: value.flags, + flags: value.flags.unwrap_or_default() as u32, } } } diff --git a/crates/core/models/src/files.rs b/crates/core/models/src/files.rs new file mode 100644 index 00000000..89654b75 --- /dev/null +++ b/crates/core/models/src/files.rs @@ -0,0 +1,95 @@ +auto_derived!( + /// File + pub struct File { + /// Unique Id + #[serde(rename = "_id")] + pub id: String, + /// Tag / bucket this file was uploaded to + pub tag: String, + /// Original filename + pub filename: String, + /// Parsed metadata of this file + pub metadata: Metadata, + /// Raw content type of this file + pub content_type: String, + /// Size of this file (in bytes) + pub size: isize, + + /// Whether this file was deleted + #[serde(skip_serializing_if = "Option::is_none")] + pub deleted: Option, + /// Whether this file was reported + #[serde(skip_serializing_if = "Option::is_none")] + pub reported: Option, + + // TODO: migrate this mess to having: + // - author_id + // - parent: Parent { Message(id), User(id), etc } + #[serde(skip_serializing_if = "Option::is_none")] + pub message_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub user_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_id: Option, + + /// Id of the object this file is associated with + #[serde(skip_serializing_if = "Option::is_none")] + pub object_id: Option, + } + + /// Metadata associated with a file + #[serde(tag = "type")] + #[derive(Default)] + pub enum Metadata { + /// File is just a generic uncategorised file + #[default] + File, + /// File contains textual data and should be displayed as such + Text, + /// File is an image with specific dimensions + Image { width: usize, height: usize }, + /// File is a video with specific dimensions + Video { width: usize, height: usize }, + /// File is audio + Audio, + } +); + +#[cfg(feature = "from_database")] +impl From for File { + fn from(value: revolt_database::File) -> Self { + File { + id: value.id, + tag: value.tag, + filename: value.filename, + metadata: value.metadata.into(), + content_type: value.content_type, + size: value.size, + deleted: value.deleted, + reported: value.reported, + message_id: value.message_id, + user_id: value.user_id, + server_id: value.server_id, + object_id: value.object_id, + } + } +} + +#[cfg(feature = "from_database")] +impl From for Metadata { + fn from(value: revolt_database::Metadata) -> Self { + match value { + revolt_database::Metadata::File => Metadata::File, + revolt_database::Metadata::Text => Metadata::Text, + revolt_database::Metadata::Image { width, height } => Metadata::Image { + width: width as usize, + height: height as usize, + }, + revolt_database::Metadata::Video { width, height } => Metadata::Video { + width: width as usize, + height: height as usize, + }, + revolt_database::Metadata::Audio => Metadata::Audio, + } + } +} diff --git a/crates/core/models/src/lib.rs b/crates/core/models/src/lib.rs index 21f199c8..8d8b50d5 100644 --- a/crates/core/models/src/lib.rs +++ b/crates/core/models/src/lib.rs @@ -18,10 +18,19 @@ macro_rules! auto_derived { } mod bots; +mod files; +mod users; pub use bots::*; +pub use files::*; +pub use users::*; /// Utility function to check if a boolean value is false pub fn if_false(t: &bool) -> bool { !t } + +/// Utility function to check if an u32 is zero +pub fn if_zero_u32(t: &u32) -> bool { + t == &0 +} diff --git a/crates/core/models/src/users.rs b/crates/core/models/src/users.rs new file mode 100644 index 00000000..09d3b0b4 --- /dev/null +++ b/crates/core/models/src/users.rs @@ -0,0 +1,233 @@ +use crate::File; + +auto_derived!( + /// User + pub struct User { + /// Unique Id + #[serde(rename = "_id")] + pub id: String, + /// Username + pub username: String, + #[serde(skip_serializing_if = "Option::is_none")] + /// Avatar attachment + pub avatar: Option, + /// Relationships with other users + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub relations: Vec, + + /// Bitfield of user badges + #[serde(skip_serializing_if = "crate::if_zero_u32", default)] + pub badges: u32, + /// User's current status + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + /// User's profile page + #[serde(skip_serializing_if = "Option::is_none")] + pub profile: Option, + + /// Enum of user flags + #[serde(skip_serializing_if = "crate::if_zero_u32", default)] + pub flags: u32, + /// Whether this user is privileged + #[serde(skip_serializing_if = "crate::if_false", default)] + pub privileged: bool, + /// Bot information + #[serde(skip_serializing_if = "Option::is_none")] + pub bot: Option, + + /// Current session user's relationship with this user + pub relationship: RelationshipStatus, + /// Whether this user is currently online + pub online: bool, + } + + /// User's relationship with another user (or themselves) + #[derive(Default)] + pub enum RelationshipStatus { + #[default] + None, + User, + Friend, + Outgoing, + Incoming, + Blocked, + BlockedOther, + } + + /// Relationship entry indicating current status with other user + pub struct Relationship { + #[serde(rename = "_id")] + pub user_id: String, + pub status: RelationshipStatus, + } + + /// Presence status + pub enum Presence { + /// User is online + Online, + /// User is not currently available + Idle, + /// User is focusing / will only receive mentions + Focus, + /// User is busy / will not receive any notifications + Busy, + /// User appears to be offline + Invisible, + } + + /// User's active status + pub struct UserStatus { + /// Custom status text + #[serde(skip_serializing_if = "String::is_empty")] + pub text: String, + /// Current presence option + #[serde(skip_serializing_if = "Option::is_none")] + pub presence: Option, + } + + /// User's profile + pub struct UserProfile { + /// Text content on user's profile + #[serde(skip_serializing_if = "String::is_empty")] + pub content: String, + /// Background visible on user's profile + #[serde(skip_serializing_if = "Option::is_none")] + pub background: Option, + } + + /// User badge bitfield + #[repr(u32)] + pub enum UserBadges { + /// Revolt Developer + Developer = 1, + /// Helped translate Revolt + Translator = 2, + /// Monetarily supported Revolt + Supporter = 4, + /// Responsibly disclosed a security issue + ResponsibleDisclosure = 8, + /// Revolt Founder + Founder = 16, + /// Platform moderator + PlatformModeration = 32, + /// Active monetary supporter + ActiveSupporter = 64, + /// 🦊🦝 + Paw = 128, + /// Joined as one of the first 1000 users in 2021 + EarlyAdopter = 256, + /// Amogus + ReservedRelevantJokeBadge1 = 512, + /// Low resolution troll face + ReservedRelevantJokeBadge2 = 1024, + } + + /// User flag enum + #[repr(u32)] + pub enum UserFlags { + /// User has been suspended from the platform + Suspended = 1, + /// User has deleted their account + Deleted = 2, + /// User was banned off the platform + Banned = 4, + /// User was marked as spam and removed from platform + Spam = 8, + } + + /// Bot information for if the user is a bot + pub struct BotInformation { + /// Id of the owner of this bot + #[serde(rename = "owner")] + pub owner_id: String, + } +); + +pub trait CheckRelationship { + fn with(&self, user: &str) -> RelationshipStatus; +} + +impl CheckRelationship for Vec { + fn with(&self, user: &str) -> RelationshipStatus { + for entry in self { + if entry.user_id == user { + return entry.status.clone(); + } + } + + RelationshipStatus::None + } +} + +#[cfg(feature = "from_database")] +impl User { + pub async fn from

(user: revolt_database::User, perspective: P) -> Self + where + P: Into>, + { + let relationship = if let Some(perspective) = perspective.into() { + perspective + .relations + .unwrap_or_default() + .into_iter() + .find(|relationship| relationship.id == user.id) + .map(|relationship| relationship.status.into()) + .unwrap_or_default() + } else { + RelationshipStatus::None + }; + + // do permission stuff here + // TODO: implement permissions =) + let can_see_profile = false; + + Self { + username: user.username, + avatar: user.avatar.map(|file| file.into()), + relations: vec![], + badges: user.badges.unwrap_or_default() as u32, + status: None, + profile: None, + flags: user.flags.unwrap_or_default() as u32, + privileged: user.privileged, + bot: user.bot.map(|bot| bot.into()), + relationship, + online: can_see_profile && revolt_presence::is_online(&user.id).await, + id: user.id, + } + } +} + +#[cfg(feature = "from_database")] +impl From for BotInformation { + fn from(value: revolt_database::BotInformation) -> Self { + BotInformation { + owner_id: value.owner, + } + } +} + +#[cfg(feature = "from_database")] +impl From for Relationship { + fn from(value: revolt_database::Relationship) -> Self { + Self { + user_id: value.id, + status: value.status.into(), + } + } +} + +#[cfg(feature = "from_database")] +impl From for RelationshipStatus { + fn from(value: revolt_database::RelationshipStatus) -> Self { + match value { + revolt_database::RelationshipStatus::None => RelationshipStatus::None, + revolt_database::RelationshipStatus::User => RelationshipStatus::User, + revolt_database::RelationshipStatus::Friend => RelationshipStatus::Friend, + revolt_database::RelationshipStatus::Outgoing => RelationshipStatus::Outgoing, + revolt_database::RelationshipStatus::Incoming => RelationshipStatus::Incoming, + revolt_database::RelationshipStatus::Blocked => RelationshipStatus::Blocked, + revolt_database::RelationshipStatus::BlockedOther => RelationshipStatus::BlockedOther, + } + } +} diff --git a/crates/delta/src/routes/bots/fetch.rs b/crates/delta/src/routes/bots/fetch.rs index ef4cb214..41aee968 100644 --- a/crates/delta/src/routes/bots/fetch.rs +++ b/crates/delta/src/routes/bots/fetch.rs @@ -1,18 +1,7 @@ use revolt_database::{util::reference::Reference, Database}; -use revolt_models::Bot; -use revolt_quark::{models::User, Db, Error, Result}; +use revolt_models::FetchBotResponse; +use revolt_quark::{models::User, Error, Result}; use rocket::{serde::json::Json, State}; -use serde::Serialize; - -/// # Bot Response -/// TODO: move to revolt-models -#[derive(Serialize, JsonSchema)] -pub struct BotResponse { - /// Bot object - bot: Bot, - /// User object - user: User, -} /// # Fetch Bot /// @@ -20,11 +9,10 @@ pub struct BotResponse { #[openapi(tag = "Bots")] #[get("/")] pub async fn fetch_bot( - legacy_db: &Db, db: &State, user: User, bot: Reference, -) -> Result> { +) -> Result> { if user.bot.is_some() { return Err(Error::IsBot); } @@ -34,8 +22,12 @@ pub async fn fetch_bot( return Err(Error::NotFound); } - Ok(Json(BotResponse { - user: legacy_db.fetch_user(&bot.id).await?.foreign(), + Ok(Json(FetchBotResponse { + user: revolt_models::User::from( + db.fetch_user(&bot.id).await.map_err(Error::from_core)?, + None, + ) + .await, bot: bot.into(), })) } diff --git a/crates/delta/src/routes/bots/fetch_public.rs b/crates/delta/src/routes/bots/fetch_public.rs index 1473eb65..87472f1c 100644 --- a/crates/delta/src/routes/bots/fetch_public.rs +++ b/crates/delta/src/routes/bots/fetch_public.rs @@ -1,6 +1,6 @@ use revolt_database::Database; use revolt_models::PublicBot; -use revolt_quark::{models::User, Db, Error, Ref, Result}; +use revolt_quark::{models::User, Error, Ref, Result}; use rocket::serde::json::Json; use rocket::State; @@ -11,7 +11,6 @@ use rocket::State; #[openapi(tag = "Bots")] #[get("//invite")] pub async fn fetch_public_bot( - legacy_db: &Db, db: &State, user: Option, target: Ref, @@ -21,12 +20,6 @@ pub async fn fetch_public_bot( return Err(Error::NotFound); } - let user = legacy_db.fetch_user(&bot.id).await?; - - Ok(Json(PublicBot::from( - bot, - user.username, - user.avatar.map(|f| f.id), - user.profile.and_then(|p| p.content), - ))) + let user = db.fetch_user(&bot.id).await.map_err(Error::from_core)?; + Ok(Json(PublicBot::from(bot, user))) } diff --git a/justfile b/justfile index 778aeaa4..2a75ff88 100644 --- a/justfile +++ b/justfile @@ -1,4 +1,5 @@ publish: cargo publish --package revolt-result cargo publish --package revolt-database + cargo publish --package revolt-presence cargo publish --package revolt-models From d5d922d8307cec167d6580aaf12339cbd127b994 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 18:07:23 +0100 Subject: [PATCH 48/56] feat(core/models): rest of the conversions to User model --- crates/core/models/src/users.rs | 59 +++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/crates/core/models/src/users.rs b/crates/core/models/src/users.rs index 09d3b0b4..bf52cafc 100644 --- a/crates/core/models/src/users.rs +++ b/crates/core/models/src/users.rs @@ -199,10 +199,16 @@ impl User { } #[cfg(feature = "from_database")] -impl From for BotInformation { - fn from(value: revolt_database::BotInformation) -> Self { - BotInformation { - owner_id: value.owner, +impl From for RelationshipStatus { + fn from(value: revolt_database::RelationshipStatus) -> Self { + match value { + revolt_database::RelationshipStatus::None => RelationshipStatus::None, + revolt_database::RelationshipStatus::User => RelationshipStatus::User, + revolt_database::RelationshipStatus::Friend => RelationshipStatus::Friend, + revolt_database::RelationshipStatus::Outgoing => RelationshipStatus::Outgoing, + revolt_database::RelationshipStatus::Incoming => RelationshipStatus::Incoming, + revolt_database::RelationshipStatus::Blocked => RelationshipStatus::Blocked, + revolt_database::RelationshipStatus::BlockedOther => RelationshipStatus::BlockedOther, } } } @@ -218,16 +224,43 @@ impl From for Relationship { } #[cfg(feature = "from_database")] -impl From for RelationshipStatus { - fn from(value: revolt_database::RelationshipStatus) -> Self { +impl From for Presence { + fn from(value: revolt_database::Presence) -> Self { match value { - revolt_database::RelationshipStatus::None => RelationshipStatus::None, - revolt_database::RelationshipStatus::User => RelationshipStatus::User, - revolt_database::RelationshipStatus::Friend => RelationshipStatus::Friend, - revolt_database::RelationshipStatus::Outgoing => RelationshipStatus::Outgoing, - revolt_database::RelationshipStatus::Incoming => RelationshipStatus::Incoming, - revolt_database::RelationshipStatus::Blocked => RelationshipStatus::Blocked, - revolt_database::RelationshipStatus::BlockedOther => RelationshipStatus::BlockedOther, + revolt_database::Presence::Online => Presence::Online, + revolt_database::Presence::Idle => Presence::Idle, + revolt_database::Presence::Focus => Presence::Focus, + revolt_database::Presence::Busy => Presence::Busy, + revolt_database::Presence::Invisible => Presence::Invisible, + } + } +} + +#[cfg(feature = "from_database")] +impl From for UserStatus { + fn from(value: revolt_database::UserStatus) -> Self { + UserStatus { + text: value.text, + presence: value.presence.map(|presence| presence.into()), + } + } +} + +#[cfg(feature = "from_database")] +impl From for UserProfile { + fn from(value: revolt_database::UserProfile) -> Self { + UserProfile { + content: value.content, + background: value.background.map(|file| file.into()), + } + } +} + +#[cfg(feature = "from_database")] +impl From for BotInformation { + fn from(value: revolt_database::BotInformation) -> Self { + BotInformation { + owner_id: value.owner, } } } From 39fec310f97000a46b8a1b76b4784a0a206f26c0 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 18:09:16 +0100 Subject: [PATCH 49/56] refactor(core/models): move models into v0 module --- crates/core/models/src/lib.rs | 8 +------- crates/core/models/src/{ => v0}/bots.rs | 2 +- crates/core/models/src/{ => v0}/files.rs | 0 crates/core/models/src/v0/mod.rs | 7 +++++++ crates/core/models/src/{ => v0}/users.rs | 2 +- crates/delta/src/routes/bots/fetch.rs | 4 ++-- crates/delta/src/routes/bots/fetch_public.rs | 2 +- 7 files changed, 13 insertions(+), 12 deletions(-) rename crates/core/models/src/{ => v0}/bots.rs (99%) rename crates/core/models/src/{ => v0}/files.rs (100%) create mode 100644 crates/core/models/src/v0/mod.rs rename crates/core/models/src/{ => v0}/users.rs (99%) diff --git a/crates/core/models/src/lib.rs b/crates/core/models/src/lib.rs index 8d8b50d5..03f6c818 100644 --- a/crates/core/models/src/lib.rs +++ b/crates/core/models/src/lib.rs @@ -17,13 +17,7 @@ macro_rules! auto_derived { }; } -mod bots; -mod files; -mod users; - -pub use bots::*; -pub use files::*; -pub use users::*; +pub mod v0; /// Utility function to check if a boolean value is false pub fn if_false(t: &bool) -> bool { diff --git a/crates/core/models/src/bots.rs b/crates/core/models/src/v0/bots.rs similarity index 99% rename from crates/core/models/src/bots.rs rename to crates/core/models/src/v0/bots.rs index f558087f..43a268b8 100644 --- a/crates/core/models/src/bots.rs +++ b/crates/core/models/src/v0/bots.rs @@ -1,4 +1,4 @@ -use crate::User; +use super::User; auto_derived!( /// Bot diff --git a/crates/core/models/src/files.rs b/crates/core/models/src/v0/files.rs similarity index 100% rename from crates/core/models/src/files.rs rename to crates/core/models/src/v0/files.rs diff --git a/crates/core/models/src/v0/mod.rs b/crates/core/models/src/v0/mod.rs new file mode 100644 index 00000000..927b1c92 --- /dev/null +++ b/crates/core/models/src/v0/mod.rs @@ -0,0 +1,7 @@ +mod bots; +mod files; +mod users; + +pub use bots::*; +pub use files::*; +pub use users::*; diff --git a/crates/core/models/src/users.rs b/crates/core/models/src/v0/users.rs similarity index 99% rename from crates/core/models/src/users.rs rename to crates/core/models/src/v0/users.rs index bf52cafc..2671d28b 100644 --- a/crates/core/models/src/users.rs +++ b/crates/core/models/src/v0/users.rs @@ -1,4 +1,4 @@ -use crate::File; +use super::File; auto_derived!( /// User diff --git a/crates/delta/src/routes/bots/fetch.rs b/crates/delta/src/routes/bots/fetch.rs index 41aee968..7ccac797 100644 --- a/crates/delta/src/routes/bots/fetch.rs +++ b/crates/delta/src/routes/bots/fetch.rs @@ -1,5 +1,5 @@ use revolt_database::{util::reference::Reference, Database}; -use revolt_models::FetchBotResponse; +use revolt_models::v0::FetchBotResponse; use revolt_quark::{models::User, Error, Result}; use rocket::{serde::json::Json, State}; @@ -23,7 +23,7 @@ pub async fn fetch_bot( } Ok(Json(FetchBotResponse { - user: revolt_models::User::from( + user: revolt_models::v0::User::from( db.fetch_user(&bot.id).await.map_err(Error::from_core)?, None, ) diff --git a/crates/delta/src/routes/bots/fetch_public.rs b/crates/delta/src/routes/bots/fetch_public.rs index 87472f1c..796dc43e 100644 --- a/crates/delta/src/routes/bots/fetch_public.rs +++ b/crates/delta/src/routes/bots/fetch_public.rs @@ -1,5 +1,5 @@ use revolt_database::Database; -use revolt_models::PublicBot; +use revolt_models::v0::PublicBot; use revolt_quark::{models::User, Error, Ref, Result}; use rocket::serde::json::Json; From f633fccbca903d10d2d90a0babdec28981db670f Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 22:11:59 +0100 Subject: [PATCH 50/56] feat(core/permissions): initial commit --- crates/core/permissions/Cargo.toml | 33 ++ crates/core/permissions/src/impl.rs | 128 ++++++ crates/core/permissions/src/lib.rs | 16 + crates/core/permissions/src/models/channel.rs | 137 +++++++ crates/core/permissions/src/models/mod.rs | 58 +++ crates/core/permissions/src/models/server.rs | 56 +++ crates/core/permissions/src/models/user.rs | 25 ++ crates/core/permissions/src/test.rs | 377 ++++++++++++++++++ crates/core/permissions/src/trait.rs | 67 ++++ 9 files changed, 897 insertions(+) create mode 100644 crates/core/permissions/Cargo.toml create mode 100644 crates/core/permissions/src/impl.rs create mode 100644 crates/core/permissions/src/lib.rs create mode 100644 crates/core/permissions/src/models/channel.rs create mode 100644 crates/core/permissions/src/models/mod.rs create mode 100644 crates/core/permissions/src/models/server.rs create mode 100644 crates/core/permissions/src/models/user.rs create mode 100644 crates/core/permissions/src/test.rs create mode 100644 crates/core/permissions/src/trait.rs diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml new file mode 100644 index 00000000..5fb3c071 --- /dev/null +++ b/crates/core/permissions/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "revolt-permissions" +version = "0.0.1" +edition = "2021" +license = "AGPL-3.0-or-later" +authors = [ "Paul Makles " ] +description = "Revolt Backend: Permission Logic" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +serde = [ "dep:serde" ] +schemas = [ "dep:schemars" ] +try-from-primitive = [ "dep:num_enum" ] + +[dev-dependencies] +# Async +async-std = { version = "1.8.0", features = ["attributes"] } + +[dependencies] +# Utility +auto_ops = "0.3.0" +once_cell = "1.17" +num_enum = { version = "0.6.1", optional = true } + +# Async +async-trait = "0.1.51" + +# Serialisation +serde = { version = "1", features = ["derive"], optional = true } + +# Spec Generation +schemars = { version = "0.8.8", optional = true } diff --git a/crates/core/permissions/src/impl.rs b/crates/core/permissions/src/impl.rs new file mode 100644 index 00000000..11c378c6 --- /dev/null +++ b/crates/core/permissions/src/impl.rs @@ -0,0 +1,128 @@ +use crate::{ + ChannelPermission, ChannelType, PermissionQuery, PermissionValue, RelationshipStatus, + UserPermission, ALLOW_IN_TIMEOUT, DEFAULT_PERMISSION_DIRECT_MESSAGE, + DEFAULT_PERMISSION_SAVED_MESSAGES, DEFAULT_PERMISSION_VIEW_ONLY, +}; + +/// Calculate permissions against a user +pub async fn calculate_user_permissions(query: &mut P) -> u32 { + if query.are_we_privileged().await { + return u32::MAX; + } + + if query.are_the_users_same().await { + return u32::MAX; + } + + let mut permissions = 0_u32; + match query.user_relationship().await { + RelationshipStatus::Friend => return u32::MAX, + RelationshipStatus::Blocked | RelationshipStatus::BlockedOther => { + return UserPermission::Access as u32 + } + RelationshipStatus::Incoming | RelationshipStatus::Outgoing => { + permissions = UserPermission::Access as u32; + } + _ => {} + } + + if query.have_mutual_connection().await { + permissions = UserPermission::Access + UserPermission::ViewProfile; + + if query.user_is_bot().await || query.are_we_a_bot().await { + permissions += UserPermission::SendMessage as u32; + } + + permissions + } else { + permissions + } +} + +/// Calculate permissions against a server +pub async fn calculate_server_permissions(query: &mut P) -> PermissionValue { + if query.are_we_privileged().await || query.are_we_server_owner().await { + return ChannelPermission::GrantAllSafe.into(); + } + + if !query.are_we_a_member().await { + return 0_u64.into(); + } + + let mut permissions: PermissionValue = query.get_default_server_permissions().await.into(); + + for role_override in query.get_our_server_role_overrides().await { + permissions.apply(role_override); + } + + if query.are_we_timed_out().await { + permissions.restrict(*ALLOW_IN_TIMEOUT); + } + + permissions +} + +/// Calculate permissions against a channel +pub async fn calculate_channel_permissions(query: &mut P) -> PermissionValue { + if query.are_we_privileged().await { + return ChannelPermission::GrantAllSafe.into(); + } + + match query.get_channel_type().await { + ChannelType::SavedMessages => { + if query.do_we_own_the_channel().await { + DEFAULT_PERMISSION_SAVED_MESSAGES.into() + } else { + 0_u64.into() + } + } + ChannelType::DirectMessage => { + if query.are_we_part_of_the_channel().await { + query.set_recipient_as_user().await; + + let permissions = calculate_user_permissions(query).await; + if (permissions & UserPermission::SendMessage as u32) + == UserPermission::SendMessage as u32 + { + (*DEFAULT_PERMISSION_DIRECT_MESSAGE).into() + } else { + (*DEFAULT_PERMISSION_VIEW_ONLY).into() + } + } else { + 0_u64.into() + } + } + ChannelType::Group => { + if query.do_we_own_the_channel().await { + ChannelPermission::GrantAllSafe.into() + } else if query.are_we_part_of_the_channel().await { + (*DEFAULT_PERMISSION_VIEW_ONLY + | query.get_default_channel_permissions().await.allow) + .into() + } else { + 0_u64.into() + } + } + ChannelType::ServerChannel => { + query.set_server_from_channel().await; + + if query.are_we_a_member().await { + let mut permissions = calculate_server_permissions(query).await; + permissions.apply(query.get_default_channel_permissions().await); + + for role_override in query.get_our_channel_role_overrides().await { + permissions.apply(role_override); + } + + if query.are_we_timed_out().await { + permissions.restrict(*ALLOW_IN_TIMEOUT); + } + + permissions + } else { + 0_u64.into() + } + } + ChannelType::Unknown => 0_u64.into(), + } +} diff --git a/crates/core/permissions/src/lib.rs b/crates/core/permissions/src/lib.rs new file mode 100644 index 00000000..734d7be1 --- /dev/null +++ b/crates/core/permissions/src/lib.rs @@ -0,0 +1,16 @@ +#[macro_use] +extern crate auto_ops; + +#[macro_use] +extern crate async_trait; + +mod r#impl; +mod models; +mod r#trait; + +pub use models::*; +pub use r#impl::*; +pub use r#trait::*; + +#[cfg(test)] +mod test; diff --git a/crates/core/permissions/src/models/channel.rs b/crates/core/permissions/src/models/channel.rs new file mode 100644 index 00000000..25dd81db --- /dev/null +++ b/crates/core/permissions/src/models/channel.rs @@ -0,0 +1,137 @@ +use once_cell::sync::Lazy; +use std::ops::Add; + +/// Abstract channel type +pub enum ChannelType { + SavedMessages, + DirectMessage, + Group, + ServerChannel, + Unknown, +} + +/// Permission value on Revolt +/// +/// This should be restricted to the lower 52 bits to prevent any +/// potential issues with Javascript. Also leave empty spaces for +/// future permission flags to be added. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "try-from-primitive", derive(num_enum::TryFromPrimitive))] +#[repr(u64)] +pub enum ChannelPermission { + // * Generic permissions + /// Manage the channel or channels on the server + ManageChannel = 1 << 0, + /// Manage the server + ManageServer = 1 << 1, + /// Manage permissions on servers or channels + ManagePermissions = 1 << 2, + /// Manage roles on server + ManageRole = 1 << 3, + /// Manage server customisation (includes emoji) + ManageCustomisation = 1 << 4, + + // % 1 bit reserved + + // * Member permissions + /// Kick other members below their ranking + KickMembers = 1 << 6, + /// Ban other members below their ranking + BanMembers = 1 << 7, + /// Timeout other members below their ranking + TimeoutMembers = 1 << 8, + /// Assign roles to members below their ranking + AssignRoles = 1 << 9, + /// Change own nickname + ChangeNickname = 1 << 10, + /// Change or remove other's nicknames below their ranking + ManageNicknames = 1 << 11, + /// Change own avatar + ChangeAvatar = 1 << 12, + /// Remove other's avatars below their ranking + RemoveAvatars = 1 << 13, + + // % 7 bits reserved + + // * Channel permissions + /// View a channel + ViewChannel = 1 << 20, + /// Read a channel's past message history + ReadMessageHistory = 1 << 21, + /// Send a message in a channel + SendMessage = 1 << 22, + /// Delete messages in a channel + ManageMessages = 1 << 23, + /// Manage webhook entries on a channel + ManageWebhooks = 1 << 24, + /// Create invites to this channel + InviteOthers = 1 << 25, + /// Send embedded content in this channel + SendEmbeds = 1 << 26, + /// Send attachments and media in this channel + UploadFiles = 1 << 27, + /// Masquerade messages using custom nickname and avatar + Masquerade = 1 << 28, + /// React to messages with emojis + React = 1 << 29, + + // * Voice permissions + /// Connect to a voice channel + Connect = 1 << 30, + /// Speak in a voice call + Speak = 1 << 31, + /// Share video in a voice call + Video = 1 << 32, + /// Mute other members with lower ranking in a voice call + MuteMembers = 1 << 33, + /// Deafen other members with lower ranking in a voice call + DeafenMembers = 1 << 34, + /// Move members between voice channels + MoveMembers = 1 << 35, + + // * Misc. permissions + // % Bits 36 to 52: free area + // % Bits 53 to 64: do not use + + // * Grant all permissions + /// Safely grant all permissions + GrantAllSafe = 0x000F_FFFF_FFFF_FFFF, + + /// Grant all permissions + GrantAll = u64::MAX, +} + +impl_op_ex!(+ |a: &ChannelPermission, b: &ChannelPermission| -> u64 { *a as u64 | *b as u64 }); +impl_op_ex_commutative!(+ |a: &u64, b: &ChannelPermission| -> u64 { *a | *b as u64 }); + +pub static ALLOW_IN_TIMEOUT: Lazy = + Lazy::new(|| ChannelPermission::ViewChannel + ChannelPermission::ReadMessageHistory); + +pub static DEFAULT_PERMISSION_VIEW_ONLY: Lazy = + Lazy::new(|| ChannelPermission::ViewChannel + ChannelPermission::ReadMessageHistory); + +pub static DEFAULT_PERMISSION: Lazy = Lazy::new(|| { + DEFAULT_PERMISSION_VIEW_ONLY.add( + ChannelPermission::SendMessage + + ChannelPermission::InviteOthers + + ChannelPermission::SendEmbeds + + ChannelPermission::UploadFiles + + ChannelPermission::Connect + + ChannelPermission::Speak, + ) +}); + +pub static DEFAULT_PERMISSION_SAVED_MESSAGES: u64 = ChannelPermission::GrantAllSafe as u64; + +pub static DEFAULT_PERMISSION_DIRECT_MESSAGE: Lazy = Lazy::new(|| { + DEFAULT_PERMISSION.add(ChannelPermission::ManageChannel + ChannelPermission::React) +}); + +pub static DEFAULT_PERMISSION_SERVER: Lazy = Lazy::new(|| { + DEFAULT_PERMISSION.add( + ChannelPermission::React + + ChannelPermission::ChangeNickname + + ChannelPermission::ChangeAvatar, + ) +}); diff --git a/crates/core/permissions/src/models/mod.rs b/crates/core/permissions/src/models/mod.rs new file mode 100644 index 00000000..e234eac5 --- /dev/null +++ b/crates/core/permissions/src/models/mod.rs @@ -0,0 +1,58 @@ +mod channel; +mod server; +mod user; + +pub use channel::*; +pub use server::*; +pub use user::*; + +/// Holds a permission value to manipulate. +#[derive(Debug)] +pub struct PermissionValue(u64); + +impl PermissionValue { + /// Apply a given override to this value + pub fn apply(&mut self, v: Override) { + self.allow(v.allow); + self.revoke(v.deny); + } + + /// Allow given permissions + pub fn allow(&mut self, v: u64) { + self.0 |= v; + } + + /// Revoke given permissions + pub fn revoke(&mut self, v: u64) { + self.0 &= !v; + } + + /// Restrict to given permissions + pub fn restrict(&mut self, v: u64) { + self.0 &= v; + } +} + +impl From for PermissionValue { + fn from(v: i64) -> Self { + Self(v as u64) + } +} + +impl From for PermissionValue { + fn from(v: u64) -> Self { + Self(v) + } +} + +impl From for u64 { + fn from(v: PermissionValue) -> Self { + v.0 + } +} + +impl From for PermissionValue { + fn from(v: ChannelPermission) -> Self { + (v as u64).into() + } +} diff --git a/crates/core/permissions/src/models/server.rs b/crates/core/permissions/src/models/server.rs new file mode 100644 index 00000000..08ae8443 --- /dev/null +++ b/crates/core/permissions/src/models/server.rs @@ -0,0 +1,56 @@ +/// Representation of a single permission override +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Override { + /// Allow bit flags + pub allow: u64, + /// Disallow bit flags + pub deny: u64, +} + +/// Representation of a single permission override +/// as it appears on models and in the database +#[derive(/*JsonSchema, */ Debug, Clone, Copy, Default)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct OverrideField { + /// Allow bit flags + a: i64, + /// Disallow bit flags + d: i64, +} + +impl Override { + /// Into allows + pub fn allows(&self) -> u64 { + self.allow + } + + /// Into denies + pub fn denies(&self) -> u64 { + self.deny + } +} + +impl From for OverrideField { + fn from(v: Override) -> Self { + Self { + a: v.allow as i64, + d: v.deny as i64, + } + } +} + +impl From for Override { + fn from(v: OverrideField) -> Self { + Self { + allow: v.a as u64, + deny: v.d as u64, + } + } +} + +/*impl From for Bson { + fn from(v: OverrideField) -> Self { + Self::Document(bson::to_document(&v).unwrap()) + } +}*/ diff --git a/crates/core/permissions/src/models/user.rs b/crates/core/permissions/src/models/user.rs new file mode 100644 index 00000000..80a8da8f --- /dev/null +++ b/crates/core/permissions/src/models/user.rs @@ -0,0 +1,25 @@ +/// User's relationship with another user (or themselves) +pub enum RelationshipStatus { + None, + User, + Friend, + Outgoing, + Incoming, + Blocked, + BlockedOther, +} + +/// User permission definitions +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "try-from-primitive", derive(num_enum::TryFromPrimitive))] +#[repr(u32)] +pub enum UserPermission { + Access = 1 << 0, + ViewProfile = 1 << 1, + SendMessage = 1 << 2, + Invite = 1 << 3, +} + +impl_op_ex!(+ |a: &UserPermission, b: &UserPermission| -> u32 { *a as u32 | *b as u32 }); +impl_op_ex_commutative!(+ |a: &u32, b: &UserPermission| -> u32 { *a | *b as u32 }); diff --git a/crates/core/permissions/src/test.rs b/crates/core/permissions/src/test.rs new file mode 100644 index 00000000..d58e8919 --- /dev/null +++ b/crates/core/permissions/src/test.rs @@ -0,0 +1,377 @@ +use crate::{ + calculate_channel_permissions, calculate_user_permissions, ChannelPermission, ChannelType, + Override, PermissionQuery, RelationshipStatus, DEFAULT_PERMISSION_DIRECT_MESSAGE, + DEFAULT_PERMISSION_SERVER, DEFAULT_PERMISSION_VIEW_ONLY, +}; + +#[async_std::test] +async fn validate_user_permissions() { + /// Scenario in which we are friends with a user + /// and we have a DM channel open with them + struct Scenario {} + let mut query = Scenario {}; + + let perms = calculate_user_permissions(&mut query).await; + assert_eq!(perms, u32::MAX); + + let perms = calculate_channel_permissions(&mut query).await; + let value: u64 = perms.into(); + assert_eq!(value, *DEFAULT_PERMISSION_DIRECT_MESSAGE); + + #[async_trait] + impl PermissionQuery for Scenario { + async fn are_we_privileged(&mut self) -> bool { + false + } + + async fn are_we_a_bot(&mut self) -> bool { + false + } + + async fn are_the_users_same(&mut self) -> bool { + false + } + + async fn user_relationship(&mut self) -> RelationshipStatus { + RelationshipStatus::Friend + } + + async fn user_is_bot(&mut self) -> bool { + false + } + + async fn have_mutual_connection(&mut self) -> bool { + false + } + + async fn are_we_server_owner(&mut self) -> bool { + unreachable!() + } + + async fn are_we_a_member(&mut self) -> bool { + unreachable!() + } + + async fn get_default_server_permissions(&mut self) -> u64 { + unreachable!() + } + + async fn get_our_server_role_overrides(&mut self) -> Vec { + unreachable!() + } + + async fn are_we_timed_out(&mut self) -> bool { + unreachable!() + } + + async fn get_channel_type(&mut self) -> ChannelType { + ChannelType::DirectMessage + } + + async fn get_default_channel_permissions(&mut self) -> Override { + unreachable!() + } + + async fn get_our_channel_role_overrides(&mut self) -> Vec { + unreachable!() + } + + async fn do_we_own_the_channel(&mut self) -> bool { + unreachable!() + } + + async fn are_we_part_of_the_channel(&mut self) -> bool { + true + } + + async fn set_recipient_as_user(&mut self) { + // no-op + } + + async fn set_server_from_channel(&mut self) { + unreachable!() + } + } +} + +#[async_std::test] +async fn validate_group_permissions() { + /// Scenario in which we are in a group channel with only talking permission + struct Scenario {} + let mut query = Scenario {}; + + let perms = calculate_channel_permissions(&mut query).await; + let value: u64 = perms.into(); + assert_eq!( + value, + *DEFAULT_PERMISSION_VIEW_ONLY | ChannelPermission::SendMessage as u64 + ); + + #[async_trait] + impl PermissionQuery for Scenario { + async fn are_we_privileged(&mut self) -> bool { + false + } + + async fn are_we_a_bot(&mut self) -> bool { + unreachable!() + } + + async fn are_the_users_same(&mut self) -> bool { + unreachable!() + } + + async fn user_relationship(&mut self) -> RelationshipStatus { + unreachable!() + } + + async fn user_is_bot(&mut self) -> bool { + unreachable!() + } + + async fn have_mutual_connection(&mut self) -> bool { + unreachable!() + } + + async fn are_we_server_owner(&mut self) -> bool { + unreachable!() + } + + async fn are_we_a_member(&mut self) -> bool { + unreachable!() + } + + async fn get_default_server_permissions(&mut self) -> u64 { + unreachable!() + } + + async fn get_our_server_role_overrides(&mut self) -> Vec { + unreachable!() + } + + async fn are_we_timed_out(&mut self) -> bool { + unreachable!() + } + + async fn get_channel_type(&mut self) -> ChannelType { + ChannelType::Group + } + + async fn get_default_channel_permissions(&mut self) -> Override { + Override { + allow: ChannelPermission::SendMessage as u64, + deny: 0, + } + } + + async fn get_our_channel_role_overrides(&mut self) -> Vec { + unreachable!() + } + + async fn do_we_own_the_channel(&mut self) -> bool { + false + } + + async fn are_we_part_of_the_channel(&mut self) -> bool { + true + } + + async fn set_recipient_as_user(&mut self) { + unreachable!() + } + + async fn set_server_from_channel(&mut self) { + unreachable!() + } + } +} + +#[async_std::test] +async fn validate_server_permissions() { + /// Scenario in which we are in a server channel where: + /// - the server grants reading history and sending messages by default + /// - we have a role that allows us to upload files and react but denies reading history + /// - however the channel disallows sending messages + /// - and removes our role specific react permission + struct Scenario {} + let mut query = Scenario {}; + + let perms = calculate_channel_permissions(&mut query).await; + let value: u64 = perms.into(); + assert_eq!( + value, + ChannelPermission::ViewChannel as u64 | ChannelPermission::UploadFiles as u64 + ); + + #[async_trait] + impl PermissionQuery for Scenario { + async fn are_we_privileged(&mut self) -> bool { + false + } + + async fn are_we_a_bot(&mut self) -> bool { + unreachable!() + } + + async fn are_the_users_same(&mut self) -> bool { + unreachable!() + } + + async fn user_relationship(&mut self) -> RelationshipStatus { + unreachable!() + } + + async fn user_is_bot(&mut self) -> bool { + unreachable!() + } + + async fn have_mutual_connection(&mut self) -> bool { + unreachable!() + } + + async fn are_we_server_owner(&mut self) -> bool { + false + } + + async fn are_we_a_member(&mut self) -> bool { + true + } + + async fn get_default_server_permissions(&mut self) -> u64 { + ChannelPermission::ViewChannel as u64 + | ChannelPermission::SendMessage as u64 + | ChannelPermission::ReadMessageHistory as u64 + } + + async fn get_our_server_role_overrides(&mut self) -> Vec { + vec![Override { + allow: ChannelPermission::UploadFiles as u64 | ChannelPermission::React as u64, + deny: ChannelPermission::ReadMessageHistory as u64, + }] + } + + async fn are_we_timed_out(&mut self) -> bool { + false + } + + async fn get_channel_type(&mut self) -> ChannelType { + ChannelType::ServerChannel + } + + async fn get_default_channel_permissions(&mut self) -> Override { + Override { + allow: 0, + deny: ChannelPermission::SendMessage as u64, + } + } + + async fn get_our_channel_role_overrides(&mut self) -> Vec { + vec![Override { + allow: 0, + deny: ChannelPermission::React as u64, + }] + } + + async fn do_we_own_the_channel(&mut self) -> bool { + unreachable!() + } + + async fn are_we_part_of_the_channel(&mut self) -> bool { + unreachable!() + } + + async fn set_recipient_as_user(&mut self) { + unreachable!() + } + + async fn set_server_from_channel(&mut self) { + // no-op + } + } +} + +#[async_std::test] +async fn validate_timed_out_member() { + /// Scenario in which we are in a server that we have been timed out from + struct Scenario {} + let mut query = Scenario {}; + + let perms = calculate_channel_permissions(&mut query).await; + let value: u64 = perms.into(); + assert_eq!(value, *DEFAULT_PERMISSION_VIEW_ONLY); + + #[async_trait] + impl PermissionQuery for Scenario { + async fn are_we_privileged(&mut self) -> bool { + false + } + + async fn are_we_a_bot(&mut self) -> bool { + unreachable!() + } + + async fn are_the_users_same(&mut self) -> bool { + unreachable!() + } + + async fn user_relationship(&mut self) -> RelationshipStatus { + unreachable!() + } + + async fn user_is_bot(&mut self) -> bool { + unreachable!() + } + + async fn have_mutual_connection(&mut self) -> bool { + unreachable!() + } + + async fn are_we_server_owner(&mut self) -> bool { + false + } + + async fn are_we_a_member(&mut self) -> bool { + true + } + + async fn get_default_server_permissions(&mut self) -> u64 { + *DEFAULT_PERMISSION_SERVER + } + + async fn get_our_server_role_overrides(&mut self) -> Vec { + vec![] + } + + async fn are_we_timed_out(&mut self) -> bool { + true + } + + async fn get_channel_type(&mut self) -> ChannelType { + ChannelType::ServerChannel + } + + async fn get_default_channel_permissions(&mut self) -> Override { + Override { allow: 0, deny: 0 } + } + + async fn get_our_channel_role_overrides(&mut self) -> Vec { + vec![] + } + + async fn do_we_own_the_channel(&mut self) -> bool { + unreachable!() + } + + async fn are_we_part_of_the_channel(&mut self) -> bool { + unreachable!() + } + + async fn set_recipient_as_user(&mut self) { + unreachable!() + } + + async fn set_server_from_channel(&mut self) { + // no-op + } + } +} diff --git a/crates/core/permissions/src/trait.rs b/crates/core/permissions/src/trait.rs new file mode 100644 index 00000000..d4a7c3d3 --- /dev/null +++ b/crates/core/permissions/src/trait.rs @@ -0,0 +1,67 @@ +use crate::{ChannelType, Override, RelationshipStatus}; + +#[async_trait] +pub trait PermissionQuery { + // * For calculating user permission + + /// Is our perspective user privileged? + async fn are_we_privileged(&mut self) -> bool; + + /// Is our perspective user a bot? + async fn are_we_a_bot(&mut self) -> bool; + + /// Is our perspective user and the currently selected user the same? + async fn are_the_users_same(&mut self) -> bool; + + /// Get the relationship with have with the currently selected user + async fn user_relationship(&mut self) -> RelationshipStatus; + + /// Whether the currently selected user is a bot + async fn user_is_bot(&mut self) -> bool; + + /// Do we have a mutual connection with the currently selected user? + async fn have_mutual_connection(&mut self) -> bool; + + // * For calculating server permission + + /// Is our perspective user the server's owner? + async fn are_we_server_owner(&mut self) -> bool; + + /// Is our perspective user a member of the server? + async fn are_we_a_member(&mut self) -> bool; + + /// Get default server permission + async fn get_default_server_permissions(&mut self) -> u64; + + /// Get the ordered role overrides (from lowest to highest) for this member in this server + async fn get_our_server_role_overrides(&mut self) -> Vec; + + /// Is our perspective user timed out on this server? + async fn are_we_timed_out(&mut self) -> bool; + + // * For calculating channel permission + + /// Get the type of the channel + async fn get_channel_type(&mut self) -> ChannelType; + + /// Get the default channel permissions + /// Group channel defaults should be mapped to an allow-only override + async fn get_default_channel_permissions(&mut self) -> Override; + + /// Get the ordered role overrides (from lowest to highest) for this member in this channel + async fn get_our_channel_role_overrides(&mut self) -> Vec; + + /// Do we own this group or saved messages channel if it is one of those? + async fn do_we_own_the_channel(&mut self) -> bool; + + /// Are we a recipient of this channel? + async fn are_we_part_of_the_channel(&mut self) -> bool; + + /// Set the current user as the recipient of this channel + /// (this will only ever be called for DirectMessage channels, use unimplemented!() for other code paths) + async fn set_recipient_as_user(&mut self); + + /// Set the current server as the server owning this channel + /// (this will only ever be called for server channels, use unimplemented!() for other code paths) + async fn set_server_from_channel(&mut self); +} From 888c22cb546eb573aa6487340bee38d6ccd36f43 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 22:12:15 +0100 Subject: [PATCH 51/56] feat(core/database): start implementing permissions --- crates/core/database/Cargo.toml | 1 + crates/core/database/src/util/mod.rs | 1 + crates/core/database/src/util/permissions.rs | 205 +++++++++++++++++++ 3 files changed, 207 insertions(+) create mode 100644 crates/core/database/src/util/permissions.rs diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 104fa271..1381edfc 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -22,6 +22,7 @@ default = [ "mongodb", "async-std-runtime" ] [dependencies] # Core revolt-result = { version = "0.0.1", path = "../result" } +revolt-permissions = { version = "0.0.1", path = "../permissions" } # Utility log = "0.4" diff --git a/crates/core/database/src/util/mod.rs b/crates/core/database/src/util/mod.rs index 7b6d3fe1..7b1c1337 100644 --- a/crates/core/database/src/util/mod.rs +++ b/crates/core/database/src/util/mod.rs @@ -1 +1,2 @@ +pub mod permissions; pub mod reference; diff --git a/crates/core/database/src/util/permissions.rs b/crates/core/database/src/util/permissions.rs new file mode 100644 index 00000000..80b02e03 --- /dev/null +++ b/crates/core/database/src/util/permissions.rs @@ -0,0 +1,205 @@ +use std::borrow::Cow; + +use revolt_permissions::{ + calculate_user_permissions, ChannelType, Override, PermissionQuery, RelationshipStatus, +}; + +use crate::{Database, User}; + +/// Permissions calculator +pub struct PermissionCalculator<'a> { + database: &'a Database, + + perspective: &'a User, + user: Option>, + // pub channel: Cow<'a, Channel>, + // pub server: Cow<'a, Server>, + // pub member: Cow<'a, Member>, + + // flag_known_relationship: Option<&'a RelationshipStatus>, + cached_user_permission: Option, + cached_permission: Option, +} + +#[async_trait] +impl PermissionQuery for PermissionCalculator<'_> { + // * For calculating user permission + + /// Is our perspective user privileged? + async fn are_we_privileged(&mut self) -> bool { + self.perspective.privileged + } + + /// Is our perspective user a bot? + async fn are_we_a_bot(&mut self) -> bool { + self.perspective.bot.is_some() + } + + /// Is our perspective user and the currently selected user the same? + async fn are_the_users_same(&mut self) -> bool { + if let Some(other_user) = &self.user { + self.perspective.id == other_user.id + } else { + false + } + } + + /// Get the relationship with have with the currently selected user + async fn user_relationship(&mut self) -> RelationshipStatus { + if let Some(other_user) = &self.user { + if let Some(relations) = &self.perspective.relations { + for entry in relations { + if entry.id == other_user.id { + return match entry.status { + crate::RelationshipStatus::None => RelationshipStatus::None, + crate::RelationshipStatus::User => RelationshipStatus::User, + crate::RelationshipStatus::Friend => RelationshipStatus::Friend, + crate::RelationshipStatus::Outgoing => RelationshipStatus::Outgoing, + crate::RelationshipStatus::Incoming => RelationshipStatus::Incoming, + crate::RelationshipStatus::Blocked => RelationshipStatus::Blocked, + crate::RelationshipStatus::BlockedOther => { + RelationshipStatus::BlockedOther + } + }; + } + } + } + } + + RelationshipStatus::None + } + + /// Whether the currently selected user is a bot + async fn user_is_bot(&mut self) -> bool { + if let Some(other_user) = &self.user { + other_user.bot.is_some() + } else { + false + } + } + + /// Do we have a mutual connection with the currently selected user? + async fn have_mutual_connection(&mut self) -> bool { + // TODO: User::has_mutual_connection + false + } + + // * For calculating server permission + + /// Is our perspective user the server's owner? + async fn are_we_server_owner(&mut self) -> bool { + todo!() + } + + /// Is our perspective user a member of the server? + async fn are_we_a_member(&mut self) -> bool { + todo!() + } + + /// Get default server permission + async fn get_default_server_permissions(&mut self) -> u64 { + todo!() + } + + /// Get the ordered role overrides (from lowest to highest) for this member in this server + async fn get_our_server_role_overrides(&mut self) -> Vec { + todo!() + } + + /// Is our perspective user timed out on this server? + async fn are_we_timed_out(&mut self) -> bool { + todo!() + } + + // * For calculating channel permission + + /// Get the type of the channel + async fn get_channel_type(&mut self) -> ChannelType { + todo!() + } + + /// Get the default channel permissions + /// Group channel defaults should be mapped to an allow-only override + async fn get_default_channel_permissions(&mut self) -> Override { + todo!() + } + + /// Get the ordered role overrides (from lowest to highest) for this member in this channel + async fn get_our_channel_role_overrides(&mut self) -> Vec { + todo!() + } + + /// Do we own this group or saved messages channel if it is one of those? + async fn do_we_own_the_channel(&mut self) -> bool { + todo!() + } + + /// Are we a recipient of this channel? + async fn are_we_part_of_the_channel(&mut self) -> bool { + todo!() + } + + /// Set the current user as the recipient of this channel + /// (this will only ever be called for DirectMessage channels, use unimplemented!() for other code paths) + async fn set_recipient_as_user(&mut self) { + todo!() + } + + /// Set the current server as the server owning this channel + /// (this will only ever be called for server channels, use unimplemented!() for other code paths) + async fn set_server_from_channel(&mut self) { + todo!() + } +} + +impl<'a> PermissionCalculator<'a> { + /// Create a new permission calculator + pub fn new(database: &'a Database, perspective: &'a User) -> PermissionCalculator<'a> { + PermissionCalculator { + database, + perspective, + user: None, + + cached_user_permission: None, + cached_permission: None, + } + } + + /// Calculate the user permission value + pub async fn calc_user(mut self) -> PermissionCalculator<'a> { + if self.cached_user_permission.is_some() { + return self; + } + + if self.user.is_none() { + panic!("Expected `PermissionCalculator.user to exist."); + } + + PermissionCalculator { + cached_user_permission: Some(calculate_user_permissions(&mut self).await), + ..self + } + } + + /// Calculate the permission value + pub async fn calc(self) -> PermissionCalculator<'a> { + if self.cached_permission.is_some() { + return self; + } + + self + } + + /// Use user + pub fn user(self, user: Cow<'a, User>) -> PermissionCalculator { + PermissionCalculator { + user: Some(user), + ..self + } + } +} + +/// Short-hand for creating a permission calculator +pub fn perms<'a>(database: &'a Database, perspective: &'a User) -> PermissionCalculator<'a> { + PermissionCalculator::new(database, perspective) +} From 050f3abc899e9c9f22fbc7064f13275761b4acc1 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 22:14:38 +0100 Subject: [PATCH 52/56] chore: update crate definitions and bump versions --- Cargo.lock | 55 ++++++++++++++++++++++++++---- crates/core/database/Cargo.toml | 6 ++-- crates/core/models/Cargo.toml | 6 ++-- crates/core/permissions/Cargo.toml | 2 +- crates/core/presence/Cargo.toml | 5 ++- crates/core/result/Cargo.toml | 2 +- justfile | 1 + 7 files changed, 61 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 558f63b0..45d652c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -363,6 +363,12 @@ dependencies = [ "validator 0.15.0", ] +[[package]] +name = "auto_ops" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7460f7dd8e100147b82a63afca1a20eb6c231ee36b90ba7272e14951cb58af59" + [[package]] name = "autocfg" version = "0.1.8" @@ -2098,7 +2104,16 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf5395665662ef45796a4ff5486c5d41d29e0c09640af4c5f17fd94ee2c119c9" dependencies = [ - "num_enum_derive", + "num_enum_derive 0.5.7", +] + +[[package]] +name = "num_enum" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a015b430d3c108a207fd776d2e2196aaf8b1cf8cf93253e3a097ff3085076a1" +dependencies = [ + "num_enum_derive 0.6.1", ] [[package]] @@ -2113,6 +2128,18 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "num_enum_derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote 1.0.26", + "syn 2.0.15", +] + [[package]] name = "object" version = "0.28.4" @@ -2816,7 +2843,7 @@ dependencies = [ [[package]] name = "revolt-database" -version = "0.0.1" +version = "0.0.2" dependencies = [ "async-recursion", "async-std", @@ -2827,6 +2854,7 @@ dependencies = [ "log", "mongodb", "nanoid", + "revolt-permissions", "revolt-result", "revolt_optional_struct", "rocket", @@ -2853,7 +2881,7 @@ dependencies = [ "log", "lru", "nanoid", - "num_enum", + "num_enum 0.5.7", "once_cell", "regex", "reqwest", @@ -2875,7 +2903,7 @@ dependencies = [ [[package]] name = "revolt-models" -version = "0.0.1" +version = "0.0.2" dependencies = [ "revolt-database", "revolt-presence", @@ -2883,9 +2911,22 @@ dependencies = [ "serde", ] +[[package]] +name = "revolt-permissions" +version = "0.0.2" +dependencies = [ + "async-std", + "async-trait", + "auto_ops", + "num_enum 0.6.1", + "once_cell", + "schemars", + "serde", +] + [[package]] name = "revolt-presence" -version = "0.0.1" +version = "0.0.2" dependencies = [ "async-std", "log", @@ -2919,7 +2960,7 @@ dependencies = [ "lru", "mongodb", "nanoid", - "num_enum", + "num_enum 0.5.7", "once_cell", "pretty_env_logger", "rand 0.8.5", @@ -2946,7 +2987,7 @@ dependencies = [ [[package]] name = "revolt-result" -version = "0.0.1" +version = "0.0.2" dependencies = [ "schemars", "serde", diff --git a/crates/core/database/Cargo.toml b/crates/core/database/Cargo.toml index 1381edfc..562718ca 100644 --- a/crates/core/database/Cargo.toml +++ b/crates/core/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-database" -version = "0.0.1" +version = "0.0.2" edition = "2021" license = "AGPL-3.0-or-later" authors = [ "Paul Makles " ] @@ -21,8 +21,8 @@ default = [ "mongodb", "async-std-runtime" ] [dependencies] # Core -revolt-result = { version = "0.0.1", path = "../result" } -revolt-permissions = { version = "0.0.1", path = "../permissions" } +revolt-result = { version = "0.0.2", path = "../result" } +revolt-permissions = { version = "0.0.2", path = "../permissions" } # Utility log = "0.4" diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 409f2278..87bf2972 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-models" -version = "0.0.1" +version = "0.0.2" edition = "2021" license = "AGPL-3.0-or-later" authors = [ "Paul Makles " ] @@ -17,8 +17,8 @@ default = [ "serde", "from_database" ] [dependencies] # Repo -revolt-database = { version = "0.0.1", path = "../database", optional = true } -revolt-presence = { version = "0.0.1", path = "../presence", optional = true } +revolt-database = { version = "0.0.2", path = "../database", optional = true } +revolt-presence = { version = "0.0.2", path = "../presence", optional = true } # Serialisation serde = { version = "1", features = ["derive"], optional = true } diff --git a/crates/core/permissions/Cargo.toml b/crates/core/permissions/Cargo.toml index 5fb3c071..4db1221f 100644 --- a/crates/core/permissions/Cargo.toml +++ b/crates/core/permissions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-permissions" -version = "0.0.1" +version = "0.0.2" edition = "2021" license = "AGPL-3.0-or-later" authors = [ "Paul Makles " ] diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index 7f904775..191fc726 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "revolt-presence" -version = "0.0.1" +version = "0.0.2" edition = "2021" +license = "AGPL-3.0-or-later" +authors = [ "Paul Makles " ] +description = "Revolt Backend: User Presence" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/crates/core/result/Cargo.toml b/crates/core/result/Cargo.toml index 5f3820db..23c87e57 100644 --- a/crates/core/result/Cargo.toml +++ b/crates/core/result/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "revolt-result" -version = "0.0.1" +version = "0.0.2" edition = "2021" license = "AGPL-3.0-or-later" authors = [ "Paul Makles " ] diff --git a/justfile b/justfile index 2a75ff88..8fd8692f 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,6 @@ publish: cargo publish --package revolt-result + cargo publish --package revolt-permissions cargo publish --package revolt-database cargo publish --package revolt-presence cargo publish --package revolt-models From ae9474b95d10e3bf6aa27d87e66233483f5a05b0 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 22:17:34 +0100 Subject: [PATCH 53/56] fix(core/database): gate rocket-impl properly --- crates/core/database/src/models/users/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/core/database/src/models/users/mod.rs b/crates/core/database/src/models/users/mod.rs index b56592f6..04963173 100644 --- a/crates/core/database/src/models/users/mod.rs +++ b/crates/core/database/src/models/users/mod.rs @@ -3,6 +3,7 @@ mod ops; #[cfg(feature = "rocket-impl")] mod rocket; +#[cfg(feature = "rocket-impl")] pub use self::rocket::*; pub use model::*; pub use ops::*; From 7f201565c05003eb4d5833f2c09e0b6db6287e1e Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 23:06:51 +0100 Subject: [PATCH 54/56] feat: implement unoptimised `filter_online` if redis is not patched --- crates/bonfire/Cargo.toml | 2 +- crates/core/models/Cargo.toml | 2 +- crates/core/presence/Cargo.toml | 3 +++ crates/core/presence/src/lib.rs | 23 ++++++++++++++++++++++- crates/quark/Cargo.toml | 2 +- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/crates/bonfire/Cargo.toml b/crates/bonfire/Cargo.toml index c7d4604d..6659d0e4 100644 --- a/crates/bonfire/Cargo.toml +++ b/crates/bonfire/Cargo.toml @@ -28,4 +28,4 @@ async-tungstenite = { version = "0.17.0", features = ["async-std-runtime"] } async-std = { version = "1.8.0", features = ["tokio1", "tokio02", "attributes"] } # core -revolt-presence = { path = "../core/presence" } +revolt-presence = { path = "../core/presence", features = [ "redis-is-patched" ] } diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 87bf2972..216610fa 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -18,7 +18,7 @@ default = [ "serde", "from_database" ] [dependencies] # Repo revolt-database = { version = "0.0.2", path = "../database", optional = true } -revolt-presence = { version = "0.0.2", path = "../presence", optional = true } +revolt-presence = { version = "0.0.2", path = "../presence", optional = true, features = [ "redis-is-patched" ] } # Serialisation serde = { version = "1", features = ["derive"], optional = true } diff --git a/crates/core/presence/Cargo.toml b/crates/core/presence/Cargo.toml index 191fc726..54598d13 100644 --- a/crates/core/presence/Cargo.toml +++ b/crates/core/presence/Cargo.toml @@ -8,6 +8,9 @@ description = "Revolt Backend: User Presence" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +redis-is-patched = [] + [dev-dependencies] # Async async-std = { version = "1.8.0", features = ["attributes"] } diff --git a/crates/core/presence/src/lib.rs b/crates/core/presence/src/lib.rs index 72418734..c510e494 100644 --- a/crates/core/presence/src/lib.rs +++ b/crates/core/presence/src/lib.rs @@ -94,6 +94,7 @@ pub async fn is_online(user_id: &str) -> bool { } /// Check whether a set of users is online, returns a set of the online user IDs +#[cfg(feature = "redis-is-patched")] pub async fn filter_online(user_ids: &'_ [String]) -> HashSet { // Ignore empty list immediately, to save time. let mut set = HashSet::new(); @@ -121,9 +122,10 @@ pub async fn filter_online(user_ids: &'_ [String]) -> HashSet { // Ok so, if this breaks, that means we've lost the Redis patch which adds SMISMEMBER // Currently it's patched in through a forked repository, investigate what happen to it let data: Vec = conn - .smismember("online", user_ids) + .smismember(ONLINE_SET, user_ids) .await .expect("this shouldn't happen, please read this code! presence/mod.rs"); + if data.is_empty() { return set; } @@ -139,6 +141,25 @@ pub async fn filter_online(user_ids: &'_ [String]) -> HashSet { set } +/// Check whether a set of users is online, returns a set of the online user IDs +#[cfg(not(feature = "redis-is-patched"))] +pub async fn filter_online(user_ids: &'_ [String]) -> HashSet { + if user_ids.is_empty() { + HashSet::new() + } else if let Ok(mut conn) = get_connection().await { + let members: Vec = conn.smembers(ONLINE_SET).await.unwrap_or_default(); + let members: HashSet<&String> = members.iter().collect(); + let user_ids: HashSet<&String> = user_ids.iter().collect(); + + members + .intersection(&user_ids) + .map(|x| x.to_string()) + .collect() + } else { + HashSet::new() + } +} + /// Reset any stale presence data pub async fn clear_region(region_id: Option<&str>) { let region_id = region_id.unwrap_or(&*REGION_KEY); diff --git a/crates/quark/Cargo.toml b/crates/quark/Cargo.toml index 06f14a96..8ebe7bb2 100644 --- a/crates/quark/Cargo.toml +++ b/crates/quark/Cargo.toml @@ -92,4 +92,4 @@ sentry = "0.25.0" # Core revolt-result = { path = "../core/result", features = [ "serde", "schemas" ] } -revolt-presence = { path = "../core/presence" } +revolt-presence = { path = "../core/presence", features = [ "redis-is-patched" ] } From 6a8481f7df1a65585f88517b8586494473b742a8 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 23 Apr 2023 23:09:21 +0100 Subject: [PATCH 55/56] refactor: is-patched transitive through models --- crates/core/models/Cargo.toml | 4 +++- crates/delta/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/core/models/Cargo.toml b/crates/core/models/Cargo.toml index 216610fa..cd7c2dc4 100644 --- a/crates/core/models/Cargo.toml +++ b/crates/core/models/Cargo.toml @@ -13,12 +13,14 @@ serde = [ "dep:serde" ] schemas = [ "dep:schemars" ] from_database = [ "revolt-database", "revolt-presence" ] +redis-is-patched = [ "revolt-presence/redis-is-patched" ] + default = [ "serde", "from_database" ] [dependencies] # Repo revolt-database = { version = "0.0.2", path = "../database", optional = true } -revolt-presence = { version = "0.0.2", path = "../presence", optional = true, features = [ "redis-is-patched" ] } +revolt-presence = { version = "0.0.2", path = "../presence", optional = true } # Serialisation serde = { version = "1", features = ["derive"], optional = true } diff --git a/crates/delta/Cargo.toml b/crates/delta/Cargo.toml index c85f4a70..0c44fef4 100644 --- a/crates/delta/Cargo.toml +++ b/crates/delta/Cargo.toml @@ -57,7 +57,7 @@ revolt-quark = { path = "../quark" } # core revolt-database = { path = "../core/database", features = [ "rocket-impl" ] } -revolt-models = { path = "../core/models", features = [ "schemas" ] } +revolt-models = { path = "../core/models", features = [ "schemas", "redis-is-patched" ] } [build-dependencies] vergen = "7.5.0" From 1afb27a508731b87612030a0a7c74c81877c76f3 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Mon, 24 Apr 2023 10:08:22 +0100 Subject: [PATCH 56/56] ci: compile in permissions crate --- Dockerfile | 1 + scripts/build-image-layer.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index f86f3e3d..6bac93d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,7 @@ COPY crates/delta/Cargo.toml ./crates/delta/ COPY crates/quark/Cargo.toml ./crates/quark/ COPY crates/core/database/Cargo.toml ./crates/core/database/ COPY crates/core/models/Cargo.toml ./crates/core/models/ +COPY crates/core/permissions/Cargo.toml ./crates/core/permissions/ COPY crates/core/presence/Cargo.toml ./crates/core/presence/ COPY crates/core/result/Cargo.toml ./crates/core/result/ RUN sh /tmp/build-image-layer.sh deps diff --git a/scripts/build-image-layer.sh b/scripts/build-image-layer.sh index 82dab680..bc59adbd 100644 --- a/scripts/build-image-layer.sh +++ b/scripts/build-image-layer.sh @@ -25,6 +25,7 @@ deps() { crates/quark/src \ crates/core/database/src \ crates/core/models/src \ + crates/core/permissions/src \ crates/core/presence/src \ crates/core/result/src echo 'fn main() { panic!("stub"); }' | @@ -34,6 +35,7 @@ deps() { tee crates/quark/src/lib.rs | tee crates/core/database/src/lib.rs | tee crates/core/models/src/lib.rs | + tee crates/core/permissions/src/lib.rs | tee crates/core/presence/src/lib.rs | tee crates/core/result/src/lib.rs cargo build --locked --release --target "${BUILD_TARGET}" @@ -46,6 +48,7 @@ apps() { crates/quark/src/lib.rs \ crates/core/database/src/lib.rs \ crates/core/models/src/lib.rs \ + crates/core/permissions/src/lib.rs \ crates/core/presence/src/lib.rs \ crates/core/result/src/lib.rs cargo build --locked --release --target "${BUILD_TARGET}"