From 6bd8221eda5edec23155f7099403155bb07febde Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sat, 4 Mar 2023 11:18:05 +0000 Subject: [PATCH] feat(core): add raw db statisics endpoint for admin panel --- crates/delta/src/routes/mod.rs | 3 +- crates/delta/src/routes/stats.rs | 13 +++ crates/quark/src/impl/dummy/admin/stats.rs | 10 ++ crates/quark/src/impl/dummy/mod.rs | 1 + crates/quark/src/impl/mongo/admin/stats.rs | 90 +++++++++++++++ crates/quark/src/impl/mongo/mod.rs | 1 + crates/quark/src/models/admin/stats.rs | 122 +++++++++++++++++++++ crates/quark/src/models/mod.rs | 1 + crates/quark/src/traits/admin/stats.rs | 6 + crates/quark/src/traits/mod.rs | 3 + 10 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 crates/delta/src/routes/stats.rs create mode 100644 crates/quark/src/impl/dummy/admin/stats.rs create mode 100644 crates/quark/src/impl/mongo/admin/stats.rs create mode 100644 crates/quark/src/models/admin/stats.rs create mode 100644 crates/quark/src/traits/admin/stats.rs diff --git a/crates/delta/src/routes/mod.rs b/crates/delta/src/routes/mod.rs index 5220f863..d7724c4a 100644 --- a/crates/delta/src/routes/mod.rs +++ b/crates/delta/src/routes/mod.rs @@ -12,6 +12,7 @@ mod push; mod root; mod safety; mod servers; +mod stats; mod sync; mod users; @@ -21,7 +22,7 @@ pub fn mount(mut rocket: Rocket) -> Rocket { mount_endpoints_and_merged_docs! { rocket, "/".to_owned(), settings, "/" => (vec![], custom_openapi_spec()), - "" => openapi_get_routes_spec![root::root, root::ping], + "" => openapi_get_routes_spec![root::root, stats::stats, root::ping], "/users" => users::routes(), "/bots" => bots::routes(), "/channels" => channels::routes(), diff --git a/crates/delta/src/routes/stats.rs b/crates/delta/src/routes/stats.rs new file mode 100644 index 00000000..ff7d1ac9 --- /dev/null +++ b/crates/delta/src/routes/stats.rs @@ -0,0 +1,13 @@ +use revolt_quark::models::stats::Stats; +use revolt_quark::{Db, Result}; + +use rocket::serde::json::Json; + +/// # Query Stats +/// +/// Fetch various technical statistics. +#[openapi(tag = "Core")] +#[get("/stats")] +pub async fn stats(db: &Db) -> Result> { + Ok(Json(db.generate_stats().await?)) +} diff --git a/crates/quark/src/impl/dummy/admin/stats.rs b/crates/quark/src/impl/dummy/admin/stats.rs new file mode 100644 index 00000000..a5fc808e --- /dev/null +++ b/crates/quark/src/impl/dummy/admin/stats.rs @@ -0,0 +1,10 @@ +use crate::{models::stats::Stats, AbstractStats, Result}; + +use super::super::DummyDb; + +#[async_trait] +impl AbstractStats for DummyDb { + async fn generate_stats(&self) -> Result { + todo!() + } +} diff --git a/crates/quark/src/impl/dummy/mod.rs b/crates/quark/src/impl/dummy/mod.rs index 46ad4593..fe26641b 100644 --- a/crates/quark/src/impl/dummy/mod.rs +++ b/crates/quark/src/impl/dummy/mod.rs @@ -2,6 +2,7 @@ use crate::AbstractDatabase; pub mod admin { pub mod migrations; + pub mod stats; } pub mod media { diff --git a/crates/quark/src/impl/mongo/admin/stats.rs b/crates/quark/src/impl/mongo/admin/stats.rs new file mode 100644 index 00000000..8fa1e2b8 --- /dev/null +++ b/crates/quark/src/impl/mongo/admin/stats.rs @@ -0,0 +1,90 @@ +use std::collections::HashMap; + +use bson::{from_document, Document}; +use futures::StreamExt; + +use crate::{ + models::stats::{Index, Stats}, + AbstractStats, Error, Result, +}; + +use super::super::MongoDb; + +#[async_trait] +impl AbstractStats for MongoDb { + async fn generate_stats(&self) -> Result { + let mut indices = HashMap::new(); + let mut coll_stats = HashMap::new(); + + let collection_names = + self.db() + .list_collection_names(None) + .await + .map_err(|_| Error::DatabaseError { + operation: "list_collection_names", + with: "database", + })?; + + for collection in collection_names { + indices.insert( + collection.to_string(), + self.col::(&collection) + .aggregate( + vec![doc! { + "$indexStats": { } + }], + None, + ) + .await + .map_err(|_| Error::DatabaseError { + operation: "aggregate", + with: "col", + })? + .filter_map(|s| async { s.ok() }) + .collect::>() + .await + .into_iter() + .filter_map(|doc| from_document(doc).ok()) + .collect::>(), + ); + + coll_stats.insert( + collection.to_string(), + self.col::(&collection) + .aggregate( + vec![doc! { + "$collStats": { + "latencyStats": { + "histograms": true + }, + "storageStats": {}, + "count": {}, + "queryExecStats": {} + } + }], + None, + ) + .await + .map_err(|_| Error::DatabaseError { + operation: "aggregate", + with: "col", + })? + .filter_map(|s| async { s.ok() }) + .collect::>() + .await + .into_iter() + .filter_map(|doc| from_document(doc).ok()) + .next() + .ok_or(Error::DatabaseError { + operation: "next aggregation", + with: "col", + })?, + ); + } + + Ok(Stats { + indices, + coll_stats, + }) + } +} diff --git a/crates/quark/src/impl/mongo/mod.rs b/crates/quark/src/impl/mongo/mod.rs index cc44bdcc..3e7e9796 100644 --- a/crates/quark/src/impl/mongo/mod.rs +++ b/crates/quark/src/impl/mongo/mod.rs @@ -13,6 +13,7 @@ use crate::{util::manipulation::prefix_keys, AbstractDatabase, Error, Result}; pub mod admin { pub mod migrations; + pub mod stats; } pub mod media { diff --git a/crates/quark/src/models/admin/stats.rs b/crates/quark/src/models/admin/stats.rs new file mode 100644 index 00000000..7866cd8c --- /dev/null +++ b/crates/quark/src/models/admin/stats.rs @@ -0,0 +1,122 @@ +use std::collections::HashMap; + +use iso8601_timestamp::Timestamp; +use serde::{Deserialize, Serialize}; + +/// Index access information +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +pub struct IndexAccess { + /// Operations since timestamp + ops: i32, + + /// Timestamp at which data keeping begun + since: Timestamp, +} + +/// Collection index +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +pub struct Index { + /// Index name + name: String, + + /// Access information + accesses: IndexAccess, +} + +/// Histogram entry +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +pub struct LatencyHistogramEntry { + /// Time + micros: i64, + + /// Count + count: i64, +} + +/// Collection latency stats +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +pub struct LatencyStats { + /// Total operations + ops: i64, + + /// Timestamp at which data keeping begun + latency: i64, + + /// Histogram representation of latency data + histogram: Vec, +} + +/// Collection storage stats +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +#[serde(rename_all = "camelCase")] +pub struct StorageStats { + /// Uncompressed data size + size: i64, + + /// Data size on disk + storage_size: i64, + + /// Total size of all indexes + total_index_size: i64, + + /// Sum of storage size and total index size + total_size: i64, + + /// Individual index sizes + index_sizes: HashMap, + + /// Number of documents in collection + count: i64, + + /// Average size of each document + avg_obj_size: i64, +} + +/// Query collection scan stats +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +#[serde(rename_all = "camelCase")] +pub struct CollectionScans { + /// Number of total collection scans + total: i64, + + /// Number of total collection scans not using a tailable cursor + non_tailable: i64, +} + +/// Collection query execution stats +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +#[serde(rename_all = "camelCase")] +pub struct QueryExecStats { + /// Stats regarding collection scans + collection_scans: CollectionScans, +} + +/// Collection stats +#[derive(Serialize, Deserialize, JsonSchema, Debug)] +#[serde(rename_all = "camelCase")] +pub struct CollectionStats { + /// Namespace + ns: String, + + /// Local time + local_time: Timestamp, + + /// Latency stats + latency_stats: HashMap, + + /// Query exec stats + query_exec_stats: QueryExecStats, + + /// Number of documents in collection + count: u64, +} + +/// Server Stats +#[derive(Serialize, JsonSchema, Debug)] +pub struct Stats { + /// Index usage information + pub indices: HashMap>, + + /// Collection stats + pub coll_stats: HashMap, +} diff --git a/crates/quark/src/models/mod.rs b/crates/quark/src/models/mod.rs index 4be15903..c0fdcf28 100644 --- a/crates/quark/src/models/mod.rs +++ b/crates/quark/src/models/mod.rs @@ -1,6 +1,7 @@ mod admin { pub mod migrations; pub mod simple; + pub mod stats; } mod media { diff --git a/crates/quark/src/traits/admin/stats.rs b/crates/quark/src/traits/admin/stats.rs new file mode 100644 index 00000000..99e10819 --- /dev/null +++ b/crates/quark/src/traits/admin/stats.rs @@ -0,0 +1,6 @@ +use crate::{models::stats::Stats, Result}; + +#[async_trait] +pub trait AbstractStats: Sync + Send { + async fn generate_stats(&self) -> Result; +} diff --git a/crates/quark/src/traits/mod.rs b/crates/quark/src/traits/mod.rs index 24855f8c..b7c2f6c9 100644 --- a/crates/quark/src/traits/mod.rs +++ b/crates/quark/src/traits/mod.rs @@ -1,5 +1,6 @@ mod admin { pub mod migrations; + pub mod stats; } mod media { @@ -32,6 +33,7 @@ mod safety { } pub use admin::migrations::AbstractMigrations; +pub use admin::stats::AbstractStats; pub use media::attachment::AbstractAttachment; pub use media::emoji::AbstractEmoji; @@ -56,6 +58,7 @@ pub trait AbstractDatabase: Sync + Send + AbstractMigrations + + AbstractStats + AbstractAttachment + AbstractEmoji + AbstractChannel