use std::time::Duration; use bson::{Bson, DateTime}; use futures::StreamExt; use mongodb::{ bson::{doc, from_bson, from_document, to_document, Document}, options::FindOptions, }; use serde::{Deserialize, Serialize}; use crate::{r#impl::mongo::MongoDb, Permission, DEFAULT_PERMISSION_SERVER}; #[derive(Serialize, Deserialize)] struct MigrationInfo { _id: i32, revision: i32, } pub const LATEST_REVISION: i32 = 18; pub async fn migrate_database(db: &MongoDb) { let migrations = db.col::("migrations"); let data = migrations .find_one(None, None) .await .expect("Failed to fetch migration data."); if let Some(doc) = data { let info: MigrationInfo = from_document(doc).expect("Failed to read migration information."); let revision = run_migrations(db, info.revision).await; migrations .update_one( doc! { "_id": info._id }, doc! { "$set": { "revision": revision } }, None, ) .await .expect("Failed to commit migration information."); info!("Migration complete. Currently at revision {}.", revision); } else { panic!("Database was configured incorrectly, possibly because initalization failed.") } } pub async fn run_migrations(db: &MongoDb, revision: i32) -> i32 { info!("Starting database migration."); if revision <= 0 { info!("Running migration [revision 0]: Test migration system."); } if revision <= 1 { info!("Running migration [revision 1 / 2021-04-24]: Migrate to Autumn v1.0.0."); let messages = db.col::("messages"); let attachments = db.col::("attachments"); messages .update_many( doc! { "attachment": { "$exists": 1_i32 } }, doc! { "$set": { "attachment.tag": "attachments", "attachment.size": 0_i32 } }, None, ) .await .expect("Failed to update messages."); attachments .update_many( doc! {}, doc! { "$set": { "tag": "attachments", "size": 0_i32 } }, None, ) .await .expect("Failed to update attachments."); } if revision <= 2 { info!("Running migration [revision 2 / 2021-05-08]: Add servers collection."); db.db() .create_collection("servers", None) .await .expect("Failed to create servers collection."); } if revision <= 3 { info!("Running migration [revision 3 / 2021-05-25]: Support multiple file uploads, add channel_unreads and user_settings."); let messages = db.col::("messages"); let mut cursor = messages .find( doc! { "attachment": { "$exists": 1_i32 } }, FindOptions::builder() .projection(doc! { "_id": 1_i32, "attachments": [ "$attachment" ] }) .build(), ) .await .expect("Failed to fetch messages."); while let Some(result) = cursor.next().await { let doc = result.unwrap(); let id = doc.get_str("_id").unwrap(); let attachments = doc.get_array("attachments").unwrap(); messages .update_one( doc! { "_id": id }, doc! { "$unset": { "attachment": 1_i32 }, "$set": { "attachments": attachments } }, None, ) .await .unwrap(); } db.db() .create_collection("channel_unreads", None) .await .expect("Failed to create channel_unreads collection."); db.db() .create_collection("user_settings", None) .await .expect("Failed to create user_settings collection."); } if revision <= 4 { info!("Running migration [revision 4 / 2021-06-01]: Add more server collections."); db.db() .create_collection("server_members", None) .await .expect("Failed to create server_members collection."); db.db() .create_collection("server_bans", None) .await .expect("Failed to create server_bans collection."); db.db() .create_collection("channel_invites", None) .await .expect("Failed to create channel_invites collection."); } if revision <= 5 { info!("Running migration [revision 5 / 2021-06-26]: Add permissions."); #[derive(Serialize)] struct Server { pub default_permissions: (i32, i32), } let server = Server { default_permissions: (0_i32, 0_i32), }; db.col::("servers") .update_many( doc! {}, doc! { "$set": to_document(&server).unwrap() }, None, ) .await .expect("Failed to migrate servers."); } if revision <= 6 { info!("Running migration [revision 6 / 2021-07-09]: Add message text index."); db.db() .run_command( doc! { "createIndexes": "messages", "indexes": [ { "key": { "content": "text" }, "name": "content" } ] }, None, ) .await .expect("Failed to create message index."); } if revision <= 7 { info!("Running migration [revision 7 / 2021-08-11]: Add message text index."); db.db() .create_collection("bots", None) .await .expect("Failed to create bots collection."); } if revision <= 8 { info!("Running migration [revision 8 / 2021-09-10]: Update to Authifier version 1."); db.db() .run_command( doc! { "dropIndexes": "accounts", "index": ["email", "email_normalised"] }, None, ) .await .expect("Failed to delete legacy account indexes."); let col = db.col::("sessions"); let mut cursor = db .col::("accounts") .find(doc! {}, None) .await .unwrap(); while let Some(doc) = cursor.next().await { if let Ok(account) = doc { let id = account.get_str("_id").unwrap(); if let Some(sessions) = account.get("sessions") { #[derive(Deserialize)] struct Session { id: String, token: String, friendly_name: String, subscription: Option, } let sessions = from_bson::>(sessions.clone()).unwrap(); for session in sessions { info!("Converting session {} to new format.", &session.id); let mut doc = doc! { "_id": session.id, "token": session.token, "user_id": id, "name": session.friendly_name, }; if let Some(sub) = session.subscription { doc.insert("subscription", sub); } col.insert_one(doc, None).await.ok(); } } else { info!("Account doesn't have any sessions!"); } } } db.col::("accounts") .update_many( doc! {}, doc! { "$unset": { "sessions": 1_i32, }, "$set": { "mfa": { "recovery_codes": [] } } }, None, ) .await .unwrap(); } if revision <= 9 { info!("Running migration [revision 9 / 2021-09-14]: Switch from last_message to last_message_id."); let mut cursor = db .col::("channels") .find(doc! {}, None) .await .unwrap(); while let Some(doc) = cursor.next().await { if let Ok(channel) = doc { let channel_id = channel.get_str("_id").unwrap(); if let Some(last_message) = channel.get("last_message") { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Obj { #[serde(rename = "_id")] id: String, } #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] pub enum LastMessage { Obj(Obj), Id(String), } let lm = from_bson::(last_message.clone()).unwrap(); let id = match lm { LastMessage::Obj(Obj { id }) => id, LastMessage::Id(id) => id, }; info!("Converting session {} to new format.", &channel_id); db.col::("channels") .update_one( doc! { "_id": channel_id }, doc! { "$set": { "last_message_id": id }, "$unset": { "last_message": 1_i32, } }, None, ) .await .unwrap(); } else { info!("{} has no last_message.", &channel_id); } } } } if revision <= 10 { info!("Running migration [revision 10 / 2021-11-01]: Remove nonce values on channels and servers."); db.col::("servers") .update_many( doc! {}, doc! { "$unset": { "nonce": 1_i32, } }, None, ) .await .unwrap(); db.col::("channels") .update_many( doc! {}, doc! { "$unset": { "nonce": 1_i32, } }, None, ) .await .unwrap(); } if revision <= 11 { info!("Running migration [revision 11 / 2021-11-14]: Add indexes to database."); db.db() .run_command( doc! { "createIndexes": "messages", "indexes": [ { "key": { "channel": 1_i32 }, "name": "channel" } ] }, None, ) .await .expect("Failed to create message index."); db.db() .run_command( doc! { "createIndexes": "channel_unreads", "indexes": [ { "key": { "_id.channel": 1_i32, "_id.user": 1_i32, }, "name": "compound_id" }, { "key": { "_id.user": 1_i32, }, "name": "user_id" } ] }, None, ) .await .expect("Failed to create channel_unreads index."); db.db() .run_command( doc! { "createIndexes": "server_members", "indexes": [ { "key": { "_id.server": 1_i32, "_id.user": 1_i32, }, "name": "compound_id" }, { "key": { "_id.user": 1_i32, }, "name": "user_id" } ] }, None, ) .await .expect("Failed to create server_members index."); } if revision <= 12 { info!("Running migration [revision 12 / 2021-11-21]: Add indexes to database."); db.db() .run_command( doc! { "createIndexes": "messages", "indexes": [ { "key": { "channel": 1_i32, "_id": 1_i32 }, "name": "channel_id_compound" } ] }, None, ) .await .expect("Failed to create message index."); } if revision <= 13 { info!("Running migration [revision 13 / 22-02-2022]: Wipe legacy permission values."); warn!("This is a destructive operation and will wipe existing permission data (excl. defaults for SendMessage)."); warn!("Taking a backup is advised."); warn!("Continuing in 10 seconds..."); async_std::task::sleep(Duration::from_secs(10)).await; let servers = db.col::("servers"); let mut cursor = servers.find(doc! {}, None).await.unwrap(); while let Some(Ok(mut document)) = cursor.next().await { let id = document.get_str("_id").unwrap().to_string(); info!("Updating server {id}"); let mut update = doc! {}; // Try to pluck channel permission SendMessage (0x2) // Structure of default_permissions used to be [server, channel] let has_send = document .get_array("default_permissions") .map(|x| { x.get(1) .map(|x| x.as_i32().map(|x| (x as u32 & 0x2) == 0x2)) }) .ok() .flatten() .flatten() .unwrap_or_default(); update.insert( "default_permissions", (*DEFAULT_PERMISSION_SERVER // Remove Send Message permission if it wasn't originally granted ^ (if has_send { 0 } else { Permission::SendMessage as u64 })) as i64, ); if let Some(Bson::Document(mut roles)) = document.remove("roles") { for role in roles.keys().cloned().collect::>() { if let Some(Bson::Document(role)) = roles.get_mut(role) { role.insert( "permissions", doc! { "a": 0_i64, "d": 0_i64, }, ); } } update.insert("roles", roles); } servers .update_one(doc! { "_id": id }, doc! { "$set": update }, None) .await .unwrap(); } let channels = db.col::("channels"); let mut cursor = channels.find(doc! {}, None).await.unwrap(); while let Some(Ok(document)) = cursor.next().await { let id = document.get_str("_id").unwrap().to_string(); info!("Updating channel {id}"); let mut unset = doc! { "permissions": 1_i32, "role_permissions": 1_i32, }; // Try to pluck channel permission SendMessage (0x2) let has_send = document .get_i32("default_permissions") .map(|x| (x as u32 & 0x2) == 0x2) .unwrap_or(true); if has_send { // Let parent permissions fall through. unset.insert("default_permissions", 1_i32); } let mut update = doc! { "$unset": unset }; if !has_send { // Block send message permission. update.insert( "$set", doc! { "default_permissions": { "a": 0_i64, "d": Permission::SendMessage as i64 } }, ); } channels .update_one(doc! { "_id": id }, update, None) .await .unwrap(); } } if revision <= 14 { info!("Running migration [revision 14 / 21-04-2022]: Split content into content and system fields."); db.col::("messages") .update_many( doc! { "content": { "$type": "object" } }, doc! { "$rename": { "content": "system" } }, None, ) .await .unwrap(); } if revision <= 15 { info!("Running migration [revision 15 / 04-06-2022]: Migrate Authifier to latest version."); let db = authifier::Database::MongoDb(authifier::database::MongoDb(db.db())); db.run_migration(authifier::Migration::M2022_06_03EnsureUpToSpec) .await .unwrap(); } if revision <= 16 { info!("Running migration [revision 16 / 07-07-2022]: Add `emojis` collection and Authifier migration."); let authifier_db = authifier::Database::MongoDb(authifier::database::MongoDb(db.db())); authifier_db .run_migration(authifier::Migration::M2022_06_09AddIndexForDeletion) .await .unwrap(); db.db() .create_collection("emojis", None) .await .expect("Failed to create emojis collection."); db.db() .run_command( doc! { "createIndexes": "emojis", "indexes": [ { "key": { "parent.id": 1_i32, }, "name": "parent_id" } ] }, None, ) .await .expect("Failed to create emoji parent index."); } if revision <= 17 { info!("Running migration [revision 17 / 15-07-2022]: Initialise `joined_at` property on server members."); db.col::("server_members") .update_many( doc! {}, doc! { "$set": { "joined_at": DateTime::now().to_rfc3339_string() } }, None, ) .await .expect("Failed to update server members."); } // Need to migrate fields on attachments, change `user_id`, `object_id`, etc to `parent`. // Reminder to update LATEST_REVISION when adding new migrations. LATEST_REVISION }