From ebbbb5e174f96066b95859aa1950463e746adf29 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Sun, 1 Sep 2024 13:58:11 +0100 Subject: [PATCH] feat(services/autumn): download and preview files --- .vscode/settings.json | 3 +- Cargo.lock | 694 +++++++++++++++++- crates/core/config/Revolt.toml | 10 +- crates/core/config/src/lib.rs | 26 + crates/core/database/src/drivers/reference.rs | 3 +- .../database/src/models/file_hashes/mod.rs | 5 + .../database/src/models/file_hashes/model.rs | 49 ++ .../database/src/models/file_hashes/ops.rs | 12 + .../src/models/file_hashes/ops/mongodb.rs | 24 + .../src/models/file_hashes/ops/reference.rs | 23 + .../core/database/src/models/files/model.rs | 50 +- crates/core/database/src/models/files/ops.rs | 3 + .../database/src/models/files/ops/mongodb.rs | 14 + .../src/models/files/ops/reference.rs | 14 + crates/core/database/src/models/mod.rs | 3 + crates/core/database/src/util/bridge/v0.rs | 3 + crates/core/files/Cargo.toml | 5 + crates/core/files/src/lib.rs | 79 +- crates/services/autumn/Cargo.toml | 14 +- crates/services/autumn/src/api.rs | 159 +++- crates/services/autumn/src/main.rs | 7 +- 21 files changed, 1119 insertions(+), 81 deletions(-) create mode 100644 crates/core/database/src/models/file_hashes/mod.rs create mode 100644 crates/core/database/src/models/file_hashes/model.rs create mode 100644 crates/core/database/src/models/file_hashes/ops.rs create mode 100644 crates/core/database/src/models/file_hashes/ops/mongodb.rs create mode 100644 crates/core/database/src/models/file_hashes/ops/reference.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index b2c3c98c..6281c6ee 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,5 @@ { "editor.formatOnSave": true, "rust-analyzer.checkOnSave.command": "clippy", - "nixEnvSelector.suggestion": false, - "nixEnvSelector.nixFile": "${workspaceRoot}/default.nix" + "nixEnvSelector.suggestion": false } diff --git a/Cargo.lock b/Cargo.lock index b5f4cf98..d6d984a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aead" version = "0.4.3" @@ -119,6 +125,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1" + [[package]] name = "allocator-api2" version = "0.2.16" @@ -155,12 +167,29 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" + [[package]] name = "arc-swap" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote 1.0.37", + "syn 2.0.76", +] + [[package]] name = "arrayref" version = "0.3.6" @@ -480,6 +509,29 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "av1-grain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6678909d8c5d46a42abcf571271e15fdbc0a225e3646cf23762cd415046c78bf" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876c75a42f6364451a033496a14c44bffe41f5f4a8236f697391f11024e596d2" +dependencies = [ + "arrayvec", +] + [[package]] name = "aws-config" version = "1.5.5" @@ -933,6 +985,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00c055ee2d014ae5981ce1016374e8213682aa14d9bf40e48ab48b5f3ef20eaa" +dependencies = [ + "heck 0.4.0", + "proc-macro2", + "quote 1.0.37", + "syn 2.0.76", +] + [[package]] name = "axum_typed_multipart" version = "0.12.1" @@ -976,7 +1040,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.5.3", "object", "rustc-demangle", ] @@ -1048,6 +1112,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e0d60973d9320722cb1206f412740e162a33b8547ea8d6be75d7cff237c7a85" +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitfield" version = "0.13.2" @@ -1066,6 +1136,12 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "bitstream-io" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b81e1519b0d82120d2fd469d5bfb2919a9361c48b02d82d04befc1cdd2002452" + [[package]] name = "blake2b_simd" version = "1.0.0" @@ -1140,6 +1216,12 @@ dependencies = [ "uuid 0.8.2", ] +[[package]] +name = "built" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "236e6289eda5a812bc6b53c3b024039382a2895fbbeef2d748b2931546d392c4" + [[package]] name = "bumpalo" version = "3.10.0" @@ -1152,12 +1234,24 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" +[[package]] +name = "bytemuck" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773d90827bc3feecfb67fab12e24de0749aad83c74b9504ecde46237b5cd24e2" + [[package]] name = "byteorder" version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "0.5.6" @@ -1239,6 +1333,16 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfg-expr" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" +dependencies = [ + "smallvec", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -1299,6 +1403,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "combine" version = "4.6.4" @@ -1346,7 +1456,7 @@ dependencies = [ "rust-ini", "serde", "serde_json", - "toml", + "toml 0.5.9", "yaml-rust", ] @@ -1450,6 +1560,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.18" @@ -1475,6 +1595,12 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-bigint" version = "0.4.9" @@ -2021,6 +2147,22 @@ dependencies = [ "pin-project-lite 0.2.13", ] +[[package]] +name = "exr" +version = "1.72.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "887d93f60543e9a9362ef8a21beedd0a833c5d9610e18c67abe15a5963dcb1a4" +dependencies = [ + "bit_field", + "flume", + "half", + "lebe", + "miniz_oxide 0.7.4", + "rayon-core", + "smallvec", + "zune-inflate", +] + [[package]] name = "fake-simd" version = "0.1.2" @@ -2055,6 +2197,15 @@ dependencies = [ "yup-oauth2", ] +[[package]] +name = "fdeflate" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f9bfee30e4dedf0ab8b422f03af778d9612b63f502710fc500a334ebe2de645" +dependencies = [ + "simd-adler32", +] + [[package]] name = "ff" version = "0.12.1" @@ -2074,7 +2225,7 @@ dependencies = [ "atomic", "pear", "serde", - "toml", + "toml 0.5.9", "uncased", "version_check", ] @@ -2091,6 +2242,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "flate2" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +dependencies = [ + "crc32fast", + "miniz_oxide 0.8.0", +] + [[package]] name = "float-cmp" version = "0.9.0" @@ -2100,6 +2261,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2359,6 +2529,16 @@ dependencies = [ "polyval 0.6.2", ] +[[package]] +name = "gif" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.26.1" @@ -2445,6 +2625,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "handlebars" version = "4.3.0" @@ -2671,7 +2861,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" dependencies = [ - "quick-error", + "quick-error 1.2.3", ] [[package]] @@ -2858,6 +3048,45 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" +[[package]] +name = "image" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904" +dependencies = [ + "byteorder-lite", + "quick-error 2.0.1", +] + +[[package]] +name = "imgref" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44feda355f4159a7c757171a77de25daf6411e217b4cabd03bd6650690468126" + [[package]] name = "impl_ops" version = "0.1.1" @@ -2910,6 +3139,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote 1.0.37", + "syn 2.0.76", +] + [[package]] name = "ipconfig" version = "0.3.0" @@ -3004,6 +3244,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.57" @@ -3080,6 +3326,12 @@ dependencies = [ "spin 0.9.8", ] +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "lettre" version = "0.10.0-rc.6" @@ -3107,6 +3359,17 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libfuzzer-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96cfd5557eb82f2b83fed4955246c988d331975a002961b07c81584d107e7f7" +dependencies = [ + "arbitrary", + "cc", + "once_cell", +] + [[package]] name = "libgit2-sys" version = "0.14.1+1.5.0" @@ -3145,6 +3408,16 @@ dependencies = [ "libc", ] +[[package]] +name = "libwebp-sys" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829b6b604f31ed6d2bccbac841fe0788de93dbd87e4eb1ba2c4adfe8c012a838" +dependencies = [ + "cc", + "glob", +] + [[package]] name = "libz-sys" version = "1.1.8" @@ -3222,6 +3495,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + [[package]] name = "lru" version = "0.7.6" @@ -3297,6 +3579,15 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", +] + [[package]] name = "md-5" version = "0.10.1" @@ -3354,6 +3645,25 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "0.8.10" @@ -3567,6 +3877,12 @@ dependencies = [ "serde", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nom" version = "7.1.1" @@ -3577,6 +3893,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "ntapi" version = "0.4.0" @@ -3599,6 +3921,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -3625,6 +3957,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote 1.0.37", + "syn 2.0.76", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -3651,6 +3994,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ + "num-bigint", "num-integer", "num-traits", ] @@ -4074,6 +4418,19 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +[[package]] +name = "png" +version = "0.17.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06e4b0d3d1312775e782c86c91a111aa1f910cbb65e1337f9975b5f9a554b5e1" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.7.4", +] + [[package]] name = "polling" version = "2.2.0" @@ -4134,7 +4491,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" dependencies = [ "thiserror", - "toml", + "toml 0.5.9", ] [[package]] @@ -4183,6 +4540,25 @@ dependencies = [ "yansi", ] +[[package]] +name = "profiling" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d84d1d7a6ac92673717f9f6d1518374ef257669c24ebc5ac25d5033828be58" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8021cf59c8ec9c432cfc2526ac6b8aa508ecaf29cd415f271b8406c1b851c3fd" +dependencies = [ + "quote 1.0.37", + "syn 2.0.76", +] + [[package]] name = "prometheus" version = "0.13.3" @@ -4197,6 +4573,15 @@ dependencies = [ "thiserror", ] +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + [[package]] name = "quanta" version = "0.12.3" @@ -4224,6 +4609,12 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "0.3.15" @@ -4382,6 +4773,55 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "rav1e" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9" +dependencies = [ + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "once_cell", + "paste", + "profiling", + "rand 0.8.5", + "rand_chacha 0.3.1", + "simd_helpers", + "system-deps", + "thiserror", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f0bfd976333248de2078d350bfdf182ff96e168a24d23d2436cef320dd4bdd" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error 2.0.1", + "rav1e", + "rgb", +] + [[package]] name = "raw-cpuid" version = "11.1.0" @@ -4391,6 +4831,26 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -4634,7 +5094,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" dependencies = [ "hostname", - "quick-error", + "quick-error 1.2.3", ] [[package]] @@ -4642,17 +5102,25 @@ name = "revolt-autumn" version = "0.7.14" dependencies = [ "axum", + "axum-macros", "axum_typed_multipart", + "image", + "lazy_static", + "moka", "revolt-config", + "revolt-database", + "revolt-files", "revolt-result", "serde", "serde_json", + "strum_macros", "tempfile", "tokio 1.35.1", "tracing", "tracing-subscriber", "utoipa", "utoipa-scalar", + "webp", ] [[package]] @@ -4802,6 +5270,10 @@ dependencies = [ "aes-gcm 0.10.3", "aws-config", "aws-sdk-s3", + "base64 0.22.1", + "revolt-config", + "revolt-result", + "typenum", ] [[package]] @@ -4983,6 +5455,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rgb" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.16.20" @@ -5696,6 +6177,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5825,6 +6315,21 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote 1.0.37", +] + [[package]] name = "slab" version = "0.4.6" @@ -5879,6 +6384,9 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] [[package]] name = "spki" @@ -5930,6 +6438,19 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote 1.0.37", + "rustversion", + "syn 2.0.76", +] + [[package]] name = "subtle" version = "2.6.1" @@ -6048,6 +6569,19 @@ dependencies = [ "libc", ] +[[package]] +name = "system-deps" +version = "6.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349" +dependencies = [ + "cfg-expr", + "heck 0.5.0", + "pkg-config", + "toml 0.8.19", + "version-compare", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -6060,6 +6594,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + [[package]] name = "tempfile" version = "3.12.0" @@ -6111,6 +6651,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "time" version = "0.3.17" @@ -6287,6 +6838,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +dependencies = [ + "indexmap 2.0.1", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "totp-lite" version = "2.0.0" @@ -6488,9 +7073,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ubyte" @@ -6750,6 +7335,17 @@ dependencies = [ "serde", ] +[[package]] +name = "v_frame" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + [[package]] name = "validator" version = "0.15.0" @@ -6847,6 +7443,12 @@ dependencies = [ "time", ] +[[package]] +name = "version-compare" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b" + [[package]] name = "version_check" version = "0.9.4" @@ -6883,26 +7485,27 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.80" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.80" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote 1.0.37", - "syn 1.0.107", + "syn 2.0.76", "wasm-bindgen-shared", ] @@ -6920,9 +7523,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.80" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote 1.0.37", "wasm-bindgen-macro-support", @@ -6930,22 +7533,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.80" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote 1.0.37", - "syn 1.0.107", + "syn 2.0.76", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.80" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "web-push" @@ -6979,6 +7582,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webp" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f53152f51fb5af0c08484c33d16cca96175881d1f3dec068c23b31a158c2d99" +dependencies = [ + "image", + "libwebp-sys", +] + [[package]] name = "webpki" version = "0.22.0" @@ -7007,6 +7620,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + [[package]] name = "wepoll-ffi" version = "0.1.2" @@ -7253,6 +7872,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.7.0" @@ -7355,6 +7983,30 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16099418600b4d8f028622f73ff6e3deaabdff330fb9a2a131dea781ee8b0768" +dependencies = [ + "zune-core", +] + [[patch.unused]] name = "redis" version = "0.23.1" diff --git a/crates/core/config/Revolt.toml b/crates/core/config/Revolt.toml index feaebaf4..0f78139d 100644 --- a/crates/core/config/Revolt.toml +++ b/crates/core/config/Revolt.toml @@ -88,6 +88,8 @@ max_concurrent_connections = 50 # Encryption key for stored files # Generate your own key using `openssl rand -base64 32` encryption_key = "qcuMA+ssxhMyKaNAKBGFfryfFtUH8NDlamQyDwGW6fU=" +# Quality used for lossy WebP previews (set to 100 for lossless) +webp_quality = 80.0 [files.limit] # Minimum image resolution @@ -118,13 +120,13 @@ emojis = [128, 128] # - default_bucket matches the name of the bucket you've created # S3 protocol endpoint -endpoint = "" +endpoint = "http://minio:9000" # S3 region name -region = "" +region = "minio" # S3 protocol key ID -access_key_id = "" +access_key_id = "minioautumn" # S3 protocol access key -secret_access_key = "" +secret_access_key = "minioautumn" # Bucket to upload to by default default_bucket = "revolt-uploads" diff --git a/crates/core/config/src/lib.rs b/crates/core/config/src/lib.rs index 912cc1ad..9868a24f 100644 --- a/crates/core/config/src/lib.rs +++ b/crates/core/config/src/lib.rs @@ -126,6 +126,31 @@ pub struct Api { pub workers: ApiWorkers, } +#[derive(Deserialize, Debug, Clone)] +pub struct FilesLimit { + pub min_resolution: [usize; 2], + pub max_mega_pixels: usize, + pub max_pixel_side: usize, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct FilesS3 { + pub endpoint: String, + pub region: String, + pub access_key_id: String, + pub secret_access_key: String, + pub default_bucket: String, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Files { + pub encryption_key: String, + pub webp_quality: f32, + pub limit: FilesLimit, + pub preview: HashMap, + pub s3: FilesS3, +} + #[derive(Deserialize, Debug, Clone)] pub struct GlobalLimits { pub group_size: usize, @@ -186,6 +211,7 @@ pub struct Settings { pub database: Database, pub hosts: Hosts, pub api: Api, + pub files: Files, pub features: Features, pub sentry: Sentry, } diff --git a/crates/core/database/src/drivers/reference.rs b/crates/core/database/src/drivers/reference.rs index 90458d24..9c51d422 100644 --- a/crates/core/database/src/drivers/reference.rs +++ b/crates/core/database/src/drivers/reference.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, sync::Arc}; use futures::lock::Mutex; use crate::{ - Bot, Channel, ChannelCompositeKey, ChannelUnread, Emoji, File, Invite, Member, + Bot, Channel, ChannelCompositeKey, ChannelUnread, Emoji, File, FileHash, Invite, Member, MemberCompositeKey, Message, RatelimitEvent, Report, Server, ServerBan, Snapshot, User, UserSettings, Webhook, }; @@ -18,6 +18,7 @@ database_derived!( pub channel_unreads: Arc>>, pub channel_webhooks: Arc>>, pub emojis: Arc>>, + pub file_hashes: Arc>>, pub files: Arc>>, pub messages: Arc>>, pub ratelimit_events: Arc>>, diff --git a/crates/core/database/src/models/file_hashes/mod.rs b/crates/core/database/src/models/file_hashes/mod.rs new file mode 100644 index 00000000..4d801b73 --- /dev/null +++ b/crates/core/database/src/models/file_hashes/mod.rs @@ -0,0 +1,5 @@ +mod model; +mod ops; + +pub use model::*; +pub use ops::*; diff --git a/crates/core/database/src/models/file_hashes/model.rs b/crates/core/database/src/models/file_hashes/model.rs new file mode 100644 index 00000000..a1208901 --- /dev/null +++ b/crates/core/database/src/models/file_hashes/model.rs @@ -0,0 +1,49 @@ +use iso8601_timestamp::Timestamp; + +auto_derived_partial!( + /// File hash + pub struct FileHash { + /// Sha256 hash of the file + #[serde(rename = "_id")] + pub id: String, + /// Sha256 hash of file after it has been processed + pub processed_hash: String, + + /// When this file was created in system + pub created_at: Timestamp, + + /// The bucket this file is stored in + pub bucket_id: String, + /// The path at which this file exists in + pub path: String, + /// Cryptographic nonce used to encrypt this file + pub iv: String, + + /// Parsed metadata of this file + pub metadata: Metadata, + /// Raw content type of this file + pub content_type: String, + /// Size of this file (in bytes) + pub size: isize, + }, + "PartialFile" +); + +auto_derived!( + /// Metadata associated with a file + #[serde(tag = "type")] + #[derive(Default)] + pub enum Metadata { + /// File is just a generic uncategorised file + #[default] + File, + /// File contains textual data and should be displayed as such + Text, + /// File is an image with specific dimensions + Image { width: isize, height: isize }, + /// File is a video with specific dimensions + Video { width: isize, height: isize }, + /// File is audio + Audio, + } +); diff --git a/crates/core/database/src/models/file_hashes/ops.rs b/crates/core/database/src/models/file_hashes/ops.rs new file mode 100644 index 00000000..d91fc8a9 --- /dev/null +++ b/crates/core/database/src/models/file_hashes/ops.rs @@ -0,0 +1,12 @@ +use revolt_result::Result; + +use crate::FileHash; + +mod mongodb; +mod reference; + +#[async_trait] +pub trait AbstractAttachmentHashes: Sync + Send { + /// Fetch an attachment hash entry by sha256 hash. + async fn fetch_attachment_hash(&self, hash: &str) -> Result; +} diff --git a/crates/core/database/src/models/file_hashes/ops/mongodb.rs b/crates/core/database/src/models/file_hashes/ops/mongodb.rs new file mode 100644 index 00000000..46aef7d9 --- /dev/null +++ b/crates/core/database/src/models/file_hashes/ops/mongodb.rs @@ -0,0 +1,24 @@ +use revolt_result::Result; + +use crate::FileHash; +use crate::MongoDb; + +use super::AbstractAttachmentHashes; + +static COL: &str = "attachment_hashes"; + +#[async_trait] +impl AbstractAttachmentHashes for MongoDb { + /// Fetch an attachment hash entry by sha256 hash. + async fn fetch_attachment_hash(&self, hash: &str) -> Result { + query!( + self, + find_one, + COL, + doc! { + "_id": hash + } + )? + .ok_or_else(|| create_error!(NotFound)) + } +} diff --git a/crates/core/database/src/models/file_hashes/ops/reference.rs b/crates/core/database/src/models/file_hashes/ops/reference.rs new file mode 100644 index 00000000..ef3795fb --- /dev/null +++ b/crates/core/database/src/models/file_hashes/ops/reference.rs @@ -0,0 +1,23 @@ +use revolt_result::Result; + +use crate::FileHash; +use crate::ReferenceDb; + +use super::AbstractAttachmentHashes; + +#[async_trait] +impl AbstractAttachmentHashes for ReferenceDb { + /// Fetch an attachment hash entry by sha256 hash. + async fn fetch_attachment_hash(&self, hash: &str) -> Result { + let hashes = self.file_hashes.lock().await; + if let Some(file) = hashes.get(hash) { + if file.id == hash { + Ok(file.clone()) + } else { + Err(create_error!(NotFound)) + } + } else { + Err(create_error!(NotFound)) + } + } +} diff --git a/crates/core/database/src/models/files/model.rs b/crates/core/database/src/models/files/model.rs index 85edcaea..786d91bc 100644 --- a/crates/core/database/src/models/files/model.rs +++ b/crates/core/database/src/models/files/model.rs @@ -1,5 +1,6 @@ -use crate::Database; +use crate::{Database, FileHash, Metadata}; +use iso8601_timestamp::Timestamp; use revolt_result::Result; auto_derived_partial!( @@ -12,13 +13,16 @@ auto_derived_partial!( pub tag: String, /// Original filename pub filename: String, - /// Parsed metadata of this file - pub metadata: Metadata, - /// Raw content type of this file - pub content_type: String, - /// Size of this file (in bytes) - pub size: isize, + /// Hash of this file + pub hash: Option, // these are Option<>s to not break file uploads on legacy Autumn + /// When this file was uploaded + pub uploaded_at: Option, // these are Option<>s to not break file uploads on legacy Autumn + /// ID of user who uploaded this file + pub uploaded_id: Option, // these are Option<>s to not break file uploads on legacy Autumn + + // TODO: used_for: field + // /// Whether this file was deleted #[serde(skip_serializing_if = "Option::is_none")] pub deleted: Option, @@ -26,6 +30,14 @@ auto_derived_partial!( #[serde(skip_serializing_if = "Option::is_none")] pub reported: Option, + // !!! DEPRECATED: + /// Parsed metadata of this file + pub metadata: Metadata, + /// Raw content type of this file + pub content_type: String, + /// Size of this file (in bytes) + pub size: isize, + // TODO: migrate this mess to having: // - author_id // - parent: Parent { Message(id), User(id), etc } @@ -43,26 +55,12 @@ auto_derived_partial!( "PartialFile" ); -auto_derived!( - /// Metadata associated with a file - #[serde(tag = "type")] - #[derive(Default)] - pub enum Metadata { - /// File is just a generic uncategorised file - #[default] - File, - /// File contains textual data and should be displayed as such - Text, - /// File is an image with specific dimensions - Image { width: isize, height: isize }, - /// File is a video with specific dimensions - Video { width: isize, height: isize }, - /// File is audio - Audio, - } -); - impl File { + /// Get the hash entry for this file + pub async fn as_hash(&self, db: &Database) -> Result { + db.fetch_attachment_hash(self.hash.as_ref().unwrap()).await + } + /// Use a file for a message attachment pub async fn use_attachment(db: &Database, id: &str, parent: &str) -> Result { db.find_and_use_attachment(id, "attachments", "message", parent) diff --git a/crates/core/database/src/models/files/ops.rs b/crates/core/database/src/models/files/ops.rs index 55d139fc..9f771aa8 100644 --- a/crates/core/database/src/models/files/ops.rs +++ b/crates/core/database/src/models/files/ops.rs @@ -10,6 +10,9 @@ pub trait AbstractAttachments: Sync + Send { /// Insert attachment into database. async fn insert_attachment(&self, attachment: &File) -> Result<()>; + /// Fetch an attachment by its id. + async fn fetch_attachment(&self, tag: &str, file_id: &str) -> Result; + /// Find an attachment by its details and mark it as used by a given parent. async fn find_and_use_attachment( &self, diff --git a/crates/core/database/src/models/files/ops/mongodb.rs b/crates/core/database/src/models/files/ops/mongodb.rs index f7c5960a..8e94a6fb 100644 --- a/crates/core/database/src/models/files/ops/mongodb.rs +++ b/crates/core/database/src/models/files/ops/mongodb.rs @@ -15,6 +15,20 @@ impl AbstractAttachments for MongoDb { query!(self, insert_one, COL, &attachment).map(|_| ()) } + /// Fetch an attachment by its id. + async fn fetch_attachment(&self, tag: &str, file_id: &str) -> Result { + query!( + self, + find_one, + COL, + doc! { + "_id": file_id, + "tag": tag + } + )? + .ok_or_else(|| create_error!(NotFound)) + } + /// Find an attachment by its details and mark it as used by a given parent. async fn find_and_use_attachment( &self, diff --git a/crates/core/database/src/models/files/ops/reference.rs b/crates/core/database/src/models/files/ops/reference.rs index b9b4f735..90533d16 100644 --- a/crates/core/database/src/models/files/ops/reference.rs +++ b/crates/core/database/src/models/files/ops/reference.rs @@ -18,6 +18,20 @@ impl AbstractAttachments for ReferenceDb { } } + /// Fetch an attachment by its id. + async fn fetch_attachment(&self, tag: &str, file_id: &str) -> Result { + let files = self.files.lock().await; + if let Some(file) = files.get(file_id) { + if file.tag == tag { + Ok(file.clone()) + } else { + Err(create_error!(NotFound)) + } + } else { + Err(create_error!(NotFound)) + } + } + /// Find an attachment by its details and mark it as used by a given parent. async fn find_and_use_attachment( &self, diff --git a/crates/core/database/src/models/mod.rs b/crates/core/database/src/models/mod.rs index 1f540fa1..671c7abb 100644 --- a/crates/core/database/src/models/mod.rs +++ b/crates/core/database/src/models/mod.rs @@ -5,6 +5,7 @@ mod channel_unreads; mod channel_webhooks; mod channels; mod emojis; +mod file_hashes; mod files; mod messages; mod ratelimit_events; @@ -23,6 +24,7 @@ pub use channel_unreads::*; pub use channel_webhooks::*; pub use channels::*; pub use emojis::*; +pub use file_hashes::*; pub use files::*; pub use messages::*; pub use ratelimit_events::*; @@ -46,6 +48,7 @@ pub trait AbstractDatabase: + channel_unreads::AbstractChannelUnreads + channel_webhooks::AbstractWebhooks + emojis::AbstractEmojis + + file_hashes::AbstractAttachmentHashes + files::AbstractAttachments + messages::AbstractMessages + ratelimit_events::AbstractRatelimitEvents diff --git a/crates/core/database/src/util/bridge/v0.rs b/crates/core/database/src/util/bridge/v0.rs index ba3f01d8..a3911a8d 100644 --- a/crates/core/database/src/util/bridge/v0.rs +++ b/crates/core/database/src/util/bridge/v0.rs @@ -419,6 +419,9 @@ impl From for crate::File { user_id: value.user_id, server_id: value.server_id, object_id: value.object_id, + hash: None, + uploaded_at: None, + uploaded_id: None, } } } diff --git a/crates/core/files/Cargo.toml b/crates/core/files/Cargo.toml index f4da042b..a2d421b9 100644 --- a/crates/core/files/Cargo.toml +++ b/crates/core/files/Cargo.toml @@ -7,7 +7,12 @@ authors = ["Paul Makles "] description = "Revolt Backend: S3 and encryption subroutines" [dependencies] +base64 = "0.22.1" aes-gcm = "0.10.3" +typenum = "1.17.0" aws-config = "1.5.5" aws-sdk-s3 = { version = "1.46.0", features = ["behavior-version-latest"] } + +revolt-config = { version = "0.7.16", path = "../config" } +revolt-result = { version = "0.7.16", path = "../result" } diff --git a/crates/core/files/src/lib.rs b/crates/core/files/src/lib.rs index b450aecd..2dec2eb8 100644 --- a/crates/core/files/src/lib.rs +++ b/crates/core/files/src/lib.rs @@ -1,3 +1,78 @@ -// pub fn +use std::io::Write; -pub fn decrypt(data: &mut [u8]) {} +use aes_gcm::{aead::AeadMutInPlace, Aes256Gcm, Key, KeyInit, Nonce}; +use revolt_config::{config, FilesS3}; +use revolt_result::{create_error, Result}; + +use aws_sdk_s3::{ + config::{Credentials, Region}, + Client, Config, +}; + +use base64::prelude::*; + +/// Create an S3 client +pub fn create_client(s3_config: FilesS3) -> Client { + let provider_name = "my-creds"; + let creds = Credentials::new( + s3_config.access_key_id, + s3_config.secret_access_key, + None, + None, + provider_name, + ); + + let config = Config::builder() + .region(Region::new(s3_config.region)) + .endpoint_url(s3_config.endpoint) + .credentials_provider(creds) + .build(); + + Client::from_conf(config) +} + +/// Create an AES-256-GCM cipher +pub fn create_cipher(key: &str) -> Aes256Gcm { + let key = &BASE64_STANDARD.decode(key).unwrap()[..]; + let key: &Key = key.into(); + Aes256Gcm::new(key) +} + +/// Fetch a file from S3 (and decrypt it) +pub async fn fetch_from_s3(bucket_id: &str, path: &str, nonce: &str) -> Result> { + let config = config().await; + let client = create_client(config.files.s3); + + // Send a request for the file + let mut obj = client + .get_object() + .bucket(bucket_id) + .key(path) + .send() + .await + .inspect_err(|err| { + revolt_config::capture_error(err); + }) + .map_err(|_| create_error!(InternalError))?; + + // Read the file from remote + let mut buf = vec![]; + while let Some(bytes) = obj.body.next().await { + let data = bytes.map_err(|_| create_error!(InternalError))?; + buf.write_all(&data) + .map_err(|_| create_error!(InternalError))?; + // is there a more efficient way to do this? + // we just want the Vec + } + + // Recover nonce as bytes + let nonce = &BASE64_STANDARD.decode(nonce).unwrap()[..]; + let nonce: &Nonce = nonce.into(); + + // Decrypt the file + create_cipher(&config.files.encryption_key) + .decrypt_in_place(nonce, b"", &mut buf) + .map_err(|_| create_error!(InternalError))?; + + Ok(buf) +} diff --git a/crates/services/autumn/Cargo.toml b/crates/services/autumn/Cargo.toml index 8eb1c614..aaa78c08 100644 --- a/crates/services/autumn/Cargo.toml +++ b/crates/services/autumn/Cargo.toml @@ -4,9 +4,18 @@ version = "0.7.14" edition = "2021" [dependencies] +# Media processing +webp = "0.3.0" +image = "0.25.2" + +# Utility +lazy_static = "1.5.0" +moka = { version = "0.12.8", features = ["future"] } + # Serialisation -serde = { version = "1.0", features = ["derive"] } +strum_macros = "0.26.4" serde_json = "1.0.68" +serde = { version = "1.0", features = ["derive"] } # Async runtime tokio = { version = "1.0", features = ["full"] } @@ -16,7 +25,9 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } # Core crates +revolt-files = { version = "0.7.16", path = "../../core/files" } revolt-config = { version = "0.7.16", path = "../../core/config" } +revolt-database = { version = "0.7.16", path = "../../core/database" } revolt-result = { version = "0.7.16", path = "../../core/result", features = [ "utoipa", "axum", @@ -24,6 +35,7 @@ revolt-result = { version = "0.7.16", path = "../../core/result", features = [ # Axum / web server tempfile = "3.12.0" +axum-macros = "0.4.1" axum_typed_multipart = "0.12.1" axum = { version = "0.7.5", features = ["multipart"] } diff --git a/crates/services/autumn/src/api.rs b/crates/services/autumn/src/api.rs index 234c5f9b..b82b8232 100644 --- a/crates/services/autumn/src/api.rs +++ b/crates/services/autumn/src/api.rs @@ -1,26 +1,57 @@ +use std::{io::Cursor, time::Duration}; + use axum::{ - extract::{DefaultBodyLimit, Multipart, Path}, - http::StatusCode, - response::{IntoResponse, Response}, + extract::{DefaultBodyLimit, Path, State}, + http::header, + response::{IntoResponse, Redirect, Response}, routing::{get, post}, Json, Router, }; use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart}; +use image::ImageReader; +use lazy_static::lazy_static; use revolt_config::config; +use revolt_database::{Database, FileHash}; +use revolt_files::fetch_from_s3; +use revolt_result::{create_error, Result}; use serde::{Deserialize, Serialize}; +use tempfile::NamedTempFile; use utoipa::ToSchema; -use tempfile::NamedTempFile; - -pub async fn router() -> Router { +/// Build the API router +pub async fn router() -> Router { let config = config().await; - Router::new().route("/", get(root)).route( - "/:tag", - post(upload_file).layer(DefaultBodyLimit::max( - config.features.limits.global.body_limit_size, - )), - ) + Router::new() + .route("/", get(root)) + .route( + "/:tag", + post(upload_file).layer(DefaultBodyLimit::max( + config.features.limits.global.body_limit_size, + )), + ) + .route("/:tag/:file_id", get(fetch_preview)) + .route("/:tag/:file_id/:file_name", get(fetch_file)) +} + +lazy_static! { + /// Short-lived file cache to allow us to populate different CDN regions without increasing bandwidth to S3 provider + /// Uploads will also be stored here to prevent immediately queued downloads from doing the entire round-trip + static ref S3_CACHE: moka::future::Cache>> = moka::future::Cache::builder() + .max_capacity(10_000) // TODO config + .time_to_live(Duration::from_secs(60)) // TODO config + .build(); +} + +/// Retrieve hash information and file data by given hash +async fn retrieve_file_by_hash(hash: &FileHash) -> Result> { + if let Some(data) = S3_CACHE.get(&hash.id).await { + data + } else { + let data = fetch_from_s3(&hash.bucket_id, &hash.path, &hash.iv).await; + S3_CACHE.insert(hash.id.to_owned(), data.clone()).await; + data + } } /// Successful root response @@ -49,7 +80,7 @@ async fn root() -> Json { } /// Available tags to upload to -#[derive(Deserialize, Debug, ToSchema)] +#[derive(Deserialize, Debug, ToSchema, strum_macros::IntoStaticStr)] #[allow(non_camel_case_types)] pub enum Tag { attachments, @@ -106,8 +137,14 @@ async fn upload_file( Ok(Json(UploadResponse { id: "aaa" })) } +/// Header value used for cache control +pub static CACHE_CONTROL: &str = "public, max-age=604800, must-revalidate"; + /// Fetch preview of file /// +/// This route will only return image content. +/// For all other file types, please use the fetch route (you will receive a redirect if you try to use this route anyways!). +/// /// Depending on the given tag, the file will be re-processed to fit the criteria: /// /// | Tag | Image Resolution † | @@ -132,18 +169,72 @@ async fn upload_file( ), )] async fn fetch_preview( - Path(tag): Path, - Path(file_id): Path, -) -> axum::response::Result { - todo!() + State(db): State, + Path((tag, file_id)): Path<(Tag, String)>, +) -> Result { + let tag: &'static str = tag.into(); + let file = db.fetch_attachment(tag, &file_id).await?; + + // Ignore deleted files + if file.deleted.is_some_and(|v| v) { + return Err(create_error!(NotFound)); + } + + let hash = file.as_hash(&db).await?; + + // Only process image files + if !hash.content_type.starts_with("image/") { + return Ok( + Redirect::permanent(&format!("/{tag}/{file_id}/{}", file.filename)).into_response(), + ); + } + + // Original image data + let data = retrieve_file_by_hash(&hash).await?; + + // Dimensions we need to resize to + let config = config().await; + let [w, h] = config.files.preview.get(tag).unwrap(); + + // Read the image and resize it + let image = ImageReader::new(Cursor::new(data)) + .with_guessed_format() + .map_err(|_| create_error!(InternalError))? + .decode() + .map_err(|_| create_error!(InternalError))? + // resize is about 2.5x slower, + // thumb approximation doesn't have terrible quality so it's fine to stick with + // .resize(width as u32, height as u32, image::imageops::FilterType::Gaussian) + // aspect ratio is preserved when scaling + .thumbnail(*w as u32, *h as u32); + + // Encode it into WEBP + let encoder = webp::Encoder::from_image(&image).expect("Could not create encoder."); + let data = if config.files.webp_quality != 100.0 { + encoder.encode(config.files.webp_quality).to_vec() + } else { + encoder.encode_lossless().to_vec() + }; + + Ok(( + [ + (header::CONTENT_TYPE, "image/webp"), + (header::CONTENT_DISPOSITION, "inline"), + (header::CACHE_CONTROL, CACHE_CONTROL), + ], + data, + ) + .into_response()) } /// Fetch original file +/// +/// Content disposition header will be set to 'attachment' to prevent browser from rendering anything. #[utoipa::path( get, path = "/{tag}/{file_id}/{file_name}", responses( - (status = 200, description = "Generated preview", body = Vec) + (status = 200, description = "Original file", body = Vec) ), params( ("tag" = Tag, Path, description = "Tag to fetch from (e.g. attachments, icons, ...)"), @@ -152,9 +243,31 @@ async fn fetch_preview( ), )] async fn fetch_file( - Path(tag): Path, - Path(file_id): Path, - Path(file_name): Path, -) -> axum::response::Result { - todo!() + State(db): State, + Path((tag, file_id, file_name)): Path<(Tag, String, String)>, +) -> Result { + let file = db.fetch_attachment(tag.into(), &file_id).await?; + + // Ignore deleted files + if file.deleted.is_some_and(|v| v) { + return Err(create_error!(NotFound)); + } + + // Ensure filename is correct + if file_name != file.filename { + return Err(create_error!(NotFound)); + } + + let hash = file.as_hash(&db).await?; + retrieve_file_by_hash(&hash).await.map(|data| { + ( + [ + (header::CONTENT_TYPE, hash.content_type), + (header::CONTENT_DISPOSITION, "attachment".to_owned()), + (header::CACHE_CONTROL, CACHE_CONTROL.to_owned()), + ], + data, + ) + .into_response() + }) } diff --git a/crates/services/autumn/src/main.rs b/crates/services/autumn/src/main.rs index 118cb062..5e6aa7db 100644 --- a/crates/services/autumn/src/main.rs +++ b/crates/services/autumn/src/main.rs @@ -2,6 +2,7 @@ use std::net::{Ipv4Addr, SocketAddr}; use axum::Router; +use revolt_database::DatabaseInfo; use tokio::net::TcpListener; use utoipa::{ openapi::security::{ApiKey, ApiKeyValue, SecurityScheme}, @@ -55,10 +56,14 @@ async fn main() -> Result<(), std::io::Error> { } } + // Connect to the database + let db = DatabaseInfo::Auto.connect().await.unwrap(); + // Configure Axum and router let app = Router::new() .merge(Scalar::with_url("/scalar", ApiDoc::openapi())) - .nest("/", api::router().await); + .nest("/", api::router().await) + .with_state(db); // Configure TCP listener and bind let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, 3000));