mirror of
https://github.com/stoatchat/stoatchat.git
synced 2026-07-06 03:06:04 +00:00
feat: initial work on tenor gif searching
This commit is contained in:
committed by
Angelo Kontaxis
parent
bfe4018e43
commit
b0c977b324
20
.github/workflows/docker.yaml
vendored
20
.github/workflows/docker.yaml
vendored
@@ -152,6 +152,26 @@ jobs:
|
||||
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest
|
||||
labels: ${{ steps.meta-january.outputs.labels }}
|
||||
|
||||
# revoltchat/gifbox
|
||||
- name: Docker meta
|
||||
id: meta-gifbox
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: |
|
||||
docker.io/revoltchat/gifbox
|
||||
ghcr.io/revoltchat/gifbox
|
||||
- name: Publish
|
||||
uses: docker/build-push-action@v4
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
file: crates/services/gifbox/Dockerfile
|
||||
tags: ${{ steps.meta-gifbox.outputs.tags }}
|
||||
build-args: |
|
||||
BASE_IMAGE=ghcr.io/${{ github.repository_owner }}/base:latest
|
||||
labels: ${{ steps.meta-gifbox.outputs.labels }}
|
||||
|
||||
# revoltchat/crond
|
||||
- name: Docker meta
|
||||
id: meta-crond
|
||||
|
||||
1150
Cargo.lock
generated
1150
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -21,9 +21,11 @@ The services and libraries that power the Revolt service.<br/>
|
||||
| `core/permissions` | [crates/core/permissions](crates/core/permissions) | Core: Permission Logic |     |
|
||||
| `core/presence` | [crates/core/presence](crates/core/presence) | Core: User Presence |     |
|
||||
| `core/result` | [crates/core/result](crates/core/result) | Core: Result and Error types |     |
|
||||
| `core/coalesced` | [crates/core/coalesced](crates/core/coalesced) | Core: Coalescion service |     |
|
||||
| `delta` | [crates/delta](crates/delta) | REST API server |  |
|
||||
| `bonfire` | [crates/bonfire](crates/bonfire) | WebSocket events server |  |
|
||||
| `services/january` | [crates/services/january](crates/services/january) | Proxy server |  |
|
||||
| `services/gifbox` | [crates/services/gifbox](crates/services/gifbox) | Tenor proxy server |  |
|
||||
| `services/autumn` | [crates/services/autumn](crates/services/autumn) | File server |  |
|
||||
| `daemons/crond` | [crates/daemons/crond](crates/daemons/crond) | Timed data clean up daemon server |  |
|
||||
| `daemons/pushd` | [crates/daemons/pushd](crates/daemons/pushd) | Push notification daemon server |  |
|
||||
@@ -66,6 +68,7 @@ As a heads-up, the development environment uses the following ports:
|
||||
| `crates/bonfire` | 14703 |
|
||||
| `crates/services/autumn` | 14704 |
|
||||
| `crates/services/january` | 14705 |
|
||||
| `crates/services/gifbox` | 14706 |
|
||||
|
||||
Now you can clone and build the project:
|
||||
|
||||
@@ -141,6 +144,8 @@ cargo run --bin revolt-bonfire
|
||||
cargo run --bin revolt-autumn
|
||||
# run the proxy server
|
||||
cargo run --bin revolt-january
|
||||
# run the tenor proxy
|
||||
cargo run --bin revolt-gifbox
|
||||
# run the push daemon (not usually needed in regular development)
|
||||
cargo run --bin revolt-pushd
|
||||
|
||||
|
||||
22
crates/core/coalesced/Cargo.toml
Normal file
22
crates/core/coalesced/Cargo.toml
Normal file
@@ -0,0 +1,22 @@
|
||||
[package]
|
||||
name = "revolt-coalesced"
|
||||
version = "0.8.8"
|
||||
edition = "2021"
|
||||
license = "MIT"
|
||||
authors = ["Paul Makles <me@insrt.uk>", "Zomatree <me@zomatree.live>"]
|
||||
description = "Revolt Backend: Coalescion service"
|
||||
|
||||
[features]
|
||||
tokio = ["dep:tokio"]
|
||||
queue = ["dep:indexmap"]
|
||||
cache = ["dep:lru"]
|
||||
|
||||
default = ["tokio", "queue", "cache"]
|
||||
|
||||
[dependencies]
|
||||
tokio = { version = "1.47.0", features = ["sync"], optional = true }
|
||||
indexmap = { version = "*", optional = true }
|
||||
lru = { version = "*", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1.47.0", features = ["rt", "rt-multi-thread", "macros", "time"] }
|
||||
9
crates/core/coalesced/LICENSE
Normal file
9
crates/core/coalesced/LICENSE
Normal file
@@ -0,0 +1,9 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Pawel Makles
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
23
crates/core/coalesced/src/config.rs
Normal file
23
crates/core/coalesced/src/config.rs
Normal file
@@ -0,0 +1,23 @@
|
||||
#[derive(Clone, PartialEq, Eq, Debug)]
|
||||
pub struct CoalescionServiceConfig {
|
||||
/// How many tasks are running at once
|
||||
pub max_concurrent: Option<usize>,
|
||||
/// Whether to queue tasks once `max_concurrent` is reached
|
||||
#[cfg(feature = "queue")]
|
||||
pub queue_requests: bool,
|
||||
/// Max amount of tasks in the buffer queue
|
||||
#[cfg(feature = "queue")]
|
||||
pub max_queue: Option<usize>,
|
||||
}
|
||||
|
||||
impl Default for CoalescionServiceConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_concurrent: Some(100),
|
||||
#[cfg(feature = "queue")]
|
||||
queue_requests: true,
|
||||
#[cfg(feature = "queue")]
|
||||
max_queue: Some(100)
|
||||
}
|
||||
}
|
||||
}
|
||||
20
crates/core/coalesced/src/error.rs
Normal file
20
crates/core/coalesced/src/error.rs
Normal file
@@ -0,0 +1,20 @@
|
||||
use std::fmt::Display;
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, Debug)]
|
||||
pub enum Error {
|
||||
RecvError,
|
||||
MaxConcurrent,
|
||||
MaxQueue,
|
||||
}
|
||||
|
||||
impl Display for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Error::RecvError => write!(f, "Unable to receive data from the channel"),
|
||||
Error::MaxConcurrent => write!(f, "Max number of tasks running at once"),
|
||||
Error::MaxQueue => write!(f, "Max number of tasks in queue"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
7
crates/core/coalesced/src/lib.rs
Normal file
7
crates/core/coalesced/src/lib.rs
Normal file
@@ -0,0 +1,7 @@
|
||||
mod config;
|
||||
mod error;
|
||||
mod service;
|
||||
|
||||
pub use config::CoalescionServiceConfig;
|
||||
pub use error::Error;
|
||||
pub use service::CoalescionService;
|
||||
168
crates/core/coalesced/src/service.rs
Normal file
168
crates/core/coalesced/src/service.rs
Normal file
@@ -0,0 +1,168 @@
|
||||
use std::{collections::{HashMap}, fmt::Debug, hash::Hash, sync::Arc, future::Future};
|
||||
|
||||
use tokio::{sync::{watch::{channel as watch_channel, Receiver}, RwLock, Mutex}};
|
||||
|
||||
#[cfg(feature = "cache")]
|
||||
use lru::LruCache;
|
||||
|
||||
#[cfg(feature = "queue")]
|
||||
use indexmap::IndexMap;
|
||||
|
||||
use crate::{CoalescionServiceConfig, Error};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub struct CoalescionService<Id: Hash + Eq, Value> {
|
||||
config: Arc<CoalescionServiceConfig>,
|
||||
watchers: Arc<RwLock<HashMap<Id, Receiver<Option<Result<Arc<Value>, Error>>>>>>,
|
||||
#[cfg(feature = "queue")]
|
||||
queue: Arc<RwLock<IndexMap<Id, Receiver<Option<Result<Arc<Value>, Error>>>>>>,
|
||||
#[cfg(feature = "cache")]
|
||||
cache: Option<Arc<Mutex<LruCache<Id, Arc<Value>>>>>,
|
||||
}
|
||||
|
||||
impl<Id: Hash + PartialEq + Eq + Clone + Ord, Value> CoalescionService<Id, Value> {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn from_config(config: CoalescionServiceConfig) -> Self {
|
||||
Self {
|
||||
config: Arc::new(config),
|
||||
watchers: Arc::new(RwLock::new(HashMap::new())),
|
||||
#[cfg(feature = "queue")]
|
||||
queue: Arc::new(RwLock::new(IndexMap::new())),
|
||||
#[cfg(feature = "cache")]
|
||||
cache: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "cache")]
|
||||
pub fn from_cache(config: CoalescionServiceConfig, cache: LruCache<Id, Arc<Value>>) -> Self {
|
||||
Self {
|
||||
cache: Some(Arc::new(Mutex::new(cache))),
|
||||
..Self::from_config(config)
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for(&self, mut receiver: Receiver<Option<Result<Arc<Value>, Error>>>) -> Result<Arc<Value>, Error> {
|
||||
receiver
|
||||
.wait_for(|v| v.is_some())
|
||||
.await
|
||||
.map_err(|_| Error::RecvError)
|
||||
.and_then(|r| r.clone().unwrap())
|
||||
}
|
||||
|
||||
async fn insert_and_execute<F: FnOnce() -> Fut, Fut: Future<Output = Value>>(&self, id: Id, func: F) -> Result<Arc<Value>, Error> {
|
||||
let (send, recv) = watch_channel(None);
|
||||
|
||||
self.watchers.write().await.insert(id.clone(), recv);
|
||||
|
||||
let value = Ok(Arc::new(func().await));
|
||||
|
||||
send.send_modify(|opt| {
|
||||
opt.replace(value.clone());
|
||||
});
|
||||
|
||||
#[cfg(feature = "cache")]
|
||||
if let Some(cache) = self.cache.as_ref() {
|
||||
if let Ok(value) = &value {
|
||||
cache.lock().await.push(id.clone(), value.clone());
|
||||
}
|
||||
};
|
||||
|
||||
self.watchers.write().await.remove(&id);
|
||||
|
||||
value
|
||||
}
|
||||
|
||||
pub async fn execute<F: FnOnce() -> Fut, Fut: Future<Output = Value>>(&self, id: Id, func: F) -> Result<Arc<Value>, Error> {
|
||||
#[cfg(feature = "cache")]
|
||||
if let Some(cache) = self.cache.as_ref() {
|
||||
if let Some(value) = cache.lock().await.get(&id) {
|
||||
return Ok(value.clone())
|
||||
}
|
||||
};
|
||||
|
||||
let (receiver, length) = {
|
||||
let watchers = self.watchers.read().await;
|
||||
let length = watchers.len();
|
||||
|
||||
(watchers.get(&id).cloned(), length)
|
||||
};
|
||||
|
||||
if let Some(receiver) = receiver {
|
||||
self.wait_for(receiver).await
|
||||
} else {
|
||||
match self.config.max_concurrent {
|
||||
Some(max_concurrent) if length >= max_concurrent => {
|
||||
#[cfg(feature = "queue")]
|
||||
if self.config.queue_requests {
|
||||
let (receiver, length) = {
|
||||
let queue = self.queue.read().await;
|
||||
|
||||
(queue.get(&id).cloned(), queue.len())
|
||||
};
|
||||
|
||||
if let Some(receiver) = receiver {
|
||||
return self.wait_for(receiver).await
|
||||
} else {
|
||||
if self.config.max_queue.is_some_and(|max_queue| max_queue >= length) {
|
||||
return Err(Error::MaxQueue)
|
||||
};
|
||||
|
||||
let (send, recv) = watch_channel(None);
|
||||
|
||||
self.queue.write().await.insert(id.clone(), recv);
|
||||
|
||||
loop {
|
||||
let length = self.watchers.read().await.len();
|
||||
|
||||
if length < max_concurrent {
|
||||
let first_key = {
|
||||
let queue = self.queue.read().await;
|
||||
queue.first().map(|v| v.0).cloned()
|
||||
};
|
||||
|
||||
if first_key == Some(id.clone()) {
|
||||
self.queue.write().await.shift_remove(&id);
|
||||
|
||||
let response = self.insert_and_execute(id, func).await;
|
||||
|
||||
send.send_modify(|opt| {
|
||||
opt.replace(response.clone());
|
||||
});
|
||||
|
||||
return response
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Err(Error::MaxConcurrent)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "queue"))]
|
||||
Err(Error::MaxConcurrent)
|
||||
}
|
||||
_ => {
|
||||
self.insert_and_execute(id, func).await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn current_task_count(&self) -> usize {
|
||||
self.watchers.read().await.len()
|
||||
}
|
||||
|
||||
pub async fn current_queue_len(&self) -> usize {
|
||||
self.queue.read().await.len()
|
||||
}
|
||||
}
|
||||
|
||||
impl<Id: Hash + PartialEq + Eq + Clone + Ord, Value> Default for CoalescionService<Id, Value> {
|
||||
fn default() -> Self {
|
||||
Self::from_config(CoalescionServiceConfig::default())
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,8 @@ voso_legacy_token = ""
|
||||
trust_cloudflare = false
|
||||
# easypwned endpoint
|
||||
easypwned = ""
|
||||
# Tenor API Key
|
||||
tenor_key = ""
|
||||
|
||||
[api.security.captcha]
|
||||
# hCaptcha configuration
|
||||
@@ -277,3 +279,4 @@ files = ""
|
||||
proxy = ""
|
||||
pushd = ""
|
||||
crond = ""
|
||||
gifbox = ""
|
||||
@@ -190,6 +190,7 @@ pub struct ApiSecurity {
|
||||
pub captcha: ApiSecurityCaptcha,
|
||||
pub trust_cloudflare: bool,
|
||||
pub easypwned: String,
|
||||
pub tenor_key: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
@@ -365,6 +366,7 @@ pub struct Sentry {
|
||||
pub proxy: String,
|
||||
pub pushd: String,
|
||||
pub crond: String,
|
||||
pub gifbox: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
|
||||
40
crates/services/gifbox/Cargo.toml
Normal file
40
crates/services/gifbox/Cargo.toml
Normal file
@@ -0,0 +1,40 @@
|
||||
[package]
|
||||
name = "revolt-gifbox"
|
||||
version = "0.8.8"
|
||||
edition = "2021"
|
||||
license = "AGPL-3.0-or-later"
|
||||
|
||||
[dependencies]
|
||||
# Serialisation
|
||||
serde = { version = "1.0", features = ["derive", "rc"] }
|
||||
serde_json = "1.0.68"
|
||||
|
||||
# Async runtime
|
||||
tokio = { version = "1.0", features = ["full"] }
|
||||
|
||||
# Web requests
|
||||
reqwest = { version = "0.12", features = ["json"] }
|
||||
|
||||
# Core crates
|
||||
revolt-config = { version = "0.8.8", path = "../../core/config" }
|
||||
revolt-models = { version = "0.8.8", path = "../../core/models" }
|
||||
revolt-result = { version = "0.8.8", path = "../../core/result", features = [
|
||||
"utoipa",
|
||||
"axum",
|
||||
] }
|
||||
revolt-coalesced = { version = "0.8.8", path = "../../core/coalesced" }
|
||||
|
||||
# Axum / web server
|
||||
axum = { version = "0.7.5" }
|
||||
axum-extra = { version = "0.9", features = ["typed-header"] }
|
||||
|
||||
# OpenAPI & documentation generation
|
||||
utoipa-scalar = { version = "0.1.0", features = ["axum"] }
|
||||
utoipa = { version = "4.2.3", features = ["axum_extras", "ulid"] }
|
||||
|
||||
# Logging
|
||||
tracing = "0.1"
|
||||
|
||||
# Utils
|
||||
lru_time_cache = "0.11.11"
|
||||
urlencoding = "2.1.3"
|
||||
12
crates/services/gifbox/Dockerfile
Normal file
12
crates/services/gifbox/Dockerfile
Normal file
@@ -0,0 +1,12 @@
|
||||
# Build Stage
|
||||
FROM ghcr.io/revoltchat/base:latest AS builder
|
||||
|
||||
# Bundle Stage
|
||||
FROM gcr.io/distroless/cc-debian12:nonroot
|
||||
COPY --from=builder /home/rust/src/target/release/revolt-gifbox ./
|
||||
COPY --from=mwader/static-ffmpeg:7.0.2 /ffmpeg /usr/local/bin/
|
||||
COPY --from=mwader/static-ffmpeg:7.0.2 /ffprobe /usr/local/bin/
|
||||
|
||||
EXPOSE 14706
|
||||
USER nonroot
|
||||
CMD ["./revolt-gifbox"]
|
||||
64
crates/services/gifbox/src/api.rs
Normal file
64
crates/services/gifbox/src/api.rs
Normal file
@@ -0,0 +1,64 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{extract::{Query, State}, routing::get, Json, Router};
|
||||
use revolt_result::{create_error, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::tenor::{Tenor, types};
|
||||
|
||||
pub async fn router() -> Router<Tenor> {
|
||||
Router::new()
|
||||
.route("/", get(root))
|
||||
.route("/search", get(search))
|
||||
}
|
||||
|
||||
/// Successful root response
|
||||
#[derive(Serialize, Debug, ToSchema)]
|
||||
pub struct RootResponse {
|
||||
message: &'static str,
|
||||
version: &'static str,
|
||||
}
|
||||
|
||||
/// Capture crate version from Cargo
|
||||
static CRATE_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
/// Root response from service
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/",
|
||||
responses(
|
||||
(status = 200, description = "Echo response", body = RootResponse)
|
||||
)
|
||||
)]
|
||||
async fn root() -> Json<RootResponse> {
|
||||
Json(RootResponse {
|
||||
message: "Gifbox lives on!",
|
||||
version: CRATE_VERSION,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SearchQueryParams {
|
||||
pub query: String,
|
||||
pub locale: String,
|
||||
pub position: Option<String>
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/search",
|
||||
responses(
|
||||
(status = 200, description = "Search results", body = SearchResponse)
|
||||
)
|
||||
)]
|
||||
async fn search(
|
||||
Query(params): Query<SearchQueryParams>,
|
||||
State(tenor): State<Tenor>,
|
||||
) -> Result<Json<Arc<types::SearchResponse>>> {
|
||||
// Todo
|
||||
tenor.search(¶ms.query, ¶ms.locale, params.position.as_deref())
|
||||
.await
|
||||
.map_err(|_| create_error!(InternalError))
|
||||
.map(Json)
|
||||
}
|
||||
65
crates/services/gifbox/src/main.rs
Normal file
65
crates/services/gifbox/src/main.rs
Normal file
@@ -0,0 +1,65 @@
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
|
||||
use axum::Router;
|
||||
|
||||
use revolt_config::config;
|
||||
use tokio::net::TcpListener;
|
||||
use utoipa::{
|
||||
openapi::security::{Http, HttpAuthScheme, SecurityScheme},
|
||||
Modify, OpenApi,
|
||||
};
|
||||
use utoipa_scalar::{Scalar, Servable as ScalarServable};
|
||||
|
||||
mod api;
|
||||
mod tenor;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), std::io::Error> {
|
||||
// Configure logging and environment
|
||||
revolt_config::configure!(gifbox);
|
||||
|
||||
// Configure API schema
|
||||
#[derive(OpenApi)]
|
||||
#[openapi(
|
||||
modifiers(&SecurityAddon),
|
||||
paths(
|
||||
api::root,
|
||||
),
|
||||
components(
|
||||
schemas(
|
||||
revolt_result::Error,
|
||||
revolt_result::ErrorType,
|
||||
)
|
||||
)
|
||||
)]
|
||||
struct ApiDoc;
|
||||
|
||||
struct SecurityAddon;
|
||||
|
||||
impl Modify for SecurityAddon {
|
||||
fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) {
|
||||
if let Some(components) = openapi.components.as_mut() {
|
||||
components.add_security_scheme(
|
||||
"api_key",
|
||||
SecurityScheme::Http(Http::new(HttpAuthScheme::Bearer)),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let config = config().await;
|
||||
let tenor = tenor::Tenor::new(&config.api.security.tenor_key);
|
||||
|
||||
// Configure Axum and router
|
||||
let app = Router::new()
|
||||
.merge(Scalar::with_url("/scalar", ApiDoc::openapi()))
|
||||
.nest("/", api::router().await)
|
||||
.with_state(tenor);
|
||||
|
||||
// Configure TCP listener and bind
|
||||
tracing::info!("Listening on 0.0.0.0:14706");
|
||||
tracing::info!("Play around with the API: http://localhost:14706/scalar");
|
||||
let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, 14706));
|
||||
let listener = TcpListener::bind(&address).await?;
|
||||
axum::serve(listener, app.into_make_service()).await
|
||||
}
|
||||
129
crates/services/gifbox/src/tenor/mod.rs
Normal file
129
crates/services/gifbox/src/tenor/mod.rs
Normal file
@@ -0,0 +1,129 @@
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use reqwest::Client;
|
||||
use tokio::sync::RwLock;
|
||||
use lru_time_cache::LruCache;
|
||||
use urlencoding::encode as url_encode;
|
||||
use revolt_coalesced::{CoalescionService, CoalescionServiceConfig};
|
||||
|
||||
pub mod types;
|
||||
|
||||
const TENOR_API_BASE_URL: &str = "https://tenor.googleapis.com/v2";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum TenorError {
|
||||
HttpError,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Tenor {
|
||||
pub key: Arc<str>,
|
||||
pub client: Client,
|
||||
pub coalescion: CoalescionService<String, Result<Arc<types::SearchResponse>, TenorError>>,
|
||||
pub cache: Arc<RwLock<LruCache<String, Arc<types::SearchResponse>>>>
|
||||
}
|
||||
|
||||
impl Tenor {
|
||||
pub fn new(key: &str) -> Self {
|
||||
// 1 hour, 1k queries
|
||||
let cache = LruCache::with_expiry_duration_and_capacity(Duration::from_secs(60*60), 1000);
|
||||
|
||||
Self {
|
||||
key: Arc::from(key),
|
||||
client: Client::new(),
|
||||
coalescion: CoalescionService::from_config(CoalescionServiceConfig {
|
||||
max_concurrent: Some(100),
|
||||
queue_requests: true,
|
||||
max_queue: None
|
||||
}),
|
||||
cache: Arc::new(RwLock::new(cache)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn search(&self, query: &str, locale: &str, position: Option<&str>) -> Result<Arc<types::SearchResponse>, TenorError> {
|
||||
let unique_key = format!("{query}:{locale}:{position:?}");
|
||||
|
||||
if self.cache.read().await.contains_key(&unique_key) {
|
||||
if let Some(response) = self.cache.write().await.get(&unique_key) {
|
||||
return Ok(response.clone())
|
||||
}
|
||||
}
|
||||
|
||||
let res = self.coalescion.execute(unique_key.clone(), || {
|
||||
let client = self.client.clone();
|
||||
|
||||
async move {
|
||||
let mut url = format!(
|
||||
"{TENOR_API_BASE_URL}/search?key={}&q={}&client_key=Gifbox&locale={}&contentfilter=medium&limit=1",
|
||||
&self.key,
|
||||
url_encode(query),
|
||||
url_encode(locale),
|
||||
);
|
||||
|
||||
if let Some(position) = position {
|
||||
url.push_str("&pos=");
|
||||
url.push_str(position);
|
||||
};
|
||||
|
||||
let response = client.get(url)
|
||||
.send()
|
||||
.await
|
||||
.inspect_err(|e| { revolt_config::capture_error(e); })
|
||||
.map_err(|_| TenorError::HttpError)?;
|
||||
|
||||
let text = response.text().await.map_err(|e| { println!("{e:?}"); TenorError::HttpError })?;
|
||||
|
||||
Ok(Arc::new(serde_json::from_str(&text).unwrap()))
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
if let Ok(resp) = &*res {
|
||||
self.cache.write().await.insert(unique_key, resp.clone());
|
||||
}
|
||||
|
||||
(*res).clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use revolt_config::config;
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn test() {
|
||||
let config = config().await;
|
||||
|
||||
let tenor = Tenor::new(&config.api.security.tenor_key);
|
||||
|
||||
let results =tenor.search("amog", "en_US", None).await.unwrap();
|
||||
|
||||
let result = &results.results[0];
|
||||
|
||||
println!("{:?}", result.media_formats.iter().next().unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn test_2() {
|
||||
let config = config().await;
|
||||
let tenor = Tenor::new(&config.api.security.tenor_key);
|
||||
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for i in 1..1001 {
|
||||
let tenor = tenor.clone();
|
||||
println!("creating search task {i}");
|
||||
|
||||
tasks.push((i, tokio::spawn(async move {
|
||||
tenor.search(&format!("amog-{i}"), "en_US", None).await
|
||||
})));
|
||||
};
|
||||
|
||||
for (i, task) in tasks {
|
||||
task.await.unwrap().unwrap();
|
||||
println!("Got result for {i}");
|
||||
};
|
||||
}
|
||||
}
|
||||
36
crates/services/gifbox/src/tenor/types.rs
Normal file
36
crates/services/gifbox/src/tenor/types.rs
Normal file
@@ -0,0 +1,36 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Serialize, Deserialize};
|
||||
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
|
||||
pub struct SearchResponse {
|
||||
pub results: Vec<TenorResult>,
|
||||
pub next: String
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
|
||||
pub struct MediaObject {
|
||||
pub url: String,
|
||||
pub dims: Vec<u64>,
|
||||
pub duration: f64,
|
||||
pub size: f64
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
|
||||
pub struct TenorResult {
|
||||
pub created: f64,
|
||||
#[serde(default)]
|
||||
pub hasaudio: bool,
|
||||
pub id: String,
|
||||
pub media_formats: HashMap<String, MediaObject>,
|
||||
pub tags: Vec<String>,
|
||||
pub title: String,
|
||||
pub content_description: String,
|
||||
pub itemurl: String,
|
||||
#[serde(default)]
|
||||
pub hascaption: bool,
|
||||
pub flags: Vec<String>,
|
||||
pub bg_color: Option<String>,
|
||||
pub url: String
|
||||
}
|
||||
@@ -32,8 +32,10 @@ deps() {
|
||||
crates/core/permissions/src \
|
||||
crates/core/presence/src \
|
||||
crates/core/result/src \
|
||||
crates/core/coalesced/src \
|
||||
crates/services/autumn/src \
|
||||
crates/services/january/src \
|
||||
crates/services/gifbox/src \
|
||||
crates/daemons/crond/src \
|
||||
crates/daemons/pushd/src
|
||||
echo 'fn main() { panic!("stub"); }' |
|
||||
@@ -41,6 +43,7 @@ deps() {
|
||||
tee crates/delta/src/main.rs |
|
||||
tee crates/services/autumn/src/main.rs |
|
||||
tee crates/services/january/src/main.rs |
|
||||
tee crates/services/gifbox/src/main.rs |
|
||||
tee crates/daemons/crond/src/main.rs |
|
||||
tee crates/daemons/pushd/src/main.rs
|
||||
echo '' |
|
||||
@@ -51,7 +54,8 @@ deps() {
|
||||
tee crates/core/parser/src/lib.rs |
|
||||
tee crates/core/permissions/src/lib.rs |
|
||||
tee crates/core/presence/src/lib.rs |
|
||||
tee crates/core/result/src/lib.rs
|
||||
tee crates/core/result/src/lib.rs |
|
||||
tee crates/core/coalesced/src/lib.rs
|
||||
|
||||
if [ -z "$TARGETARCH" ]; then
|
||||
cargo build -j 10 --locked --release
|
||||
@@ -72,7 +76,8 @@ apps() {
|
||||
crates/core/parser/src/lib.rs \
|
||||
crates/core/permissions/src/lib.rs \
|
||||
crates/core/presence/src/lib.rs \
|
||||
crates/core/result/src/lib.rs
|
||||
crates/core/result/src/lib.rs \
|
||||
crates/core/coalesced/src/lib .rs
|
||||
|
||||
if [ -z "$TARGETARCH" ]; then
|
||||
cargo build -j 10 --locked --release
|
||||
|
||||
@@ -25,6 +25,7 @@ docker build -t ghcr.io/revoltchat/server:$TAG - < crates/delta/Dockerfile
|
||||
docker build -t ghcr.io/revoltchat/bonfire:$TAG - < crates/bonfire/Dockerfile
|
||||
docker build -t ghcr.io/revoltchat/autumn:$TAG - < crates/services/autumn/Dockerfile
|
||||
docker build -t ghcr.io/revoltchat/january:$TAG - < crates/services/january/Dockerfile
|
||||
docker build -t ghcr.io/revoltchat/gifbox:$TAG - < crates/services/gifbox/Dockerfile
|
||||
docker build -t ghcr.io/revoltchat/crond:$TAG - < crates/daemons/crond/Dockerfile
|
||||
docker build -t ghcr.io/revoltchat/pushd:$TAG - < crates/daemons/pushd/Dockerfile
|
||||
|
||||
@@ -36,5 +37,6 @@ docker push ghcr.io/revoltchat/server:$TAG
|
||||
docker push ghcr.io/revoltchat/bonfire:$TAG
|
||||
docker push ghcr.io/revoltchat/autumn:$TAG
|
||||
docker push ghcr.io/revoltchat/january:$TAG
|
||||
docker push ghcr.io/revoltchat/gifbox:$TAG
|
||||
docker push ghcr.io/revoltchat/crond:$TAG
|
||||
docker push ghcr.io/revoltchat/pushd:$TAG
|
||||
|
||||
@@ -4,10 +4,12 @@ cargo build \
|
||||
--bin revolt-delta \
|
||||
--bin revolt-bonfire \
|
||||
--bin revolt-autumn \
|
||||
--bin revolt-january
|
||||
--bin revolt-january \
|
||||
--bin revolt-gifbox
|
||||
|
||||
trap 'pkill -f revolt-' SIGINT
|
||||
cargo run --bin revolt-delta &
|
||||
cargo run --bin revolt-bonfire &
|
||||
cargo run --bin revolt-autumn &
|
||||
cargo run --bin revolt-january
|
||||
cargo run --bin revolt-january &
|
||||
cargo run --bin revolt-gifbox
|
||||
|
||||
Reference in New Issue
Block a user