535 lines
17 KiB
Rust
535 lines
17 KiB
Rust
use crate::providers::{
|
|
ALL_PROVIDERS, DynProvider, build_status_response, panic_payload_to_string,
|
|
report_provider_error, resolve_provider_for_build, run_provider_guarded,
|
|
};
|
|
use crate::util::cache::VideoCache;
|
|
use crate::util::discord::send_discord_error_report;
|
|
use crate::util::proxy::{Proxy, all_proxies_snapshot};
|
|
use crate::util::requester::Requester;
|
|
use crate::{DbPool, db, status::*, videos::*};
|
|
use ntex::http::header;
|
|
use ntex::web;
|
|
use ntex::web::HttpRequest;
|
|
use std::cmp::Ordering;
|
|
use std::io;
|
|
use tokio::task;
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct ClientVersion {
|
|
version: u32,
|
|
subversion: u32,
|
|
name: String,
|
|
}
|
|
|
|
impl ClientVersion {
|
|
pub fn new(version: u32, subversion: u32, name: String) -> ClientVersion {
|
|
ClientVersion {
|
|
version,
|
|
subversion,
|
|
name,
|
|
}
|
|
}
|
|
|
|
pub fn parse(input: &str) -> Option<Self> {
|
|
// Example input: "Hot%20Tub/22c CFNetwork/1494.0.7 Darwin/23.4.0 0.002478"
|
|
let first_part = input.split_whitespace().next()?;
|
|
let mut name_version = first_part.splitn(2, '/');
|
|
|
|
let name = name_version.next()?;
|
|
let version_str = name_version.next()?;
|
|
|
|
// Find the index where the numeric part ends
|
|
let split_idx = version_str
|
|
.find(|c: char| !c.is_ascii_digit())
|
|
.unwrap_or(version_str.len());
|
|
|
|
let (v_num, v_alpha) = version_str.split_at(split_idx);
|
|
|
|
// Parse the numeric version
|
|
let version = v_num.parse::<u32>().ok()?;
|
|
|
|
// Convert the first character of the subversion to u32 (ASCII value),
|
|
// or 0 if it doesn't exist.
|
|
let subversion = v_alpha.chars().next().map(|ch| ch as u32).unwrap_or(0);
|
|
|
|
Some(Self {
|
|
version,
|
|
subversion,
|
|
name: name.to_string(),
|
|
})
|
|
}
|
|
}
|
|
|
|
// Implement comparisons
|
|
impl PartialEq for ClientVersion {
|
|
fn eq(&self, other: &Self) -> bool {
|
|
self.name == other.name
|
|
}
|
|
}
|
|
|
|
impl Eq for ClientVersion {}
|
|
|
|
impl PartialOrd for ClientVersion {
|
|
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
|
Some(self.cmp(other))
|
|
}
|
|
}
|
|
|
|
impl Ord for ClientVersion {
|
|
fn cmp(&self, other: &Self) -> Ordering {
|
|
self.version
|
|
.cmp(&other.version)
|
|
.then_with(|| self.subversion.cmp(&other.subversion))
|
|
}
|
|
}
|
|
|
|
fn normalize_query(raw_query: Option<&str>) -> (Option<String>, Option<String>) {
|
|
let Some(raw_query) = raw_query else {
|
|
return (None, None);
|
|
};
|
|
|
|
let mut query = raw_query.trim();
|
|
if query.is_empty() {
|
|
return (None, None);
|
|
}
|
|
|
|
while let Some(stripped) = query.strip_prefix('#') {
|
|
query = stripped.trim_start();
|
|
}
|
|
|
|
if query.is_empty() {
|
|
return (None, None);
|
|
}
|
|
|
|
let literal_query = if query.len() >= 2
|
|
&& ((query.starts_with('"') && query.ends_with('"'))
|
|
|| (query.starts_with('\'') && query.ends_with('\'')))
|
|
{
|
|
let inner = query[1..query.len() - 1].trim();
|
|
if inner.is_empty() {
|
|
None
|
|
} else {
|
|
query = inner;
|
|
Some(inner.to_ascii_lowercase())
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
(Some(query.to_string()), literal_query)
|
|
}
|
|
|
|
fn video_matches_literal_query(video: &VideoItem, literal_query: &str) -> bool {
|
|
let contains_literal = |value: &str| value.to_ascii_lowercase().contains(literal_query);
|
|
|
|
contains_literal(&video.title)
|
|
|| video.uploader.as_deref().is_some_and(contains_literal)
|
|
|| video
|
|
.tags
|
|
.as_ref()
|
|
.is_some_and(|tags| tags.iter().any(|tag| contains_literal(tag)))
|
|
}
|
|
|
|
pub fn config(cfg: &mut web::ServiceConfig) {
|
|
cfg.service(
|
|
web::resource("/status")
|
|
.route(web::post().to(status))
|
|
.route(web::get().to(status)),
|
|
)
|
|
.service(
|
|
web::resource("/videos")
|
|
// .route(web::get().to(videos_get))
|
|
.route(web::post().to(videos_post)),
|
|
)
|
|
.service(web::resource("/test").route(web::get().to(test)))
|
|
.service(web::resource("/proxies").route(web::get().to(proxies)));
|
|
}
|
|
|
|
async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
|
|
let trace_id = crate::util::flow_debug::next_trace_id("status");
|
|
let clientversion: ClientVersion = match req.headers().get("User-Agent") {
|
|
Some(v) => match v.to_str() {
|
|
Ok(useragent) => ClientVersion::parse(useragent)
|
|
.unwrap_or_else(|| ClientVersion::new(999, 0, "Hot%20Tub".to_string())),
|
|
Err(_) => ClientVersion::new(999, 0, "Hot%20Tub".to_string()),
|
|
},
|
|
_ => ClientVersion::new(999, 0, "Hot%20Tub".to_string()),
|
|
};
|
|
|
|
println!(
|
|
"Received status request with client version: {:?}",
|
|
clientversion
|
|
);
|
|
crate::flow_debug!(
|
|
"trace={} status request host={} client={:?}",
|
|
trace_id,
|
|
req.connection_info().host(),
|
|
&clientversion
|
|
);
|
|
|
|
let host = req
|
|
.headers()
|
|
.get(header::HOST)
|
|
.and_then(|h| h.to_str().ok())
|
|
.unwrap_or_default()
|
|
.to_string();
|
|
let public_url_base = format!("{}://{}", req.connection_info().scheme(), host);
|
|
let mut status = Status::new();
|
|
let mut channel_count = 0usize;
|
|
|
|
for (provider_name, provider) in ALL_PROVIDERS.iter() {
|
|
crate::flow_debug!(
|
|
"trace={} status inspecting provider={}",
|
|
trace_id,
|
|
provider_name
|
|
);
|
|
let channel_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
|
provider.get_channel(clientversion.clone())
|
|
}));
|
|
match channel_result {
|
|
Ok(Some(mut channel)) => {
|
|
if channel.favicon.starts_with('/') {
|
|
channel.favicon = format!("{}{}", public_url_base, channel.favicon);
|
|
}
|
|
channel_count += 1;
|
|
crate::flow_debug!(
|
|
"trace={} status added channel id={} provider={}",
|
|
trace_id,
|
|
channel.id.as_str(),
|
|
provider_name
|
|
);
|
|
status.add_channel(channel)
|
|
}
|
|
Ok(None) => {}
|
|
Err(payload) => {
|
|
let panic_msg = panic_payload_to_string(payload);
|
|
crate::flow_debug!(
|
|
"trace={} status provider panic provider={} panic={}",
|
|
trace_id,
|
|
provider_name,
|
|
&panic_msg
|
|
);
|
|
report_provider_error(provider_name, "status.get_channel", &panic_msg).await;
|
|
}
|
|
}
|
|
}
|
|
status.iconUrl = format!("{}/favicon.ico", public_url_base).to_string();
|
|
let response = build_status_response(status);
|
|
crate::flow_debug!(
|
|
"trace={} status response channels={} groups={}",
|
|
trace_id,
|
|
channel_count,
|
|
response.channelGroups.len()
|
|
);
|
|
Ok(web::HttpResponse::Ok().json(&response))
|
|
}
|
|
|
|
async fn videos_post(
|
|
video_request: web::types::Json<VideosRequest>,
|
|
cache: web::types::State<VideoCache>,
|
|
pool: web::types::State<DbPool>,
|
|
requester: web::types::State<Requester>,
|
|
req: HttpRequest,
|
|
) -> Result<impl web::Responder, web::Error> {
|
|
let trace_id = crate::util::flow_debug::next_trace_id("videos");
|
|
let clientversion: ClientVersion = match req.headers().get("User-Agent") {
|
|
Some(v) => match v.to_str() {
|
|
Ok(useragent) => ClientVersion::parse(useragent)
|
|
.unwrap_or_else(|| ClientVersion::new(999, 0, "Hot%20Tub".to_string())),
|
|
Err(_) => ClientVersion::new(999, 0, "Hot%20Tub".to_string()),
|
|
},
|
|
_ => ClientVersion::new(999, 0, "Hot%20Tub".to_string()),
|
|
};
|
|
let requester = requester.get_ref().clone();
|
|
// Ensure "videos" table exists with two string columns.
|
|
match pool.get() {
|
|
Ok(mut conn) => match db::has_table(&mut conn, "videos") {
|
|
Ok(false) => {
|
|
if let Err(e) = db::create_table(
|
|
&mut conn,
|
|
"CREATE TABLE videos (id TEXT NOT NULL, url TEXT NOT NULL);",
|
|
) {
|
|
report_provider_error("db", "videos_post.create_table", &e.to_string()).await;
|
|
}
|
|
}
|
|
Ok(true) => {}
|
|
Err(e) => {
|
|
report_provider_error("db", "videos_post.has_table", &e.to_string()).await;
|
|
}
|
|
},
|
|
Err(e) => {
|
|
report_provider_error("db", "videos_post.pool_get", &e.to_string()).await;
|
|
}
|
|
}
|
|
|
|
let mut videos = Videos {
|
|
pageInfo: PageInfo {
|
|
hasNextPage: true,
|
|
resultsPerPage: 10,
|
|
},
|
|
items: vec![],
|
|
};
|
|
let requested_channel: String = video_request
|
|
.channel
|
|
.as_deref()
|
|
.unwrap_or("all")
|
|
.to_string();
|
|
let channel = resolve_provider_for_build(requested_channel.as_str()).to_string();
|
|
let sort: String = video_request.sort.as_deref().unwrap_or("date").to_string();
|
|
let (query, literal_query) = normalize_query(video_request.query.as_deref());
|
|
let page: u8 = video_request
|
|
.page
|
|
.as_ref()
|
|
.and_then(|value| value.to_u8())
|
|
.unwrap_or(1);
|
|
let perPage: u8 = video_request
|
|
.perPage
|
|
.as_ref()
|
|
.and_then(|value| value.to_u8())
|
|
.unwrap_or(10);
|
|
let featured = video_request
|
|
.featured
|
|
.as_deref()
|
|
.unwrap_or("all")
|
|
.to_string();
|
|
let provider = get_provider(channel.as_str())
|
|
.ok_or_else(|| web::error::ErrorBadRequest("Invalid channel".to_string()))?;
|
|
let category = video_request
|
|
.category
|
|
.as_deref()
|
|
.unwrap_or("all")
|
|
.to_string();
|
|
let sites = if channel == "all" {
|
|
video_request
|
|
.all_provider_sites
|
|
.as_deref()
|
|
.or(video_request.sites.as_deref())
|
|
.unwrap_or("")
|
|
.to_string()
|
|
} else {
|
|
video_request.sites.as_deref().unwrap_or("").to_string()
|
|
};
|
|
let filter = video_request.filter.as_deref().unwrap_or("new").to_string();
|
|
let language = video_request
|
|
.language
|
|
.as_deref()
|
|
.unwrap_or("en")
|
|
.to_string();
|
|
let network = video_request.networks.as_deref().unwrap_or("").to_string();
|
|
let stars = video_request.stars.as_deref().unwrap_or("").to_string();
|
|
let categories = video_request
|
|
.categories
|
|
.as_deref()
|
|
.unwrap_or("")
|
|
.to_string();
|
|
let duration = video_request.duration.as_deref().unwrap_or("").to_string();
|
|
let sexuality = video_request.sexuality.as_deref().unwrap_or("").to_string();
|
|
let public_url_base = format!(
|
|
"{}://{}",
|
|
req.connection_info().scheme(),
|
|
req.connection_info().host()
|
|
);
|
|
crate::flow_debug!(
|
|
"trace={} videos request requested_channel={} resolved_channel={} sort={} query={:?} page={} per_page={} filter={} category={} sites={} client={:?}",
|
|
trace_id,
|
|
&requested_channel,
|
|
&channel,
|
|
&sort,
|
|
&query,
|
|
page,
|
|
perPage,
|
|
&filter,
|
|
&category,
|
|
&sites,
|
|
&clientversion
|
|
);
|
|
let mut requester = requester;
|
|
requester.set_debug_trace_id(Some(trace_id.clone()));
|
|
let options = ServerOptions {
|
|
featured: Some(featured),
|
|
category: Some(category),
|
|
sites: Some(sites),
|
|
filter: Some(filter),
|
|
language: Some(language),
|
|
public_url_base: Some(public_url_base),
|
|
requester: Some(requester),
|
|
network: Some(network),
|
|
stars: Some(stars),
|
|
categories: Some(categories),
|
|
duration: Some(duration),
|
|
sort: Some(sort.clone()),
|
|
sexuality: Some(sexuality),
|
|
};
|
|
crate::flow_debug!(
|
|
"trace={} videos provider dispatch provider={} literal_query={:?}",
|
|
trace_id,
|
|
&channel,
|
|
&literal_query
|
|
);
|
|
let mut video_items = run_provider_guarded(
|
|
&channel,
|
|
"videos_post.get_videos",
|
|
provider.get_videos(
|
|
cache.get_ref().clone(),
|
|
pool.get_ref().clone(),
|
|
sort.clone(),
|
|
query.clone(),
|
|
page.to_string(),
|
|
perPage.to_string(),
|
|
options.clone(),
|
|
),
|
|
)
|
|
.await;
|
|
crate::flow_debug!(
|
|
"trace={} videos provider returned count={}",
|
|
trace_id,
|
|
video_items.len()
|
|
);
|
|
|
|
// There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine
|
|
if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) {
|
|
// filter out videos without preview for old clients
|
|
video_items = video_items
|
|
.into_iter()
|
|
.filter_map(|video| {
|
|
let last_url = video
|
|
.formats
|
|
.as_ref()
|
|
.and_then(|formats| formats.last().map(|f| f.url.clone()));
|
|
if let Some(url) = last_url {
|
|
let mut v = video;
|
|
v.url = url;
|
|
return Some(v);
|
|
}
|
|
Some(video)
|
|
})
|
|
.collect();
|
|
}
|
|
|
|
if let Some(literal_query) = literal_query.as_deref() {
|
|
let before = video_items.len();
|
|
video_items.retain(|video| video_matches_literal_query(video, literal_query));
|
|
crate::flow_debug!(
|
|
"trace={} videos literal filter kept={} removed={}",
|
|
trace_id,
|
|
video_items.len(),
|
|
before.saturating_sub(video_items.len())
|
|
);
|
|
}
|
|
|
|
videos.items = video_items.clone();
|
|
if video_items.len() == 0 {
|
|
videos.pageInfo = PageInfo {
|
|
hasNextPage: false,
|
|
resultsPerPage: 10,
|
|
}
|
|
}
|
|
//###
|
|
let next_page = page.to_string().parse::<i32>().unwrap_or(1) + 1;
|
|
let provider_clone = provider.clone();
|
|
let cache_clone = cache.get_ref().clone();
|
|
let pool_clone = pool.get_ref().clone();
|
|
let sort_clone = sort.clone();
|
|
let query_clone = query.clone();
|
|
let per_page_clone = perPage.to_string();
|
|
let options_clone = options.clone();
|
|
let channel_clone = channel.clone();
|
|
let prefetch_trace_id = trace_id.clone();
|
|
task::spawn_local(async move {
|
|
crate::flow_debug!(
|
|
"trace={} videos prefetch spawn next_page={} provider={}",
|
|
prefetch_trace_id,
|
|
next_page,
|
|
&channel_clone
|
|
);
|
|
// if let AnyProvider::Spankbang(_) = provider_clone {
|
|
// // Spankbang has a delay for the next page
|
|
// ntex::time::sleep(ntex::time::Seconds(80)).await;
|
|
// }
|
|
let _ = run_provider_guarded(
|
|
&channel_clone,
|
|
"videos_post.prefetch_next_page",
|
|
provider_clone.get_videos(
|
|
cache_clone,
|
|
pool_clone,
|
|
sort_clone,
|
|
query_clone,
|
|
next_page.to_string(),
|
|
per_page_clone,
|
|
options_clone,
|
|
),
|
|
)
|
|
.await;
|
|
});
|
|
//###
|
|
|
|
for video in videos.items.iter_mut() {
|
|
if video.duration <= 120 {
|
|
let mut preview_url = video.url.clone();
|
|
if let Some(x) = &video.formats {
|
|
if let Some(first) = x.first() {
|
|
preview_url = first.url.clone();
|
|
}
|
|
}
|
|
video.preview = Some(preview_url);
|
|
}
|
|
}
|
|
|
|
crate::flow_debug!(
|
|
"trace={} videos response items={} has_next={}",
|
|
trace_id,
|
|
videos.items.len(),
|
|
videos.pageInfo.hasNextPage
|
|
);
|
|
Ok(web::HttpResponse::Ok().json(&videos))
|
|
}
|
|
|
|
pub fn get_provider(channel: &str) -> Option<DynProvider> {
|
|
let provider = ALL_PROVIDERS.get(channel).cloned();
|
|
crate::flow_debug!(
|
|
"provider lookup channel={} found={}",
|
|
channel,
|
|
provider.is_some()
|
|
);
|
|
provider
|
|
}
|
|
|
|
pub async fn test() -> Result<impl web::Responder, web::Error> {
|
|
let e = io::Error::new(io::ErrorKind::Other, "test error");
|
|
let _ = send_discord_error_report(
|
|
e.to_string(),
|
|
Some("chain_str".to_string()),
|
|
Some("Context"),
|
|
Some("xtra info"),
|
|
file!(),
|
|
line!(),
|
|
module_path!(),
|
|
)
|
|
.await;
|
|
|
|
Ok(web::HttpResponse::Ok())
|
|
}
|
|
|
|
pub async fn proxies() -> Result<impl web::Responder, web::Error> {
|
|
let proxies = all_proxies_snapshot().await.unwrap_or_default();
|
|
crate::flow_debug!("proxies endpoint snapshot_count={}", proxies.len());
|
|
let mut by_protocol: std::collections::BTreeMap<String, Vec<Proxy>> =
|
|
std::collections::BTreeMap::new();
|
|
for proxy in proxies {
|
|
by_protocol
|
|
.entry(proxy.protocol.clone())
|
|
.or_default()
|
|
.push(proxy);
|
|
}
|
|
for proxies in by_protocol.values_mut() {
|
|
proxies.sort_by(|a, b| {
|
|
a.host
|
|
.cmp(&b.host)
|
|
.then(a.port.cmp(&b.port))
|
|
.then(a.username.cmp(&b.username))
|
|
.then(a.password.cmp(&b.password))
|
|
});
|
|
}
|
|
Ok(web::HttpResponse::Ok().json(&by_protocol))
|
|
}
|