1045 lines
34 KiB
Rust
1045 lines
34 KiB
Rust
use crate::providers::{
|
|
ALL_PROVIDERS, DynProvider, build_status_response, panic_payload_to_string,
|
|
report_provider_error, resolve_provider_for_build, run_provider_guarded,
|
|
run_uploader_provider_guarded,
|
|
};
|
|
use crate::uploaders::{UploaderProfile, UploadersRequest};
|
|
use crate::util::cache::VideoCache;
|
|
use crate::util::discord::send_discord_error_report;
|
|
use crate::util::proxy::{Proxy, all_proxies_snapshot};
|
|
use crate::util::requester::Requester;
|
|
use crate::{DbPool, db, status::*, videos::*};
|
|
use ntex::http::header;
|
|
use ntex::web;
|
|
use ntex::web::HttpRequest;
|
|
use serde_json::Value;
|
|
use std::cmp::Ordering;
|
|
use std::io;
|
|
use std::process::Command;
|
|
use tokio::task;
|
|
use url::Url;
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct ClientVersion {
|
|
version: u32,
|
|
subversion: u32,
|
|
name: String,
|
|
}
|
|
|
|
impl ClientVersion {
|
|
pub fn new(version: u32, subversion: u32, name: String) -> ClientVersion {
|
|
ClientVersion {
|
|
version,
|
|
subversion,
|
|
name,
|
|
}
|
|
}
|
|
|
|
pub fn parse(input: &str) -> Option<Self> {
|
|
// Example input: "Hot%20Tub/22c CFNetwork/1494.0.7 Darwin/23.4.0 0.002478"
|
|
let first_part = input.split_whitespace().next()?;
|
|
let mut name_version = first_part.splitn(2, '/');
|
|
|
|
let name = name_version.next()?;
|
|
let version_str = name_version.next()?;
|
|
|
|
// Find the index where the numeric part ends
|
|
let split_idx = version_str
|
|
.find(|c: char| !c.is_ascii_digit())
|
|
.unwrap_or(version_str.len());
|
|
|
|
let (v_num, v_alpha) = version_str.split_at(split_idx);
|
|
|
|
// Parse the numeric version
|
|
let version = v_num.parse::<u32>().ok()?;
|
|
|
|
// Convert the first character of the subversion to u32 (ASCII value),
|
|
// or 0 if it doesn't exist.
|
|
let subversion = v_alpha.chars().next().map(|ch| ch as u32).unwrap_or(0);
|
|
|
|
Some(Self {
|
|
version,
|
|
subversion,
|
|
name: name.to_string(),
|
|
})
|
|
}
|
|
}
|
|
|
|
// Implement comparisons
|
|
impl PartialEq for ClientVersion {
|
|
fn eq(&self, other: &Self) -> bool {
|
|
self.name == other.name
|
|
}
|
|
}
|
|
|
|
impl Eq for ClientVersion {}
|
|
|
|
impl PartialOrd for ClientVersion {
|
|
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
|
Some(self.cmp(other))
|
|
}
|
|
}
|
|
|
|
impl Ord for ClientVersion {
|
|
fn cmp(&self, other: &Self) -> Ordering {
|
|
self.version
|
|
.cmp(&other.version)
|
|
.then_with(|| self.subversion.cmp(&other.subversion))
|
|
}
|
|
}
|
|
|
|
fn normalize_query(raw_query: Option<&str>) -> (Option<String>, Option<String>) {
|
|
let Some(raw_query) = raw_query else {
|
|
return (None, None);
|
|
};
|
|
|
|
let mut query = raw_query.trim();
|
|
if query.is_empty() {
|
|
return (None, None);
|
|
}
|
|
|
|
while let Some(stripped) = query.strip_prefix('#') {
|
|
query = stripped.trim_start();
|
|
}
|
|
|
|
if query.is_empty() {
|
|
return (None, None);
|
|
}
|
|
|
|
let literal_query = if query.len() >= 2
|
|
&& ((query.starts_with('"') && query.ends_with('"'))
|
|
|| (query.starts_with('\'') && query.ends_with('\'')))
|
|
{
|
|
let inner = query[1..query.len() - 1].trim();
|
|
if inner.is_empty() {
|
|
None
|
|
} else {
|
|
query = inner;
|
|
Some(inner.to_ascii_lowercase())
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
(Some(query.to_string()), literal_query)
|
|
}
|
|
|
|
fn video_matches_literal_query(video: &VideoItem, literal_query: &str) -> bool {
|
|
let contains_literal = |value: &str| value.to_ascii_lowercase().contains(literal_query);
|
|
|
|
contains_literal(&video.title)
|
|
|| video.uploader.as_deref().is_some_and(contains_literal)
|
|
|| video
|
|
.tags
|
|
.as_ref()
|
|
.is_some_and(|tags| tags.iter().any(|tag| contains_literal(tag)))
|
|
}
|
|
|
|
fn normalize_query_url(query: &str) -> Option<String> {
|
|
let trimmed = query.trim();
|
|
if trimmed.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
let parsed = Url::parse(trimmed).ok()?;
|
|
match parsed.scheme() {
|
|
"http" | "https" => Some(parsed.to_string()),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
fn video_item_from_ytdlp_payload(
|
|
channel: &str,
|
|
fallback_url: &str,
|
|
payload: &Value,
|
|
) -> Option<VideoItem> {
|
|
let title = payload
|
|
.get("title")
|
|
.and_then(|value| value.as_str())
|
|
.filter(|value| !value.trim().is_empty())?
|
|
.to_string();
|
|
let page_url = payload
|
|
.get("webpage_url")
|
|
.and_then(|value| value.as_str())
|
|
.filter(|value| value.starts_with("http://") || value.starts_with("https://"))
|
|
.unwrap_or(fallback_url)
|
|
.to_string();
|
|
let id = payload
|
|
.get("id")
|
|
.and_then(|value| value.as_str())
|
|
.filter(|value| !value.trim().is_empty())
|
|
.map(ToOwned::to_owned)
|
|
.or_else(|| {
|
|
Url::parse(&page_url)
|
|
.ok()
|
|
.and_then(|parsed| parsed.path_segments()?.next_back().map(ToOwned::to_owned))
|
|
})?;
|
|
let thumb = payload
|
|
.get("thumbnail")
|
|
.and_then(|value| value.as_str())
|
|
.unwrap_or("")
|
|
.to_string();
|
|
let duration = payload
|
|
.get("duration")
|
|
.and_then(|value| value.as_u64())
|
|
.and_then(|value| u32::try_from(value).ok())
|
|
.unwrap_or(0);
|
|
|
|
let mut item = VideoItem::new(id, title, page_url, channel.to_string(), thumb, duration);
|
|
item.views = payload
|
|
.get("view_count")
|
|
.and_then(|value| value.as_u64())
|
|
.and_then(|value| u32::try_from(value).ok());
|
|
item.uploader = payload
|
|
.get("uploader")
|
|
.and_then(|value| value.as_str())
|
|
.filter(|value| !value.trim().is_empty())
|
|
.map(ToOwned::to_owned);
|
|
item.uploaderUrl = payload
|
|
.get("uploader_url")
|
|
.and_then(|value| value.as_str())
|
|
.filter(|value| !value.trim().is_empty())
|
|
.map(ToOwned::to_owned);
|
|
item.preview = payload
|
|
.get("thumbnail")
|
|
.and_then(|value| value.as_str())
|
|
.filter(|value| !value.trim().is_empty())
|
|
.map(ToOwned::to_owned);
|
|
|
|
let formats = payload
|
|
.get("formats")
|
|
.and_then(|value| value.as_array())
|
|
.map(|entries| {
|
|
entries
|
|
.iter()
|
|
.filter_map(|format| {
|
|
let format_url =
|
|
format
|
|
.get("url")
|
|
.and_then(|value| value.as_str())
|
|
.filter(|value| {
|
|
value.starts_with("http://") || value.starts_with("https://")
|
|
})?;
|
|
let quality = format
|
|
.get("format_id")
|
|
.and_then(|value| value.as_str())
|
|
.or_else(|| format.get("format").and_then(|value| value.as_str()))
|
|
.or_else(|| format.get("resolution").and_then(|value| value.as_str()))
|
|
.unwrap_or("auto")
|
|
.to_string();
|
|
let ext = format
|
|
.get("ext")
|
|
.and_then(|value| value.as_str())
|
|
.unwrap_or("mp4")
|
|
.to_string();
|
|
|
|
let mut video_format =
|
|
VideoFormat::new(format_url.to_string(), quality.clone(), ext)
|
|
.format_id(quality.clone());
|
|
if let Some(note) = format.get("format_note").and_then(|value| value.as_str()) {
|
|
if !note.trim().is_empty() {
|
|
video_format = video_format.format_note(note.to_string());
|
|
}
|
|
}
|
|
Some(video_format)
|
|
})
|
|
.collect::<Vec<_>>()
|
|
})
|
|
.unwrap_or_default();
|
|
if !formats.is_empty() {
|
|
item.formats = Some(formats);
|
|
}
|
|
|
|
Some(item)
|
|
}
|
|
|
|
fn videos_from_ytdlp_query_url(
|
|
channel: &str,
|
|
query_url: &str,
|
|
limit: usize,
|
|
) -> Option<Vec<VideoItem>> {
|
|
let output = Command::new("yt-dlp")
|
|
.arg("-J")
|
|
.arg("--no-warnings")
|
|
.arg("--extractor-args")
|
|
.arg("generic:impersonate=chrome")
|
|
.arg(query_url)
|
|
.output()
|
|
.ok()?;
|
|
if !output.status.success() {
|
|
return None;
|
|
}
|
|
|
|
let payload: Value = serde_json::from_slice(&output.stdout).ok()?;
|
|
if let Some(entries) = payload.get("entries").and_then(|value| value.as_array()) {
|
|
let items = entries
|
|
.iter()
|
|
.filter_map(|entry| video_item_from_ytdlp_payload(channel, query_url, entry))
|
|
.take(limit)
|
|
.collect::<Vec<_>>();
|
|
return (!items.is_empty()).then_some(items);
|
|
}
|
|
|
|
video_item_from_ytdlp_payload(channel, query_url, &payload).map(|item| vec![item])
|
|
}
|
|
|
|
pub fn config(cfg: &mut web::ServiceConfig) {
|
|
cfg.service(
|
|
web::resource("/status")
|
|
.route(web::post().to(status))
|
|
.route(web::get().to(status)),
|
|
)
|
|
.service(
|
|
web::resource("/videos")
|
|
// .route(web::get().to(videos_get))
|
|
.route(web::post().to(videos_post)),
|
|
)
|
|
.service(web::resource("/uploaders").route(web::post().to(uploaders_post)))
|
|
.service(web::resource("/test").route(web::get().to(test)))
|
|
.service(web::resource("/proxies").route(web::get().to(proxies)));
|
|
}
|
|
|
|
fn uploader_request_is_valid(request: &UploadersRequest) -> bool {
|
|
request.uploaderId.is_some() || request.uploaderName.is_some()
|
|
}
|
|
|
|
fn provider_hint_from_uploader_id(uploader_id: &str) -> Option<String> {
|
|
let (channel, _) = uploader_id.split_once(':')?;
|
|
Some(resolve_provider_for_build(channel).to_string())
|
|
}
|
|
|
|
fn uploader_provider_ids() -> Vec<String> {
|
|
let mut ids = ALL_PROVIDERS
|
|
.iter()
|
|
.filter_map(|(provider_id, _)| (*provider_id != "all").then(|| (*provider_id).to_string()))
|
|
.collect::<Vec<_>>();
|
|
ids.sort();
|
|
ids
|
|
}
|
|
|
|
fn uploader_match_sort_key(profile: &UploaderProfile) -> (u64, String, String) {
|
|
(
|
|
profile.videoCount,
|
|
profile.channel.clone().unwrap_or_default(),
|
|
profile.id.clone(),
|
|
)
|
|
}
|
|
|
|
async fn lookup_uploader_with_provider(
|
|
provider_id: &str,
|
|
provider: DynProvider,
|
|
cache: VideoCache,
|
|
pool: DbPool,
|
|
request: &UploadersRequest,
|
|
options: crate::videos::ServerOptions,
|
|
) -> Result<Option<UploaderProfile>, String> {
|
|
run_uploader_provider_guarded(
|
|
provider_id,
|
|
"uploaders_post.get_uploader",
|
|
provider.get_uploader(
|
|
cache,
|
|
pool,
|
|
request.uploaderId.clone(),
|
|
request.uploaderName.clone(),
|
|
request.query.clone(),
|
|
request.profileContent,
|
|
options,
|
|
),
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
|
|
#[cfg(feature = "debug")]
|
|
let trace_id = crate::util::flow_debug::next_trace_id("status");
|
|
let clientversion: ClientVersion = match req.headers().get("User-Agent") {
|
|
Some(v) => match v.to_str() {
|
|
Ok(useragent) => ClientVersion::parse(useragent)
|
|
.unwrap_or_else(|| ClientVersion::new(999, 0, "Hot%20Tub".to_string())),
|
|
Err(_) => ClientVersion::new(999, 0, "Hot%20Tub".to_string()),
|
|
},
|
|
_ => ClientVersion::new(999, 0, "Hot%20Tub".to_string()),
|
|
};
|
|
|
|
println!(
|
|
"Received status request with client version: {:?}",
|
|
clientversion
|
|
);
|
|
crate::flow_debug!(
|
|
"trace={} status request host={} client={:?}",
|
|
trace_id,
|
|
req.connection_info().host(),
|
|
&clientversion
|
|
);
|
|
|
|
let host = req
|
|
.headers()
|
|
.get(header::HOST)
|
|
.and_then(|h| h.to_str().ok())
|
|
.unwrap_or_default()
|
|
.to_string();
|
|
let public_url_base = format!("{}://{}", req.connection_info().scheme(), host);
|
|
let mut status = Status::new();
|
|
#[cfg(feature = "debug")]
|
|
let mut channel_count = 0usize;
|
|
|
|
for (provider_name, provider) in ALL_PROVIDERS.iter() {
|
|
crate::flow_debug!(
|
|
"trace={} status inspecting provider={}",
|
|
trace_id,
|
|
provider_name
|
|
);
|
|
let channel_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
|
provider.get_channel(clientversion.clone())
|
|
}));
|
|
match channel_result {
|
|
Ok(Some(mut channel)) => {
|
|
if channel.favicon.starts_with('/') {
|
|
channel.favicon = format!("{}{}", public_url_base, channel.favicon);
|
|
}
|
|
#[cfg(feature = "debug")]
|
|
{
|
|
channel_count += 1;
|
|
}
|
|
crate::flow_debug!(
|
|
"trace={} status added channel id={} provider={}",
|
|
trace_id,
|
|
channel.id.as_str(),
|
|
provider_name
|
|
);
|
|
status.add_channel(channel)
|
|
}
|
|
Ok(None) => {}
|
|
Err(payload) => {
|
|
let panic_msg = panic_payload_to_string(payload);
|
|
crate::flow_debug!(
|
|
"trace={} status provider panic provider={} panic={}",
|
|
trace_id,
|
|
provider_name,
|
|
&panic_msg
|
|
);
|
|
report_provider_error(provider_name, "status.get_channel", &panic_msg).await;
|
|
}
|
|
}
|
|
}
|
|
status.iconUrl = format!("{}/favicon.ico", public_url_base).to_string();
|
|
let response = build_status_response(status);
|
|
crate::flow_debug!(
|
|
"trace={} status response channels={} groups={}",
|
|
trace_id,
|
|
channel_count,
|
|
response.channelGroups.len()
|
|
);
|
|
Ok(web::HttpResponse::Ok().json(&response))
|
|
}
|
|
|
|
async fn videos_post(
|
|
video_request: web::types::Json<VideosRequest>,
|
|
cache: web::types::State<VideoCache>,
|
|
pool: web::types::State<DbPool>,
|
|
requester: web::types::State<Requester>,
|
|
req: HttpRequest,
|
|
) -> Result<impl web::Responder, web::Error> {
|
|
let trace_id = crate::util::flow_debug::next_trace_id("videos");
|
|
let clientversion: ClientVersion = match req.headers().get("User-Agent") {
|
|
Some(v) => match v.to_str() {
|
|
Ok(useragent) => ClientVersion::parse(useragent)
|
|
.unwrap_or_else(|| ClientVersion::new(999, 0, "Hot%20Tub".to_string())),
|
|
Err(_) => ClientVersion::new(999, 0, "Hot%20Tub".to_string()),
|
|
},
|
|
_ => ClientVersion::new(999, 0, "Hot%20Tub".to_string()),
|
|
};
|
|
let requester = requester.get_ref().clone();
|
|
// Ensure "videos" table exists with two string columns.
|
|
match pool.get() {
|
|
Ok(mut conn) => match db::has_table(&mut conn, "videos") {
|
|
Ok(false) => {
|
|
if let Err(e) = db::create_table(
|
|
&mut conn,
|
|
"CREATE TABLE videos (id TEXT NOT NULL, url TEXT NOT NULL);",
|
|
) {
|
|
report_provider_error("db", "videos_post.create_table", &e.to_string()).await;
|
|
}
|
|
}
|
|
Ok(true) => {}
|
|
Err(e) => {
|
|
report_provider_error("db", "videos_post.has_table", &e.to_string()).await;
|
|
}
|
|
},
|
|
Err(e) => {
|
|
report_provider_error("db", "videos_post.pool_get", &e.to_string()).await;
|
|
}
|
|
}
|
|
|
|
let mut videos = Videos {
|
|
pageInfo: PageInfo {
|
|
hasNextPage: true,
|
|
resultsPerPage: 10,
|
|
},
|
|
items: vec![],
|
|
};
|
|
let requested_channel: String = video_request
|
|
.channel
|
|
.as_deref()
|
|
.unwrap_or("all")
|
|
.to_string();
|
|
let channel = resolve_provider_for_build(requested_channel.as_str()).to_string();
|
|
let sort: String = video_request.sort.as_deref().unwrap_or("date").to_string();
|
|
let (query, literal_query) = normalize_query(video_request.query.as_deref());
|
|
let page: u8 = video_request
|
|
.page
|
|
.as_ref()
|
|
.and_then(|value| value.to_u8())
|
|
.unwrap_or(1);
|
|
let perPage: u8 = video_request
|
|
.perPage
|
|
.as_ref()
|
|
.and_then(|value| value.to_u8())
|
|
.unwrap_or(10);
|
|
let featured = video_request
|
|
.featured
|
|
.as_deref()
|
|
.unwrap_or("all")
|
|
.to_string();
|
|
let provider = get_provider(channel.as_str())
|
|
.ok_or_else(|| web::error::ErrorBadRequest("Invalid channel".to_string()))?;
|
|
let category = video_request
|
|
.category
|
|
.as_deref()
|
|
.unwrap_or("all")
|
|
.to_string();
|
|
let sites = if channel == "all" {
|
|
video_request
|
|
.all_provider_sites
|
|
.as_deref()
|
|
.or(video_request.sites.as_deref())
|
|
.unwrap_or("")
|
|
.to_string()
|
|
} else {
|
|
video_request.sites.as_deref().unwrap_or("").to_string()
|
|
};
|
|
let filter = video_request.filter.as_deref().unwrap_or("new").to_string();
|
|
let language = video_request
|
|
.language
|
|
.as_deref()
|
|
.unwrap_or("en")
|
|
.to_string();
|
|
let network = video_request.networks.as_deref().unwrap_or("").to_string();
|
|
let stars = video_request.stars.as_deref().unwrap_or("").to_string();
|
|
let categories = video_request
|
|
.categories
|
|
.as_deref()
|
|
.unwrap_or("")
|
|
.to_string();
|
|
let duration = video_request.duration.as_deref().unwrap_or("").to_string();
|
|
let sexuality = video_request.sexuality.as_deref().unwrap_or("").to_string();
|
|
let public_url_base = format!(
|
|
"{}://{}",
|
|
req.connection_info().scheme(),
|
|
req.connection_info().host()
|
|
);
|
|
crate::flow_debug!(
|
|
"trace={} videos request requested_channel={} resolved_channel={} sort={} query={:?} page={} per_page={} filter={} category={} sites={} client={:?}",
|
|
trace_id,
|
|
&requested_channel,
|
|
&channel,
|
|
&sort,
|
|
&query,
|
|
page,
|
|
perPage,
|
|
&filter,
|
|
&category,
|
|
&sites,
|
|
&clientversion
|
|
);
|
|
let mut requester = requester;
|
|
requester.set_debug_trace_id(Some(trace_id.clone()));
|
|
let options = ServerOptions {
|
|
featured: Some(featured),
|
|
category: Some(category),
|
|
sites: Some(sites),
|
|
filter: Some(filter),
|
|
language: Some(language),
|
|
public_url_base: Some(public_url_base),
|
|
requester: Some(requester),
|
|
network: Some(network),
|
|
stars: Some(stars),
|
|
categories: Some(categories),
|
|
duration: Some(duration),
|
|
sort: Some(sort.clone()),
|
|
sexuality: Some(sexuality),
|
|
};
|
|
|
|
if let Some(query_url) = query.as_deref().and_then(normalize_query_url) {
|
|
crate::flow_debug!(
|
|
"trace={} videos attempting ytdlp url fast path provider={} url={}",
|
|
trace_id,
|
|
&channel,
|
|
crate::util::flow_debug::preview(&query_url, 160)
|
|
);
|
|
if let Some(mut video_items) =
|
|
videos_from_ytdlp_query_url(&channel, &query_url, perPage as usize)
|
|
{
|
|
if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) {
|
|
video_items = video_items
|
|
.into_iter()
|
|
.filter_map(|video| {
|
|
let last_url = video
|
|
.formats
|
|
.as_ref()
|
|
.and_then(|formats| formats.last().map(|f| f.url.clone()));
|
|
if let Some(url) = last_url {
|
|
let mut v = video;
|
|
v.url = url;
|
|
return Some(v);
|
|
}
|
|
Some(video)
|
|
})
|
|
.collect();
|
|
}
|
|
|
|
for video in video_items.iter_mut() {
|
|
if video.duration <= 120 {
|
|
let mut preview_url = video.url.clone();
|
|
if let Some(formats) = &video.formats {
|
|
if let Some(first) = formats.first() {
|
|
preview_url = first.url.clone();
|
|
}
|
|
}
|
|
video.preview = Some(preview_url);
|
|
}
|
|
}
|
|
|
|
videos.pageInfo = PageInfo {
|
|
hasNextPage: false,
|
|
resultsPerPage: perPage as u32,
|
|
};
|
|
videos.items = video_items;
|
|
crate::flow_debug!(
|
|
"trace={} videos ytdlp url fast path returned count={}",
|
|
trace_id,
|
|
videos.items.len()
|
|
);
|
|
return Ok(web::HttpResponse::Ok().json(&videos));
|
|
}
|
|
crate::flow_debug!(
|
|
"trace={} videos ytdlp url fast path fell back to provider",
|
|
trace_id
|
|
);
|
|
}
|
|
|
|
crate::flow_debug!(
|
|
"trace={} videos provider dispatch provider={} literal_query={:?}",
|
|
trace_id,
|
|
&channel,
|
|
&literal_query
|
|
);
|
|
let mut video_items = run_provider_guarded(
|
|
&channel,
|
|
"videos_post.get_videos",
|
|
provider.get_videos(
|
|
cache.get_ref().clone(),
|
|
pool.get_ref().clone(),
|
|
sort.clone(),
|
|
query.clone(),
|
|
page.to_string(),
|
|
perPage.to_string(),
|
|
options.clone(),
|
|
),
|
|
)
|
|
.await;
|
|
crate::flow_debug!(
|
|
"trace={} videos provider returned count={}",
|
|
trace_id,
|
|
video_items.len()
|
|
);
|
|
|
|
// There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine
|
|
if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) {
|
|
// filter out videos without preview for old clients
|
|
video_items = video_items
|
|
.into_iter()
|
|
.filter_map(|video| {
|
|
let last_url = video
|
|
.formats
|
|
.as_ref()
|
|
.and_then(|formats| formats.last().map(|f| f.url.clone()));
|
|
if let Some(url) = last_url {
|
|
let mut v = video;
|
|
v.url = url;
|
|
return Some(v);
|
|
}
|
|
Some(video)
|
|
})
|
|
.collect();
|
|
}
|
|
|
|
if let Some(literal_query) = literal_query.as_deref() {
|
|
#[cfg(feature = "debug")]
|
|
let before = video_items.len();
|
|
video_items.retain(|video| video_matches_literal_query(video, literal_query));
|
|
crate::flow_debug!(
|
|
"trace={} videos literal filter kept={} removed={}",
|
|
trace_id,
|
|
video_items.len(),
|
|
before.saturating_sub(video_items.len())
|
|
);
|
|
}
|
|
|
|
videos.items = video_items.clone();
|
|
if video_items.len() == 0 {
|
|
videos.pageInfo = PageInfo {
|
|
hasNextPage: false,
|
|
resultsPerPage: 10,
|
|
}
|
|
}
|
|
//###
|
|
let next_page = page.to_string().parse::<i32>().unwrap_or(1) + 1;
|
|
let provider_clone = provider.clone();
|
|
let cache_clone = cache.get_ref().clone();
|
|
let pool_clone = pool.get_ref().clone();
|
|
let sort_clone = sort.clone();
|
|
let query_clone = query.clone();
|
|
let per_page_clone = perPage.to_string();
|
|
let options_clone = options.clone();
|
|
let channel_clone = channel.clone();
|
|
#[cfg(feature = "debug")]
|
|
let prefetch_trace_id = trace_id.clone();
|
|
task::spawn_local(async move {
|
|
crate::flow_debug!(
|
|
"trace={} videos prefetch spawn next_page={} provider={}",
|
|
prefetch_trace_id,
|
|
next_page,
|
|
&channel_clone
|
|
);
|
|
// if let AnyProvider::Spankbang(_) = provider_clone {
|
|
// // Spankbang has a delay for the next page
|
|
// ntex::time::sleep(ntex::time::Seconds(80)).await;
|
|
// }
|
|
let _ = run_provider_guarded(
|
|
&channel_clone,
|
|
"videos_post.prefetch_next_page",
|
|
provider_clone.get_videos(
|
|
cache_clone,
|
|
pool_clone,
|
|
sort_clone,
|
|
query_clone,
|
|
next_page.to_string(),
|
|
per_page_clone,
|
|
options_clone,
|
|
),
|
|
)
|
|
.await;
|
|
});
|
|
//###
|
|
|
|
for video in videos.items.iter_mut() {
|
|
if video.duration <= 120 {
|
|
let mut preview_url = video.url.clone();
|
|
if let Some(x) = &video.formats {
|
|
if let Some(first) = x.first() {
|
|
preview_url = first.url.clone();
|
|
}
|
|
}
|
|
video.preview = Some(preview_url);
|
|
}
|
|
}
|
|
|
|
crate::flow_debug!(
|
|
"trace={} videos response items={} has_next={}",
|
|
trace_id,
|
|
videos.items.len(),
|
|
videos.pageInfo.hasNextPage
|
|
);
|
|
Ok(web::HttpResponse::Ok().json(&videos))
|
|
}
|
|
|
|
async fn uploaders_post(
|
|
uploader_request: web::types::Json<UploadersRequest>,
|
|
cache: web::types::State<VideoCache>,
|
|
pool: web::types::State<DbPool>,
|
|
requester: web::types::State<Requester>,
|
|
req: HttpRequest,
|
|
) -> Result<impl web::Responder, web::Error> {
|
|
let trace_id = crate::util::flow_debug::next_trace_id("uploaders");
|
|
let request = uploader_request.into_inner().normalized();
|
|
if !uploader_request_is_valid(&request) {
|
|
return Ok(web::HttpResponse::BadRequest()
|
|
.body("At least one of uploaderId or uploaderName must be provided"));
|
|
}
|
|
|
|
let public_url_base = format!(
|
|
"{}://{}",
|
|
req.connection_info().scheme(),
|
|
req.connection_info().host()
|
|
);
|
|
let mut requester = requester.get_ref().clone();
|
|
requester.set_debug_trace_id(Some(trace_id.clone()));
|
|
let options = ServerOptions {
|
|
featured: None,
|
|
category: None,
|
|
sites: None,
|
|
filter: None,
|
|
language: None,
|
|
public_url_base: Some(public_url_base),
|
|
requester: Some(requester),
|
|
network: None,
|
|
stars: None,
|
|
categories: None,
|
|
duration: None,
|
|
sort: None,
|
|
sexuality: None,
|
|
};
|
|
|
|
crate::flow_debug!(
|
|
"trace={} uploaders request uploader_id={:?} uploader_name={:?} profile_content={} query={:?}",
|
|
trace_id,
|
|
&request.uploaderId,
|
|
&request.uploaderName,
|
|
request.profileContent,
|
|
&request.query
|
|
);
|
|
|
|
if let Some(uploader_id) = request.uploaderId.as_deref() {
|
|
if let Some(provider_id) = provider_hint_from_uploader_id(uploader_id) {
|
|
let Some(provider) = get_provider(&provider_id) else {
|
|
return Ok(web::HttpResponse::NotFound().finish());
|
|
};
|
|
let result = lookup_uploader_with_provider(
|
|
&provider_id,
|
|
provider,
|
|
cache.get_ref().clone(),
|
|
pool.get_ref().clone(),
|
|
&request,
|
|
options,
|
|
)
|
|
.await;
|
|
return match result {
|
|
Ok(Some(profile)) => Ok(web::HttpResponse::Ok().json(&profile)),
|
|
Ok(None) => Ok(web::HttpResponse::NotFound().finish()),
|
|
Err(_error) => {
|
|
crate::flow_debug!(
|
|
"trace={} uploaders targeted provider failed provider={} error={}",
|
|
trace_id,
|
|
&provider_id,
|
|
&_error
|
|
);
|
|
Ok(web::HttpResponse::InternalServerError().finish())
|
|
}
|
|
};
|
|
}
|
|
}
|
|
|
|
let mut matches = Vec::new();
|
|
let mut saw_error = false;
|
|
let requested_name = request
|
|
.uploaderName
|
|
.as_ref()
|
|
.map(|value| value.to_ascii_lowercase());
|
|
|
|
for provider_id in uploader_provider_ids() {
|
|
let Some(provider) = get_provider(&provider_id) else {
|
|
continue;
|
|
};
|
|
let result = lookup_uploader_with_provider(
|
|
&provider_id,
|
|
provider,
|
|
cache.get_ref().clone(),
|
|
pool.get_ref().clone(),
|
|
&request,
|
|
options.clone(),
|
|
)
|
|
.await;
|
|
|
|
match result {
|
|
Ok(Some(profile)) => {
|
|
if let Some(requested_name) = requested_name.as_deref() {
|
|
if profile.name.to_ascii_lowercase() != requested_name {
|
|
crate::flow_debug!(
|
|
"trace={} uploaders ignoring non_exact_match provider={} requested={} returned={}",
|
|
trace_id,
|
|
&provider_id,
|
|
requested_name,
|
|
&profile.name
|
|
);
|
|
continue;
|
|
}
|
|
}
|
|
matches.push(profile);
|
|
}
|
|
Ok(None) => {}
|
|
Err(_error) => {
|
|
saw_error = true;
|
|
crate::flow_debug!(
|
|
"trace={} uploaders provider failed provider={} error={}",
|
|
trace_id,
|
|
&provider_id,
|
|
&_error
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
if matches.is_empty() {
|
|
if saw_error {
|
|
return Ok(web::HttpResponse::InternalServerError().finish());
|
|
}
|
|
return Ok(web::HttpResponse::NotFound().finish());
|
|
}
|
|
|
|
matches.sort_by(|a, b| uploader_match_sort_key(b).cmp(&uploader_match_sort_key(a)));
|
|
Ok(web::HttpResponse::Ok().json(&matches[0]))
|
|
}
|
|
|
|
pub fn get_provider(channel: &str) -> Option<DynProvider> {
|
|
let provider = ALL_PROVIDERS.get(channel).cloned();
|
|
crate::flow_debug!(
|
|
"provider lookup channel={} found={}",
|
|
channel,
|
|
provider.is_some()
|
|
);
|
|
provider
|
|
}
|
|
|
|
pub async fn test() -> Result<impl web::Responder, web::Error> {
|
|
let e = io::Error::new(io::ErrorKind::Other, "test error");
|
|
let _ = send_discord_error_report(
|
|
e.to_string(),
|
|
Some("chain_str".to_string()),
|
|
Some("Context"),
|
|
Some("xtra info"),
|
|
file!(),
|
|
line!(),
|
|
module_path!(),
|
|
)
|
|
.await;
|
|
|
|
Ok(web::HttpResponse::Ok())
|
|
}
|
|
|
|
pub async fn proxies() -> Result<impl web::Responder, web::Error> {
|
|
let proxies = all_proxies_snapshot().await.unwrap_or_default();
|
|
crate::flow_debug!("proxies endpoint snapshot_count={}", proxies.len());
|
|
let mut by_protocol: std::collections::BTreeMap<String, Vec<Proxy>> =
|
|
std::collections::BTreeMap::new();
|
|
for proxy in proxies {
|
|
by_protocol
|
|
.entry(proxy.protocol.clone())
|
|
.or_default()
|
|
.push(proxy);
|
|
}
|
|
for proxies in by_protocol.values_mut() {
|
|
proxies.sort_by(|a, b| {
|
|
a.host
|
|
.cmp(&b.host)
|
|
.then(a.port.cmp(&b.port))
|
|
.then(a.username.cmp(&b.username))
|
|
.then(a.password.cmp(&b.password))
|
|
});
|
|
}
|
|
Ok(web::HttpResponse::Ok().json(&by_protocol))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn uploaders_request_requires_id_or_name() {
|
|
let invalid = UploadersRequest::default();
|
|
let valid = UploadersRequest {
|
|
uploaderName: Some("Example".to_string()),
|
|
..UploadersRequest::default()
|
|
};
|
|
|
|
assert!(!uploader_request_is_valid(&invalid));
|
|
assert!(uploader_request_is_valid(&valid));
|
|
}
|
|
|
|
#[test]
|
|
fn uploader_provider_hint_uses_channel_prefix() {
|
|
assert_eq!(
|
|
provider_hint_from_uploader_id("hsex:xihongshiddd").as_deref(),
|
|
Some("hsex")
|
|
);
|
|
assert_eq!(provider_hint_from_uploader_id("plain-id"), None);
|
|
}
|
|
|
|
#[test]
|
|
fn uploader_match_prefers_higher_video_count() {
|
|
let a = UploaderProfile {
|
|
id: "a".to_string(),
|
|
name: "Example".to_string(),
|
|
channel: Some("alpha".to_string()),
|
|
videoCount: 3,
|
|
..UploaderProfile::default()
|
|
};
|
|
let b = UploaderProfile {
|
|
id: "b".to_string(),
|
|
name: "Example".to_string(),
|
|
channel: Some("beta".to_string()),
|
|
videoCount: 9,
|
|
..UploaderProfile::default()
|
|
};
|
|
|
|
assert!(uploader_match_sort_key(&b) > uploader_match_sort_key(&a));
|
|
}
|
|
|
|
#[test]
|
|
fn detects_http_and_https_query_urls() {
|
|
assert_eq!(
|
|
normalize_query_url(" https://www.freeuseporn.com/video/9579/example "),
|
|
Some("https://www.freeuseporn.com/video/9579/example".to_string())
|
|
);
|
|
assert_eq!(
|
|
normalize_query_url("http://example.com/video"),
|
|
Some("http://example.com/video".to_string())
|
|
);
|
|
assert_eq!(normalize_query_url("Nicole Kitt"), None);
|
|
assert_eq!(normalize_query_url("ftp://example.com/video"), None);
|
|
}
|
|
|
|
#[test]
|
|
fn builds_video_item_from_ytdlp_payload() {
|
|
let payload = serde_json::json!({
|
|
"id": "9579",
|
|
"title": "Nicole Kitt - Example",
|
|
"webpage_url": "https://www.freeuseporn.com/video/9579/nicole-kitt-example",
|
|
"thumbnail": "https://www.freeuseporn.com/media/videos/tmb/9579/1.jpg",
|
|
"duration": 3549,
|
|
"view_count": 52180,
|
|
"uploader": "FreeusePorn",
|
|
"formats": [
|
|
{
|
|
"url": "https://www.freeuseporn.com/media/videos/h264/9579_720p.mp4",
|
|
"format_id": "720p",
|
|
"format_note": "720p",
|
|
"ext": "mp4"
|
|
},
|
|
{
|
|
"url": "https://www.freeuseporn.com/media/videos/h264/9579_480p.mp4",
|
|
"format_id": "480p",
|
|
"ext": "mp4"
|
|
}
|
|
]
|
|
});
|
|
|
|
let item = video_item_from_ytdlp_payload(
|
|
"freeuseporn",
|
|
"https://www.freeuseporn.com/video/9579/nicole-kitt-example",
|
|
&payload,
|
|
)
|
|
.expect("item should parse");
|
|
|
|
assert_eq!(item.id, "9579");
|
|
assert_eq!(item.title, "Nicole Kitt - Example");
|
|
assert_eq!(
|
|
item.url,
|
|
"https://www.freeuseporn.com/video/9579/nicole-kitt-example"
|
|
);
|
|
assert_eq!(item.views, Some(52180));
|
|
assert_eq!(item.uploader.as_deref(), Some("FreeusePorn"));
|
|
assert_eq!(item.formats.as_ref().map(|formats| formats.len()), Some(2));
|
|
}
|
|
}
|