uploaders

This commit is contained in:
Simon
2026-03-31 13:39:11 +00:00
parent 80207efa73
commit bdc7d61121
8 changed files with 913 additions and 4 deletions

View File

@@ -0,0 +1,60 @@
# Uploaders Endpoint Plan
## Summary
Implement `POST /api/uploaders` using the Hot Tub uploader profile contract and ship it framework-first. The server will expose shared uploader request/response types, a provider hook for uploader lookup, endpoint routing in `src/api.rs`, and a first real provider implementation in `hsex`.
## Implementation
- Add dedicated uploader API types in `src/uploaders.rs`:
- `UploadersRequest`
- `UploaderProfile`
- `UploaderChannelStat`
- `UploaderVideoRef`
- `UploaderLayoutRow`
- Keep camelCase as the canonical serialized shape.
- Accept documented decode aliases:
- `uploader_id`
- `uploader_name`
- `profile_content`
- `profile_picture_url`
- `video_ids`
- `horizontal_videos`
- Add `POST /api/uploaders` in `src/api.rs`.
- Validate that at least one of `uploaderId` or `uploaderName` is present.
- Return:
- `400` for invalid request
- `404` for no match
- `500` for provider execution failure
- Add `Provider::get_uploader(...)` with a default `Ok(None)` implementation.
- Add a guarded uploader execution helper in `src/providers/mod.rs`.
- Use canonical uploader IDs in the format `<channel>:<provider-local-id>`.
- Implement the first provider-backed uploader profile in `src/providers/hsex.rs`.
## Hsex Strategy
- Resolve uploader lookup by canonical uploader ID or exact uploader name.
- Reuse existing uploader archive discovery and archive page fetching.
- Build uploader profile metadata from uploader archive pages.
- Populate `videos` with `UploaderVideoRef` values derived from existing `VideoItem`s.
- Always return `layout`.
- When `profileContent == true`, return:
- `videos`
- `tapes: []`
- `playlists: []`
- a `"For You"` horizontal row plus the default videos row
- When `profileContent == false`, return metadata and layout only.
## Tests
- Request alias decoding for uploader request fields.
- Response alias decoding for avatar and layout row compatibility fields.
- Endpoint helper tests for request validation and provider routing.
- Hsex uploader ID generation and uploader page parsing coverage.
## Assumptions
- The first ship focuses on the endpoint framework and one real provider implementation.
- Providers without explicit uploader support remain unsupported by `/api/uploaders`.
- Name-based resolution uses exact display-name matching.
- `videoCount` and `totalViews` are best-effort when the upstream site does not expose authoritative profile totals.

View File

@@ -1,7 +1,9 @@
use crate::providers::{ use crate::providers::{
ALL_PROVIDERS, DynProvider, build_status_response, panic_payload_to_string, ALL_PROVIDERS, DynProvider, build_status_response, panic_payload_to_string,
report_provider_error, resolve_provider_for_build, run_provider_guarded, report_provider_error, resolve_provider_for_build, run_provider_guarded,
run_uploader_provider_guarded,
}; };
use crate::uploaders::{UploaderProfile, UploadersRequest};
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report; use crate::util::discord::send_discord_error_report;
use crate::util::proxy::{Proxy, all_proxies_snapshot}; use crate::util::proxy::{Proxy, all_proxies_snapshot};
@@ -141,10 +143,61 @@ pub fn config(cfg: &mut web::ServiceConfig) {
// .route(web::get().to(videos_get)) // .route(web::get().to(videos_get))
.route(web::post().to(videos_post)), .route(web::post().to(videos_post)),
) )
.service(web::resource("/uploaders").route(web::post().to(uploaders_post)))
.service(web::resource("/test").route(web::get().to(test))) .service(web::resource("/test").route(web::get().to(test)))
.service(web::resource("/proxies").route(web::get().to(proxies))); .service(web::resource("/proxies").route(web::get().to(proxies)));
} }
fn uploader_request_is_valid(request: &UploadersRequest) -> bool {
request.uploaderId.is_some() || request.uploaderName.is_some()
}
fn provider_hint_from_uploader_id(uploader_id: &str) -> Option<String> {
let (channel, _) = uploader_id.split_once(':')?;
Some(resolve_provider_for_build(channel).to_string())
}
fn uploader_provider_ids() -> Vec<String> {
let mut ids = ALL_PROVIDERS
.iter()
.filter_map(|(provider_id, _)| (*provider_id != "all").then(|| (*provider_id).to_string()))
.collect::<Vec<_>>();
ids.sort();
ids
}
fn uploader_match_sort_key(profile: &UploaderProfile) -> (u64, String, String) {
(
profile.videoCount,
profile.channel.clone().unwrap_or_default(),
profile.id.clone(),
)
}
async fn lookup_uploader_with_provider(
provider_id: &str,
provider: DynProvider,
cache: VideoCache,
pool: DbPool,
request: &UploadersRequest,
options: crate::videos::ServerOptions,
) -> Result<Option<UploaderProfile>, String> {
run_uploader_provider_guarded(
provider_id,
"uploaders_post.get_uploader",
provider.get_uploader(
cache,
pool,
request.uploaderId.clone(),
request.uploaderName.clone(),
request.query.clone(),
request.profileContent,
options,
),
)
.await
}
async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> { async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
#[cfg(feature = "debug")] #[cfg(feature = "debug")]
let trace_id = crate::util::flow_debug::next_trace_id("status"); let trace_id = crate::util::flow_debug::next_trace_id("status");
@@ -491,6 +544,144 @@ async fn videos_post(
Ok(web::HttpResponse::Ok().json(&videos)) Ok(web::HttpResponse::Ok().json(&videos))
} }
async fn uploaders_post(
uploader_request: web::types::Json<UploadersRequest>,
cache: web::types::State<VideoCache>,
pool: web::types::State<DbPool>,
requester: web::types::State<Requester>,
req: HttpRequest,
) -> Result<impl web::Responder, web::Error> {
let trace_id = crate::util::flow_debug::next_trace_id("uploaders");
let request = uploader_request.into_inner().normalized();
if !uploader_request_is_valid(&request) {
return Ok(web::HttpResponse::BadRequest().body(
"At least one of uploaderId or uploaderName must be provided",
));
}
let public_url_base = format!(
"{}://{}",
req.connection_info().scheme(),
req.connection_info().host()
);
let mut requester = requester.get_ref().clone();
requester.set_debug_trace_id(Some(trace_id.clone()));
let options = ServerOptions {
featured: None,
category: None,
sites: None,
filter: None,
language: None,
public_url_base: Some(public_url_base),
requester: Some(requester),
network: None,
stars: None,
categories: None,
duration: None,
sort: None,
sexuality: None,
};
crate::flow_debug!(
"trace={} uploaders request uploader_id={:?} uploader_name={:?} profile_content={} query={:?}",
trace_id,
&request.uploaderId,
&request.uploaderName,
request.profileContent,
&request.query
);
if let Some(uploader_id) = request.uploaderId.as_deref() {
if let Some(provider_id) = provider_hint_from_uploader_id(uploader_id) {
let Some(provider) = get_provider(&provider_id) else {
return Ok(web::HttpResponse::NotFound().finish());
};
let result = lookup_uploader_with_provider(
&provider_id,
provider,
cache.get_ref().clone(),
pool.get_ref().clone(),
&request,
options,
)
.await;
return match result {
Ok(Some(profile)) => Ok(web::HttpResponse::Ok().json(&profile)),
Ok(None) => Ok(web::HttpResponse::NotFound().finish()),
Err(_error) => {
crate::flow_debug!(
"trace={} uploaders targeted provider failed provider={} error={}",
trace_id,
&provider_id,
&_error
);
Ok(web::HttpResponse::InternalServerError().finish())
}
};
}
}
let mut matches = Vec::new();
let mut saw_error = false;
let requested_name = request
.uploaderName
.as_ref()
.map(|value| value.to_ascii_lowercase());
for provider_id in uploader_provider_ids() {
let Some(provider) = get_provider(&provider_id) else {
continue;
};
let result = lookup_uploader_with_provider(
&provider_id,
provider,
cache.get_ref().clone(),
pool.get_ref().clone(),
&request,
options.clone(),
)
.await;
match result {
Ok(Some(profile)) => {
if let Some(requested_name) = requested_name.as_deref() {
if profile.name.to_ascii_lowercase() != requested_name {
crate::flow_debug!(
"trace={} uploaders ignoring non_exact_match provider={} requested={} returned={}",
trace_id,
&provider_id,
requested_name,
&profile.name
);
continue;
}
}
matches.push(profile);
}
Ok(None) => {}
Err(_error) => {
saw_error = true;
crate::flow_debug!(
"trace={} uploaders provider failed provider={} error={}",
trace_id,
&provider_id,
&_error
);
}
}
}
if matches.is_empty() {
if saw_error {
return Ok(web::HttpResponse::InternalServerError().finish());
}
return Ok(web::HttpResponse::NotFound().finish());
}
matches.sort_by(|a, b| uploader_match_sort_key(b).cmp(&uploader_match_sort_key(a)));
Ok(web::HttpResponse::Ok().json(&matches[0]))
}
pub fn get_provider(channel: &str) -> Option<DynProvider> { pub fn get_provider(channel: &str) -> Option<DynProvider> {
let provider = ALL_PROVIDERS.get(channel).cloned(); let provider = ALL_PROVIDERS.get(channel).cloned();
crate::flow_debug!( crate::flow_debug!(
@@ -539,3 +730,49 @@ pub async fn proxies() -> Result<impl web::Responder, web::Error> {
} }
Ok(web::HttpResponse::Ok().json(&by_protocol)) Ok(web::HttpResponse::Ok().json(&by_protocol))
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn uploaders_request_requires_id_or_name() {
let invalid = UploadersRequest::default();
let valid = UploadersRequest {
uploaderName: Some("Example".to_string()),
..UploadersRequest::default()
};
assert!(!uploader_request_is_valid(&invalid));
assert!(uploader_request_is_valid(&valid));
}
#[test]
fn uploader_provider_hint_uses_channel_prefix() {
assert_eq!(
provider_hint_from_uploader_id("hsex:xihongshiddd").as_deref(),
Some("hsex")
);
assert_eq!(provider_hint_from_uploader_id("plain-id"), None);
}
#[test]
fn uploader_match_prefers_higher_video_count() {
let a = UploaderProfile {
id: "a".to_string(),
name: "Example".to_string(),
channel: Some("alpha".to_string()),
videoCount: 3,
..UploaderProfile::default()
};
let b = UploaderProfile {
id: "b".to_string(),
name: "Example".to_string(),
channel: Some("beta".to_string()),
videoCount: 9,
..UploaderProfile::default()
};
assert!(uploader_match_sort_key(&b) > uploader_match_sort_key(&a));
}
}

View File

@@ -19,6 +19,7 @@ mod proxies;
mod proxy; mod proxy;
mod schema; mod schema;
mod status; mod status;
mod uploaders;
mod util; mod util;
mod videos; mod videos;

View File

@@ -4,6 +4,9 @@ use crate::providers::{
Provider, report_provider_error, report_provider_error_background, requester_or_default, Provider, report_provider_error, report_provider_error_background, requester_or_default,
}; };
use crate::status::*; use crate::status::*;
use crate::uploaders::{
UploaderChannelStat, UploaderLayoutRow, UploaderProfile, UploaderVideoRef,
};
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number; use crate::util::parse_abbreviated_number;
use crate::util::requester::Requester; use crate::util::requester::Requester;
@@ -14,6 +17,7 @@ use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, Utc};
use error_chain::error_chain; use error_chain::error_chain;
use futures::stream::{self, StreamExt}; use futures::stream::{self, StreamExt};
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use percent_encoding::{NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
use regex::Regex; use regex::Regex;
use scraper::{ElementRef, Html, Selector}; use scraper::{ElementRef, Html, Selector};
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
@@ -507,6 +511,263 @@ impl HsexProvider {
} }
} }
fn canonical_uploader_id(author: &str) -> String {
format!(
"{CHANNEL_ID}:{}",
utf8_percent_encode(author, NON_ALPHANUMERIC)
)
}
fn author_from_uploader_id(value: &str) -> Option<String> {
let suffix = match value.split_once(':') {
Some((channel, suffix)) if channel.eq_ignore_ascii_case(CHANNEL_ID) => suffix,
Some(_) => return None,
None => value,
};
percent_decode_str(suffix)
.decode_utf8()
.ok()
.map(|value| value.into_owned())
.and_then(|value| (!value.trim().is_empty()).then_some(value))
}
fn author_from_uploader_href(&self, href: &str) -> Option<String> {
let url = Url::parse(&self.absolute_url(href)).ok()?;
url.query_pairs()
.find(|(key, _)| key == "author")
.map(|(_, value)| value.to_string())
}
fn pagination_last_page(html: &str) -> Option<u16> {
let regex = Regex::new(r#"user-(?P<page>\d+)\.htm\?author="#).ok()?;
regex
.captures_iter(html)
.filter_map(|captures| captures.name("page")?.as_str().parse::<u16>().ok())
.max()
}
fn uploader_option_by_name(&self, uploader_name: &str) -> Option<FilterOption> {
let normalized = uploader_name.trim();
if normalized.is_empty() {
return None;
}
let lowered = normalized.to_lowercase();
self.uploaders
.read()
.ok()?
.iter()
.find(|option| {
option.title == normalized
|| option.title.to_lowercase() == lowered
|| option.id.eq_ignore_ascii_case(normalized)
})
.cloned()
}
fn resolve_uploader_author(
&self,
uploader_id: Option<&str>,
uploader_name: Option<&str>,
) -> Option<String> {
if let Some(uploader_id) = uploader_id {
if let Some(author) = Self::author_from_uploader_id(uploader_id) {
return Some(author);
}
}
if let Some(uploader_name) = uploader_name {
if let Some(option) = self.uploader_option_by_name(uploader_name) {
if let Some(Target::Uploader { author }) = self.target_from_filter_id(&option.id) {
return Some(author);
}
}
let trimmed = uploader_name.trim();
if !trimmed.is_empty() {
return Some(trimmed.to_string());
}
}
None
}
fn display_name_for_uploader(
&self,
author: &str,
requested_name: Option<&str>,
first_page_items: &[VideoItem],
) -> String {
if let Some(requested_name) = requested_name {
let trimmed = requested_name.trim();
if !trimmed.is_empty() {
return trimmed.to_string();
}
}
if let Some(name) = first_page_items
.iter()
.find_map(|item| item.uploader.as_deref())
.filter(|value| !value.trim().is_empty())
{
return name.to_string();
}
if let Some(option) = self
.uploaders
.read()
.ok()
.and_then(|values| {
values
.iter()
.find(|value| value.id.contains(author) || value.title == author)
.cloned()
})
{
return option.title;
}
author.to_string()
}
fn rank_videos_for_query(
videos: &[UploaderVideoRef],
query: Option<&str>,
) -> Vec<UploaderVideoRef> {
let Some(query) = query.map(|value| value.trim()).filter(|value| !value.is_empty()) else {
return videos.to_vec();
};
let query = query.to_lowercase();
let mut ranked = videos.to_vec();
ranked.sort_by(|a, b| {
let score = |video: &UploaderVideoRef| {
let mut score = 0u8;
if video.title.to_lowercase().contains(&query) {
score += 2;
}
if video.uploader.to_lowercase().contains(&query) {
score += 1;
}
score
};
score(b)
.cmp(&score(a))
.then(b.views.cmp(&a.views))
.then_with(|| a.id.cmp(&b.id))
});
ranked
}
async fn build_uploader_profile(
&self,
cache: VideoCache,
author: &str,
requested_name: Option<&str>,
query: Option<&str>,
profile_content: bool,
options: &ServerOptions,
) -> Result<Option<UploaderProfile>> {
let first_page_url = self.build_uploader_url(author, 1);
let first_page_items = self
.fetch_items_for_url(
cache.clone(),
first_page_url.clone(),
64,
profile_content,
options,
)
.await?;
if first_page_items.is_empty() {
return Ok(None);
}
let mut requester = requester_or_default(options, CHANNEL_ID, "get_uploader.profile_page");
let first_page_html = self
.fetch_html(&mut requester, &first_page_url, &format!("{}/", self.url))
.await?;
let last_page = Self::pagination_last_page(&first_page_html).unwrap_or(1);
let first_page_size = first_page_items.len() as u64;
let last_page_items = if last_page > 1 {
self.fetch_items_for_url(
cache,
self.build_uploader_url(author, last_page),
64,
false,
options,
)
.await
.unwrap_or_default()
} else {
Vec::new()
};
let display_name = self.display_name_for_uploader(author, requested_name, &first_page_items);
let canonical_id = Self::canonical_uploader_id(author);
let mut videos = first_page_items
.iter()
.map(|item| UploaderVideoRef::from_video_item(item, &display_name, &canonical_id))
.collect::<Vec<_>>();
let ranked_videos = Self::rank_videos_for_query(&videos, query);
let horizontal_ids = ranked_videos
.iter()
.take(12)
.map(|video| video.id.clone())
.collect::<Vec<_>>();
let newest_seen = first_page_items
.iter()
.filter_map(|item| item.uploadedAt)
.max();
let oldest_seen = last_page_items
.iter()
.filter_map(|item| item.uploadedAt)
.min()
.or_else(|| first_page_items.iter().filter_map(|item| item.uploadedAt).min());
let video_count = if last_page > 1 {
((last_page as u64 - 1) * first_page_size) + last_page_items.len() as u64
} else {
first_page_size
};
let total_views = first_page_items
.iter()
.chain(last_page_items.iter())
.filter_map(|item| item.views)
.map(u64::from)
.sum();
for item in &mut videos {
item.uploader = display_name.clone();
item.uploaderId = canonical_id.clone();
}
let layout = if horizontal_ids.is_empty() {
vec![UploaderLayoutRow::videos(None)]
} else {
vec![
UploaderLayoutRow::horizontal(Some("For You".to_string()), horizontal_ids),
UploaderLayoutRow::videos(None),
]
};
Ok(Some(UploaderProfile {
id: canonical_id,
name: display_name,
url: Some(first_page_url),
channel: Some(CHANNEL_ID.to_string()),
verified: false,
videoCount: video_count,
totalViews: total_views,
channels: Some(vec![UploaderChannelStat {
channel: CHANNEL_ID.to_string(),
videoCount: video_count,
firstSeenAt: crate::uploaders::iso_timestamp_from_unix(oldest_seen),
lastSeenAt: crate::uploaders::iso_timestamp_from_unix(newest_seen),
}]),
avatar: None,
description: None,
bio: None,
videos: profile_content.then_some(videos),
tapes: profile_content.then_some(Vec::new()),
playlists: profile_content.then_some(Vec::new()),
layout: Some(layout),
}))
}
fn first_video_link<'a>(&self, element: &'a ElementRef<'a>) -> Result<Option<ElementRef<'a>>> { fn first_video_link<'a>(&self, element: &'a ElementRef<'a>) -> Result<Option<ElementRef<'a>>> {
let selector = Self::selector("a[href]")?; let selector = Self::selector("a[href]")?;
Ok(element.select(&selector).find(|link| { Ok(element.select(&selector).find(|link| {
@@ -638,6 +899,9 @@ impl HsexProvider {
} }
if let Some(uploader_href) = uploader.value().attr("href") { if let Some(uploader_href) = uploader.value().attr("href") {
item.uploaderUrl = Some(self.absolute_url(uploader_href)); item.uploaderUrl = Some(self.absolute_url(uploader_href));
item.uploaderId = self
.author_from_uploader_href(uploader_href)
.map(|author| Self::canonical_uploader_id(&author));
} }
} }
@@ -701,6 +965,9 @@ impl HsexProvider {
} }
if let Some(href) = author.value().attr("href") { if let Some(href) = author.value().attr("href") {
item.uploaderUrl = Some(self.absolute_url(href)); item.uploaderUrl = Some(self.absolute_url(href));
item.uploaderId = self
.author_from_uploader_href(href)
.map(|author| Self::canonical_uploader_id(&author));
} }
} }
} }
@@ -934,6 +1201,34 @@ impl Provider for HsexProvider {
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> { fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion)) Some(self.build_channel(clientversion))
} }
async fn get_uploader(
&self,
cache: VideoCache,
pool: DbPool,
uploader_id: Option<String>,
uploader_name: Option<String>,
query: Option<String>,
profile_content: bool,
options: ServerOptions,
) -> std::result::Result<Option<UploaderProfile>, String> {
let _ = pool;
let Some(author) =
self.resolve_uploader_author(uploader_id.as_deref(), uploader_name.as_deref())
else {
return Ok(None);
};
self.build_uploader_profile(
cache,
&author,
uploader_name.as_deref(),
query.as_deref(),
profile_content,
&options,
)
.await
.map_err(|error| error.to_string())
}
} }
#[cfg(test)] #[cfg(test)]
@@ -1020,6 +1315,29 @@ mod tests {
); );
} }
#[test]
fn canonical_uploader_id_round_trips() {
let canonical = HsexProvider::canonical_uploader_id("xihongshiddd");
assert_eq!(canonical, "hsex:xihongshiddd");
assert_eq!(
HsexProvider::author_from_uploader_id(&canonical).as_deref(),
Some("xihongshiddd")
);
}
#[test]
fn parses_last_page_from_pagination() {
let html = r#"
<ul class="pagination1">
<li><a href="user-1.htm?author=xihongshiddd">1</a></li>
<li><a href="user-2.htm?author=xihongshiddd">2</a></li>
<li><a href="user-7.htm?author=xihongshiddd">7</a></li>
</ul>
"#;
assert_eq!(HsexProvider::pagination_last_page(html), Some(7));
}
#[tokio::test] #[tokio::test]
#[ignore] #[ignore]
async fn fetches_page_two_items() { async fn fetches_page_two_items() {

View File

@@ -12,6 +12,7 @@ use crate::{
DbPool, DbPool,
api::ClientVersion, api::ClientVersion,
status::{Channel, ChannelGroup, ChannelView, FilterOption, Status, StatusResponse}, status::{Channel, ChannelGroup, ChannelView, FilterOption, Status, StatusResponse},
uploaders::UploaderProfile,
util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester}, util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester},
videos::{FlexibleNumber, ServerOptions, VideoItem, VideosRequest}, videos::{FlexibleNumber, ServerOptions, VideoItem, VideosRequest},
}; };
@@ -577,6 +578,53 @@ where
} }
} }
pub async fn run_uploader_provider_guarded<F>(
provider_name: &str,
context: &str,
fut: F,
) -> Result<Option<UploaderProfile>, String>
where
F: Future<Output = Result<Option<UploaderProfile>, String>>,
{
crate::flow_debug!(
"provider uploader guard enter provider={} context={}",
provider_name,
context
);
match AssertUnwindSafe(fut).catch_unwind().await {
Ok(result) => {
crate::flow_debug!(
"provider uploader guard exit provider={} context={} matched={}",
provider_name,
context,
result.as_ref().ok().and_then(|value| value.as_ref()).is_some()
);
result
}
Err(payload) => {
let panic_msg = panic_payload_to_string(payload);
crate::flow_debug!(
"provider uploader guard panic provider={} context={} panic={}",
provider_name,
context,
&panic_msg
);
let _ = send_discord_error_report(
format!("Provider panic: {}", provider_name),
None,
Some("Provider Guard"),
Some(&format!("context={}; panic={}", context, panic_msg)),
file!(),
line!(),
module_path!(),
)
.await;
schedule_provider_validation(provider_name, context, &panic_msg);
Err(panic_msg)
}
}
}
pub async fn report_provider_error(provider_name: &str, context: &str, msg: &str) { pub async fn report_provider_error(provider_name: &str, context: &str, msg: &str) {
let _ = send_discord_error_report( let _ = send_discord_error_report(
format!("Provider error: {}", provider_name), format!("Provider error: {}", provider_name),
@@ -868,6 +916,19 @@ pub trait Provider: Send + Sync {
cacheDuration: None, cacheDuration: None,
}) })
} }
async fn get_uploader(
&self,
_cache: VideoCache,
_pool: DbPool,
_uploader_id: Option<String>,
_uploader_name: Option<String>,
_query: Option<String>,
_profile_content: bool,
_options: ServerOptions,
) -> Result<Option<UploaderProfile>, String> {
Ok(None)
}
} }
#[cfg(all(test, not(hottub_single_provider)))] #[cfg(all(test, not(hottub_single_provider)))]

View File

@@ -474,6 +474,19 @@ impl NoodlemagazineProvider {
if normalized.is_empty() || !self.is_allowed_thumb_url(&normalized) { if normalized.is_empty() || !self.is_allowed_thumb_url(&normalized) {
return String::new(); return String::new();
} }
let Some(url) = Url::parse(&normalized).ok() else {
return String::new();
};
if url
.host_str()
.is_some_and(|host| host.eq_ignore_ascii_case("img.pvvstream.pro"))
{
return crate::providers::build_proxy_url(
_options,
"noodlemagazine-thumb",
&crate::providers::strip_url_scheme(&normalized),
);
}
normalized normalized
} }
@@ -707,7 +720,7 @@ mod tests {
assert_eq!(items.len(), 1); assert_eq!(items.len(), 1);
assert_eq!( assert_eq!(
items[0].thumb, items[0].thumb,
"https://img.pvvstream.pro/preview/abc/-111_222/240/iv.okcdn.ru/getVideoPreview?id=1&type=39&fn=vid_l" "https://example.com/proxy/noodlemagazine-thumb/img.pvvstream.pro/preview/abc/-111_222/240/iv.okcdn.ru/getVideoPreview?id=1&type=39&fn=vid_l"
); );
} }

216
src/uploaders.rs Normal file
View File

@@ -0,0 +1,216 @@
use chrono::{SecondsFormat, TimeZone, Utc};
use serde::{Deserialize, Serialize};
use crate::videos::VideoItem;
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct UploadersRequest {
#[serde(default, alias = "uploader_id")]
pub uploaderId: Option<String>,
#[serde(default, alias = "uploader_name")]
pub uploaderName: Option<String>,
#[serde(default, alias = "profile_content")]
pub profileContent: bool,
#[serde(default)]
pub query: Option<String>,
}
impl UploadersRequest {
pub fn normalized(self) -> Self {
Self {
uploaderId: normalize_optional_string(self.uploaderId),
uploaderName: normalize_optional_string(self.uploaderName),
profileContent: self.profileContent,
query: normalize_optional_string(self.query),
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct UploaderProfile {
pub id: String,
pub name: String,
pub url: Option<String>,
pub channel: Option<String>,
pub verified: bool,
pub videoCount: u64,
pub totalViews: u64,
#[serde(default)]
pub channels: Option<Vec<UploaderChannelStat>>,
#[serde(default, alias = "profile_picture_url")]
pub avatar: Option<String>,
pub description: Option<String>,
pub bio: Option<String>,
#[serde(default)]
pub videos: Option<Vec<UploaderVideoRef>>,
#[serde(default)]
pub tapes: Option<Vec<serde_json::Value>>,
#[serde(default)]
pub playlists: Option<Vec<serde_json::Value>>,
#[serde(default)]
pub layout: Option<Vec<UploaderLayoutRow>>,
}
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct UploaderChannelStat {
pub channel: String,
pub videoCount: u64,
#[serde(default, alias = "first_seen_at")]
pub firstSeenAt: Option<String>,
#[serde(default, alias = "last_seen_at")]
pub lastSeenAt: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct UploaderVideoRef {
pub id: String,
pub url: String,
pub title: String,
pub duration: u32,
pub channel: String,
#[serde(default, alias = "uploaded_at")]
pub uploadedAt: Option<String>,
pub uploader: String,
#[serde(alias = "uploader_id")]
pub uploaderId: String,
pub thumb: String,
pub preview: Option<String>,
pub views: u32,
pub rating: u32,
#[serde(default, alias = "aspect_ratio")]
pub aspectRatio: Option<f32>,
}
impl UploaderVideoRef {
pub fn from_video_item(item: &VideoItem, uploader_name: &str, uploader_id: &str) -> Self {
Self {
id: item.id.clone(),
url: item.url.clone(),
title: item.title.clone(),
duration: item.duration,
channel: item.channel.clone(),
uploadedAt: iso_timestamp_from_unix(item.uploadedAt),
uploader: item
.uploader
.clone()
.unwrap_or_else(|| uploader_name.to_string()),
uploaderId: item
.uploaderId
.clone()
.unwrap_or_else(|| uploader_id.to_string()),
thumb: item.thumb.clone(),
preview: item.preview.clone(),
views: item.views.unwrap_or_default(),
rating: item.rating.map(normalize_rating).unwrap_or_default(),
aspectRatio: item.aspectRatio,
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct UploaderLayoutRow {
#[serde(rename = "type")]
pub rowType: UploaderLayoutRowType,
pub title: Option<String>,
#[serde(default, alias = "video_ids")]
pub videoIds: Option<Vec<String>>,
}
impl UploaderLayoutRow {
pub fn horizontal(title: Option<String>, video_ids: Vec<String>) -> Self {
Self {
rowType: UploaderLayoutRowType::Horizontal,
title,
videoIds: Some(video_ids),
}
}
pub fn videos(title: Option<String>) -> Self {
Self {
rowType: UploaderLayoutRowType::Videos,
title,
videoIds: None,
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
pub enum UploaderLayoutRowType {
#[default]
#[serde(rename = "videos")]
Videos,
#[serde(rename = "horizontal", alias = "horizontal_videos")]
Horizontal,
}
pub fn normalize_optional_string(value: Option<String>) -> Option<String> {
value.and_then(|value| {
let trimmed = value.trim();
(!trimmed.is_empty()).then(|| trimmed.to_string())
})
}
pub fn iso_timestamp_from_unix(value: Option<u64>) -> Option<String> {
let timestamp = value?;
let dt = Utc.timestamp_opt(timestamp as i64, 0).single()?;
Some(dt.to_rfc3339_opts(SecondsFormat::Millis, true))
}
fn normalize_rating(value: f32) -> u32 {
value.clamp(0.0, 100.0).round() as u32
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn request_accepts_snake_case_aliases() {
let request: UploadersRequest = serde_json::from_str(
r#"{
"uploader_id": "hsex:xihongshiddd",
"uploader_name": "xihongshiddd",
"profile_content": true,
"query": "teacher"
}"#,
)
.expect("request should decode");
assert_eq!(request.uploaderId.as_deref(), Some("hsex:xihongshiddd"));
assert_eq!(request.uploaderName.as_deref(), Some("xihongshiddd"));
assert!(request.profileContent);
assert_eq!(request.query.as_deref(), Some("teacher"));
}
#[test]
fn layout_aliases_decode() {
let row: UploaderLayoutRow = serde_json::from_str(
r#"{
"type": "horizontal_videos",
"title": "For You",
"video_ids": ["one", "two"]
}"#,
)
.expect("row should decode");
assert_eq!(row.rowType, UploaderLayoutRowType::Horizontal);
assert_eq!(row.videoIds.as_ref().map(Vec::len), Some(2));
}
#[test]
fn avatar_alias_decodes() {
let profile: UploaderProfile = serde_json::from_str(
r#"{
"id": "abc",
"name": "Example",
"verified": false,
"videoCount": 1,
"totalViews": 2,
"profile_picture_url": "https://example.com/a.jpg"
}"#,
)
.expect("profile should decode");
assert_eq!(profile.avatar.as_deref(), Some("https://example.com/a.jpg"));
}
}

View File

@@ -100,6 +100,8 @@ pub struct VideoItem {
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub uploaderUrl: Option<String>, // "https://www.youtube.com/@petcollective", pub uploaderUrl: Option<String>, // "https://www.youtube.com/@petcollective",
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub uploaderId: Option<String>, // "petcollective",
#[serde(skip_serializing_if = "Option::is_none")]
pub verified: Option<bool>, // false, pub verified: Option<bool>, // false,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub tags: Option<Vec<String>>, // [], pub tags: Option<Vec<String>>, // [],
@@ -135,6 +137,7 @@ impl VideoItem {
thumb, thumb,
uploader: None, uploader: None,
uploaderUrl: None, uploaderUrl: None,
uploaderId: None,
verified: None, verified: None,
tags: None, // Placeholder, adjust as needed tags: None, // Placeholder, adjust as needed
uploadedAt: None, uploadedAt: None,