uploaders
This commit is contained in:
60
docs/uploaders-endpoint-plan.md
Normal file
60
docs/uploaders-endpoint-plan.md
Normal file
@@ -0,0 +1,60 @@
|
||||
# Uploaders Endpoint Plan
|
||||
|
||||
## Summary
|
||||
|
||||
Implement `POST /api/uploaders` using the Hot Tub uploader profile contract and ship it framework-first. The server will expose shared uploader request/response types, a provider hook for uploader lookup, endpoint routing in `src/api.rs`, and a first real provider implementation in `hsex`.
|
||||
|
||||
## Implementation
|
||||
|
||||
- Add dedicated uploader API types in `src/uploaders.rs`:
|
||||
- `UploadersRequest`
|
||||
- `UploaderProfile`
|
||||
- `UploaderChannelStat`
|
||||
- `UploaderVideoRef`
|
||||
- `UploaderLayoutRow`
|
||||
- Keep camelCase as the canonical serialized shape.
|
||||
- Accept documented decode aliases:
|
||||
- `uploader_id`
|
||||
- `uploader_name`
|
||||
- `profile_content`
|
||||
- `profile_picture_url`
|
||||
- `video_ids`
|
||||
- `horizontal_videos`
|
||||
- Add `POST /api/uploaders` in `src/api.rs`.
|
||||
- Validate that at least one of `uploaderId` or `uploaderName` is present.
|
||||
- Return:
|
||||
- `400` for invalid request
|
||||
- `404` for no match
|
||||
- `500` for provider execution failure
|
||||
- Add `Provider::get_uploader(...)` with a default `Ok(None)` implementation.
|
||||
- Add a guarded uploader execution helper in `src/providers/mod.rs`.
|
||||
- Use canonical uploader IDs in the format `<channel>:<provider-local-id>`.
|
||||
- Implement the first provider-backed uploader profile in `src/providers/hsex.rs`.
|
||||
|
||||
## Hsex Strategy
|
||||
|
||||
- Resolve uploader lookup by canonical uploader ID or exact uploader name.
|
||||
- Reuse existing uploader archive discovery and archive page fetching.
|
||||
- Build uploader profile metadata from uploader archive pages.
|
||||
- Populate `videos` with `UploaderVideoRef` values derived from existing `VideoItem`s.
|
||||
- Always return `layout`.
|
||||
- When `profileContent == true`, return:
|
||||
- `videos`
|
||||
- `tapes: []`
|
||||
- `playlists: []`
|
||||
- a `"For You"` horizontal row plus the default videos row
|
||||
- When `profileContent == false`, return metadata and layout only.
|
||||
|
||||
## Tests
|
||||
|
||||
- Request alias decoding for uploader request fields.
|
||||
- Response alias decoding for avatar and layout row compatibility fields.
|
||||
- Endpoint helper tests for request validation and provider routing.
|
||||
- Hsex uploader ID generation and uploader page parsing coverage.
|
||||
|
||||
## Assumptions
|
||||
|
||||
- The first ship focuses on the endpoint framework and one real provider implementation.
|
||||
- Providers without explicit uploader support remain unsupported by `/api/uploaders`.
|
||||
- Name-based resolution uses exact display-name matching.
|
||||
- `videoCount` and `totalViews` are best-effort when the upstream site does not expose authoritative profile totals.
|
||||
237
src/api.rs
237
src/api.rs
@@ -1,7 +1,9 @@
|
||||
use crate::providers::{
|
||||
ALL_PROVIDERS, DynProvider, build_status_response, panic_payload_to_string,
|
||||
report_provider_error, resolve_provider_for_build, run_provider_guarded,
|
||||
run_uploader_provider_guarded,
|
||||
};
|
||||
use crate::uploaders::{UploaderProfile, UploadersRequest};
|
||||
use crate::util::cache::VideoCache;
|
||||
use crate::util::discord::send_discord_error_report;
|
||||
use crate::util::proxy::{Proxy, all_proxies_snapshot};
|
||||
@@ -141,10 +143,61 @@ pub fn config(cfg: &mut web::ServiceConfig) {
|
||||
// .route(web::get().to(videos_get))
|
||||
.route(web::post().to(videos_post)),
|
||||
)
|
||||
.service(web::resource("/uploaders").route(web::post().to(uploaders_post)))
|
||||
.service(web::resource("/test").route(web::get().to(test)))
|
||||
.service(web::resource("/proxies").route(web::get().to(proxies)));
|
||||
}
|
||||
|
||||
fn uploader_request_is_valid(request: &UploadersRequest) -> bool {
|
||||
request.uploaderId.is_some() || request.uploaderName.is_some()
|
||||
}
|
||||
|
||||
fn provider_hint_from_uploader_id(uploader_id: &str) -> Option<String> {
|
||||
let (channel, _) = uploader_id.split_once(':')?;
|
||||
Some(resolve_provider_for_build(channel).to_string())
|
||||
}
|
||||
|
||||
fn uploader_provider_ids() -> Vec<String> {
|
||||
let mut ids = ALL_PROVIDERS
|
||||
.iter()
|
||||
.filter_map(|(provider_id, _)| (*provider_id != "all").then(|| (*provider_id).to_string()))
|
||||
.collect::<Vec<_>>();
|
||||
ids.sort();
|
||||
ids
|
||||
}
|
||||
|
||||
fn uploader_match_sort_key(profile: &UploaderProfile) -> (u64, String, String) {
|
||||
(
|
||||
profile.videoCount,
|
||||
profile.channel.clone().unwrap_or_default(),
|
||||
profile.id.clone(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn lookup_uploader_with_provider(
|
||||
provider_id: &str,
|
||||
provider: DynProvider,
|
||||
cache: VideoCache,
|
||||
pool: DbPool,
|
||||
request: &UploadersRequest,
|
||||
options: crate::videos::ServerOptions,
|
||||
) -> Result<Option<UploaderProfile>, String> {
|
||||
run_uploader_provider_guarded(
|
||||
provider_id,
|
||||
"uploaders_post.get_uploader",
|
||||
provider.get_uploader(
|
||||
cache,
|
||||
pool,
|
||||
request.uploaderId.clone(),
|
||||
request.uploaderName.clone(),
|
||||
request.query.clone(),
|
||||
request.profileContent,
|
||||
options,
|
||||
),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
|
||||
#[cfg(feature = "debug")]
|
||||
let trace_id = crate::util::flow_debug::next_trace_id("status");
|
||||
@@ -491,6 +544,144 @@ async fn videos_post(
|
||||
Ok(web::HttpResponse::Ok().json(&videos))
|
||||
}
|
||||
|
||||
async fn uploaders_post(
|
||||
uploader_request: web::types::Json<UploadersRequest>,
|
||||
cache: web::types::State<VideoCache>,
|
||||
pool: web::types::State<DbPool>,
|
||||
requester: web::types::State<Requester>,
|
||||
req: HttpRequest,
|
||||
) -> Result<impl web::Responder, web::Error> {
|
||||
let trace_id = crate::util::flow_debug::next_trace_id("uploaders");
|
||||
let request = uploader_request.into_inner().normalized();
|
||||
if !uploader_request_is_valid(&request) {
|
||||
return Ok(web::HttpResponse::BadRequest().body(
|
||||
"At least one of uploaderId or uploaderName must be provided",
|
||||
));
|
||||
}
|
||||
|
||||
let public_url_base = format!(
|
||||
"{}://{}",
|
||||
req.connection_info().scheme(),
|
||||
req.connection_info().host()
|
||||
);
|
||||
let mut requester = requester.get_ref().clone();
|
||||
requester.set_debug_trace_id(Some(trace_id.clone()));
|
||||
let options = ServerOptions {
|
||||
featured: None,
|
||||
category: None,
|
||||
sites: None,
|
||||
filter: None,
|
||||
language: None,
|
||||
public_url_base: Some(public_url_base),
|
||||
requester: Some(requester),
|
||||
network: None,
|
||||
stars: None,
|
||||
categories: None,
|
||||
duration: None,
|
||||
sort: None,
|
||||
sexuality: None,
|
||||
};
|
||||
|
||||
crate::flow_debug!(
|
||||
"trace={} uploaders request uploader_id={:?} uploader_name={:?} profile_content={} query={:?}",
|
||||
trace_id,
|
||||
&request.uploaderId,
|
||||
&request.uploaderName,
|
||||
request.profileContent,
|
||||
&request.query
|
||||
);
|
||||
|
||||
if let Some(uploader_id) = request.uploaderId.as_deref() {
|
||||
if let Some(provider_id) = provider_hint_from_uploader_id(uploader_id) {
|
||||
let Some(provider) = get_provider(&provider_id) else {
|
||||
return Ok(web::HttpResponse::NotFound().finish());
|
||||
};
|
||||
let result = lookup_uploader_with_provider(
|
||||
&provider_id,
|
||||
provider,
|
||||
cache.get_ref().clone(),
|
||||
pool.get_ref().clone(),
|
||||
&request,
|
||||
options,
|
||||
)
|
||||
.await;
|
||||
return match result {
|
||||
Ok(Some(profile)) => Ok(web::HttpResponse::Ok().json(&profile)),
|
||||
Ok(None) => Ok(web::HttpResponse::NotFound().finish()),
|
||||
Err(_error) => {
|
||||
crate::flow_debug!(
|
||||
"trace={} uploaders targeted provider failed provider={} error={}",
|
||||
trace_id,
|
||||
&provider_id,
|
||||
&_error
|
||||
);
|
||||
Ok(web::HttpResponse::InternalServerError().finish())
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
let mut matches = Vec::new();
|
||||
let mut saw_error = false;
|
||||
let requested_name = request
|
||||
.uploaderName
|
||||
.as_ref()
|
||||
.map(|value| value.to_ascii_lowercase());
|
||||
|
||||
for provider_id in uploader_provider_ids() {
|
||||
let Some(provider) = get_provider(&provider_id) else {
|
||||
continue;
|
||||
};
|
||||
let result = lookup_uploader_with_provider(
|
||||
&provider_id,
|
||||
provider,
|
||||
cache.get_ref().clone(),
|
||||
pool.get_ref().clone(),
|
||||
&request,
|
||||
options.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(Some(profile)) => {
|
||||
if let Some(requested_name) = requested_name.as_deref() {
|
||||
if profile.name.to_ascii_lowercase() != requested_name {
|
||||
crate::flow_debug!(
|
||||
"trace={} uploaders ignoring non_exact_match provider={} requested={} returned={}",
|
||||
trace_id,
|
||||
&provider_id,
|
||||
requested_name,
|
||||
&profile.name
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
matches.push(profile);
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(_error) => {
|
||||
saw_error = true;
|
||||
crate::flow_debug!(
|
||||
"trace={} uploaders provider failed provider={} error={}",
|
||||
trace_id,
|
||||
&provider_id,
|
||||
&_error
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if matches.is_empty() {
|
||||
if saw_error {
|
||||
return Ok(web::HttpResponse::InternalServerError().finish());
|
||||
}
|
||||
return Ok(web::HttpResponse::NotFound().finish());
|
||||
}
|
||||
|
||||
matches.sort_by(|a, b| uploader_match_sort_key(b).cmp(&uploader_match_sort_key(a)));
|
||||
Ok(web::HttpResponse::Ok().json(&matches[0]))
|
||||
}
|
||||
|
||||
pub fn get_provider(channel: &str) -> Option<DynProvider> {
|
||||
let provider = ALL_PROVIDERS.get(channel).cloned();
|
||||
crate::flow_debug!(
|
||||
@@ -539,3 +730,49 @@ pub async fn proxies() -> Result<impl web::Responder, web::Error> {
|
||||
}
|
||||
Ok(web::HttpResponse::Ok().json(&by_protocol))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn uploaders_request_requires_id_or_name() {
|
||||
let invalid = UploadersRequest::default();
|
||||
let valid = UploadersRequest {
|
||||
uploaderName: Some("Example".to_string()),
|
||||
..UploadersRequest::default()
|
||||
};
|
||||
|
||||
assert!(!uploader_request_is_valid(&invalid));
|
||||
assert!(uploader_request_is_valid(&valid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uploader_provider_hint_uses_channel_prefix() {
|
||||
assert_eq!(
|
||||
provider_hint_from_uploader_id("hsex:xihongshiddd").as_deref(),
|
||||
Some("hsex")
|
||||
);
|
||||
assert_eq!(provider_hint_from_uploader_id("plain-id"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uploader_match_prefers_higher_video_count() {
|
||||
let a = UploaderProfile {
|
||||
id: "a".to_string(),
|
||||
name: "Example".to_string(),
|
||||
channel: Some("alpha".to_string()),
|
||||
videoCount: 3,
|
||||
..UploaderProfile::default()
|
||||
};
|
||||
let b = UploaderProfile {
|
||||
id: "b".to_string(),
|
||||
name: "Example".to_string(),
|
||||
channel: Some("beta".to_string()),
|
||||
videoCount: 9,
|
||||
..UploaderProfile::default()
|
||||
};
|
||||
|
||||
assert!(uploader_match_sort_key(&b) > uploader_match_sort_key(&a));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ mod proxies;
|
||||
mod proxy;
|
||||
mod schema;
|
||||
mod status;
|
||||
mod uploaders;
|
||||
mod util;
|
||||
mod videos;
|
||||
|
||||
|
||||
@@ -4,6 +4,9 @@ use crate::providers::{
|
||||
Provider, report_provider_error, report_provider_error_background, requester_or_default,
|
||||
};
|
||||
use crate::status::*;
|
||||
use crate::uploaders::{
|
||||
UploaderChannelStat, UploaderLayoutRow, UploaderProfile, UploaderVideoRef,
|
||||
};
|
||||
use crate::util::cache::VideoCache;
|
||||
use crate::util::parse_abbreviated_number;
|
||||
use crate::util::requester::Requester;
|
||||
@@ -14,6 +17,7 @@ use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, Utc};
|
||||
use error_chain::error_chain;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use htmlentity::entity::{ICodedDataTrait, decode};
|
||||
use percent_encoding::{NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
|
||||
use regex::Regex;
|
||||
use scraper::{ElementRef, Html, Selector};
|
||||
use std::sync::{Arc, RwLock};
|
||||
@@ -507,6 +511,263 @@ impl HsexProvider {
|
||||
}
|
||||
}
|
||||
|
||||
fn canonical_uploader_id(author: &str) -> String {
|
||||
format!(
|
||||
"{CHANNEL_ID}:{}",
|
||||
utf8_percent_encode(author, NON_ALPHANUMERIC)
|
||||
)
|
||||
}
|
||||
|
||||
fn author_from_uploader_id(value: &str) -> Option<String> {
|
||||
let suffix = match value.split_once(':') {
|
||||
Some((channel, suffix)) if channel.eq_ignore_ascii_case(CHANNEL_ID) => suffix,
|
||||
Some(_) => return None,
|
||||
None => value,
|
||||
};
|
||||
percent_decode_str(suffix)
|
||||
.decode_utf8()
|
||||
.ok()
|
||||
.map(|value| value.into_owned())
|
||||
.and_then(|value| (!value.trim().is_empty()).then_some(value))
|
||||
}
|
||||
|
||||
fn author_from_uploader_href(&self, href: &str) -> Option<String> {
|
||||
let url = Url::parse(&self.absolute_url(href)).ok()?;
|
||||
url.query_pairs()
|
||||
.find(|(key, _)| key == "author")
|
||||
.map(|(_, value)| value.to_string())
|
||||
}
|
||||
|
||||
fn pagination_last_page(html: &str) -> Option<u16> {
|
||||
let regex = Regex::new(r#"user-(?P<page>\d+)\.htm\?author="#).ok()?;
|
||||
regex
|
||||
.captures_iter(html)
|
||||
.filter_map(|captures| captures.name("page")?.as_str().parse::<u16>().ok())
|
||||
.max()
|
||||
}
|
||||
|
||||
fn uploader_option_by_name(&self, uploader_name: &str) -> Option<FilterOption> {
|
||||
let normalized = uploader_name.trim();
|
||||
if normalized.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let lowered = normalized.to_lowercase();
|
||||
self.uploaders
|
||||
.read()
|
||||
.ok()?
|
||||
.iter()
|
||||
.find(|option| {
|
||||
option.title == normalized
|
||||
|| option.title.to_lowercase() == lowered
|
||||
|| option.id.eq_ignore_ascii_case(normalized)
|
||||
})
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn resolve_uploader_author(
|
||||
&self,
|
||||
uploader_id: Option<&str>,
|
||||
uploader_name: Option<&str>,
|
||||
) -> Option<String> {
|
||||
if let Some(uploader_id) = uploader_id {
|
||||
if let Some(author) = Self::author_from_uploader_id(uploader_id) {
|
||||
return Some(author);
|
||||
}
|
||||
}
|
||||
if let Some(uploader_name) = uploader_name {
|
||||
if let Some(option) = self.uploader_option_by_name(uploader_name) {
|
||||
if let Some(Target::Uploader { author }) = self.target_from_filter_id(&option.id) {
|
||||
return Some(author);
|
||||
}
|
||||
}
|
||||
let trimmed = uploader_name.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn display_name_for_uploader(
|
||||
&self,
|
||||
author: &str,
|
||||
requested_name: Option<&str>,
|
||||
first_page_items: &[VideoItem],
|
||||
) -> String {
|
||||
if let Some(requested_name) = requested_name {
|
||||
let trimmed = requested_name.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return trimmed.to_string();
|
||||
}
|
||||
}
|
||||
if let Some(name) = first_page_items
|
||||
.iter()
|
||||
.find_map(|item| item.uploader.as_deref())
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
{
|
||||
return name.to_string();
|
||||
}
|
||||
if let Some(option) = self
|
||||
.uploaders
|
||||
.read()
|
||||
.ok()
|
||||
.and_then(|values| {
|
||||
values
|
||||
.iter()
|
||||
.find(|value| value.id.contains(author) || value.title == author)
|
||||
.cloned()
|
||||
})
|
||||
{
|
||||
return option.title;
|
||||
}
|
||||
author.to_string()
|
||||
}
|
||||
|
||||
fn rank_videos_for_query(
|
||||
videos: &[UploaderVideoRef],
|
||||
query: Option<&str>,
|
||||
) -> Vec<UploaderVideoRef> {
|
||||
let Some(query) = query.map(|value| value.trim()).filter(|value| !value.is_empty()) else {
|
||||
return videos.to_vec();
|
||||
};
|
||||
let query = query.to_lowercase();
|
||||
let mut ranked = videos.to_vec();
|
||||
ranked.sort_by(|a, b| {
|
||||
let score = |video: &UploaderVideoRef| {
|
||||
let mut score = 0u8;
|
||||
if video.title.to_lowercase().contains(&query) {
|
||||
score += 2;
|
||||
}
|
||||
if video.uploader.to_lowercase().contains(&query) {
|
||||
score += 1;
|
||||
}
|
||||
score
|
||||
};
|
||||
score(b)
|
||||
.cmp(&score(a))
|
||||
.then(b.views.cmp(&a.views))
|
||||
.then_with(|| a.id.cmp(&b.id))
|
||||
});
|
||||
ranked
|
||||
}
|
||||
|
||||
async fn build_uploader_profile(
|
||||
&self,
|
||||
cache: VideoCache,
|
||||
author: &str,
|
||||
requested_name: Option<&str>,
|
||||
query: Option<&str>,
|
||||
profile_content: bool,
|
||||
options: &ServerOptions,
|
||||
) -> Result<Option<UploaderProfile>> {
|
||||
let first_page_url = self.build_uploader_url(author, 1);
|
||||
let first_page_items = self
|
||||
.fetch_items_for_url(
|
||||
cache.clone(),
|
||||
first_page_url.clone(),
|
||||
64,
|
||||
profile_content,
|
||||
options,
|
||||
)
|
||||
.await?;
|
||||
if first_page_items.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let mut requester = requester_or_default(options, CHANNEL_ID, "get_uploader.profile_page");
|
||||
let first_page_html = self
|
||||
.fetch_html(&mut requester, &first_page_url, &format!("{}/", self.url))
|
||||
.await?;
|
||||
let last_page = Self::pagination_last_page(&first_page_html).unwrap_or(1);
|
||||
let first_page_size = first_page_items.len() as u64;
|
||||
|
||||
let last_page_items = if last_page > 1 {
|
||||
self.fetch_items_for_url(
|
||||
cache,
|
||||
self.build_uploader_url(author, last_page),
|
||||
64,
|
||||
false,
|
||||
options,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
let display_name = self.display_name_for_uploader(author, requested_name, &first_page_items);
|
||||
let canonical_id = Self::canonical_uploader_id(author);
|
||||
let mut videos = first_page_items
|
||||
.iter()
|
||||
.map(|item| UploaderVideoRef::from_video_item(item, &display_name, &canonical_id))
|
||||
.collect::<Vec<_>>();
|
||||
let ranked_videos = Self::rank_videos_for_query(&videos, query);
|
||||
let horizontal_ids = ranked_videos
|
||||
.iter()
|
||||
.take(12)
|
||||
.map(|video| video.id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let newest_seen = first_page_items
|
||||
.iter()
|
||||
.filter_map(|item| item.uploadedAt)
|
||||
.max();
|
||||
let oldest_seen = last_page_items
|
||||
.iter()
|
||||
.filter_map(|item| item.uploadedAt)
|
||||
.min()
|
||||
.or_else(|| first_page_items.iter().filter_map(|item| item.uploadedAt).min());
|
||||
|
||||
let video_count = if last_page > 1 {
|
||||
((last_page as u64 - 1) * first_page_size) + last_page_items.len() as u64
|
||||
} else {
|
||||
first_page_size
|
||||
};
|
||||
let total_views = first_page_items
|
||||
.iter()
|
||||
.chain(last_page_items.iter())
|
||||
.filter_map(|item| item.views)
|
||||
.map(u64::from)
|
||||
.sum();
|
||||
|
||||
for item in &mut videos {
|
||||
item.uploader = display_name.clone();
|
||||
item.uploaderId = canonical_id.clone();
|
||||
}
|
||||
|
||||
let layout = if horizontal_ids.is_empty() {
|
||||
vec![UploaderLayoutRow::videos(None)]
|
||||
} else {
|
||||
vec![
|
||||
UploaderLayoutRow::horizontal(Some("For You".to_string()), horizontal_ids),
|
||||
UploaderLayoutRow::videos(None),
|
||||
]
|
||||
};
|
||||
|
||||
Ok(Some(UploaderProfile {
|
||||
id: canonical_id,
|
||||
name: display_name,
|
||||
url: Some(first_page_url),
|
||||
channel: Some(CHANNEL_ID.to_string()),
|
||||
verified: false,
|
||||
videoCount: video_count,
|
||||
totalViews: total_views,
|
||||
channels: Some(vec![UploaderChannelStat {
|
||||
channel: CHANNEL_ID.to_string(),
|
||||
videoCount: video_count,
|
||||
firstSeenAt: crate::uploaders::iso_timestamp_from_unix(oldest_seen),
|
||||
lastSeenAt: crate::uploaders::iso_timestamp_from_unix(newest_seen),
|
||||
}]),
|
||||
avatar: None,
|
||||
description: None,
|
||||
bio: None,
|
||||
videos: profile_content.then_some(videos),
|
||||
tapes: profile_content.then_some(Vec::new()),
|
||||
playlists: profile_content.then_some(Vec::new()),
|
||||
layout: Some(layout),
|
||||
}))
|
||||
}
|
||||
|
||||
fn first_video_link<'a>(&self, element: &'a ElementRef<'a>) -> Result<Option<ElementRef<'a>>> {
|
||||
let selector = Self::selector("a[href]")?;
|
||||
Ok(element.select(&selector).find(|link| {
|
||||
@@ -638,6 +899,9 @@ impl HsexProvider {
|
||||
}
|
||||
if let Some(uploader_href) = uploader.value().attr("href") {
|
||||
item.uploaderUrl = Some(self.absolute_url(uploader_href));
|
||||
item.uploaderId = self
|
||||
.author_from_uploader_href(uploader_href)
|
||||
.map(|author| Self::canonical_uploader_id(&author));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -701,6 +965,9 @@ impl HsexProvider {
|
||||
}
|
||||
if let Some(href) = author.value().attr("href") {
|
||||
item.uploaderUrl = Some(self.absolute_url(href));
|
||||
item.uploaderId = self
|
||||
.author_from_uploader_href(href)
|
||||
.map(|author| Self::canonical_uploader_id(&author));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -934,6 +1201,34 @@ impl Provider for HsexProvider {
|
||||
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
||||
Some(self.build_channel(clientversion))
|
||||
}
|
||||
|
||||
async fn get_uploader(
|
||||
&self,
|
||||
cache: VideoCache,
|
||||
pool: DbPool,
|
||||
uploader_id: Option<String>,
|
||||
uploader_name: Option<String>,
|
||||
query: Option<String>,
|
||||
profile_content: bool,
|
||||
options: ServerOptions,
|
||||
) -> std::result::Result<Option<UploaderProfile>, String> {
|
||||
let _ = pool;
|
||||
let Some(author) =
|
||||
self.resolve_uploader_author(uploader_id.as_deref(), uploader_name.as_deref())
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
self.build_uploader_profile(
|
||||
cache,
|
||||
&author,
|
||||
uploader_name.as_deref(),
|
||||
query.as_deref(),
|
||||
profile_content,
|
||||
&options,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1020,6 +1315,29 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_uploader_id_round_trips() {
|
||||
let canonical = HsexProvider::canonical_uploader_id("xihongshiddd");
|
||||
assert_eq!(canonical, "hsex:xihongshiddd");
|
||||
assert_eq!(
|
||||
HsexProvider::author_from_uploader_id(&canonical).as_deref(),
|
||||
Some("xihongshiddd")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_last_page_from_pagination() {
|
||||
let html = r#"
|
||||
<ul class="pagination1">
|
||||
<li><a href="user-1.htm?author=xihongshiddd">1</a></li>
|
||||
<li><a href="user-2.htm?author=xihongshiddd">2</a></li>
|
||||
<li><a href="user-7.htm?author=xihongshiddd">7</a></li>
|
||||
</ul>
|
||||
"#;
|
||||
|
||||
assert_eq!(HsexProvider::pagination_last_page(html), Some(7));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn fetches_page_two_items() {
|
||||
|
||||
@@ -12,6 +12,7 @@ use crate::{
|
||||
DbPool,
|
||||
api::ClientVersion,
|
||||
status::{Channel, ChannelGroup, ChannelView, FilterOption, Status, StatusResponse},
|
||||
uploaders::UploaderProfile,
|
||||
util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester},
|
||||
videos::{FlexibleNumber, ServerOptions, VideoItem, VideosRequest},
|
||||
};
|
||||
@@ -577,6 +578,53 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run_uploader_provider_guarded<F>(
|
||||
provider_name: &str,
|
||||
context: &str,
|
||||
fut: F,
|
||||
) -> Result<Option<UploaderProfile>, String>
|
||||
where
|
||||
F: Future<Output = Result<Option<UploaderProfile>, String>>,
|
||||
{
|
||||
crate::flow_debug!(
|
||||
"provider uploader guard enter provider={} context={}",
|
||||
provider_name,
|
||||
context
|
||||
);
|
||||
match AssertUnwindSafe(fut).catch_unwind().await {
|
||||
Ok(result) => {
|
||||
crate::flow_debug!(
|
||||
"provider uploader guard exit provider={} context={} matched={}",
|
||||
provider_name,
|
||||
context,
|
||||
result.as_ref().ok().and_then(|value| value.as_ref()).is_some()
|
||||
);
|
||||
result
|
||||
}
|
||||
Err(payload) => {
|
||||
let panic_msg = panic_payload_to_string(payload);
|
||||
crate::flow_debug!(
|
||||
"provider uploader guard panic provider={} context={} panic={}",
|
||||
provider_name,
|
||||
context,
|
||||
&panic_msg
|
||||
);
|
||||
let _ = send_discord_error_report(
|
||||
format!("Provider panic: {}", provider_name),
|
||||
None,
|
||||
Some("Provider Guard"),
|
||||
Some(&format!("context={}; panic={}", context, panic_msg)),
|
||||
file!(),
|
||||
line!(),
|
||||
module_path!(),
|
||||
)
|
||||
.await;
|
||||
schedule_provider_validation(provider_name, context, &panic_msg);
|
||||
Err(panic_msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn report_provider_error(provider_name: &str, context: &str, msg: &str) {
|
||||
let _ = send_discord_error_report(
|
||||
format!("Provider error: {}", provider_name),
|
||||
@@ -868,6 +916,19 @@ pub trait Provider: Send + Sync {
|
||||
cacheDuration: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn get_uploader(
|
||||
&self,
|
||||
_cache: VideoCache,
|
||||
_pool: DbPool,
|
||||
_uploader_id: Option<String>,
|
||||
_uploader_name: Option<String>,
|
||||
_query: Option<String>,
|
||||
_profile_content: bool,
|
||||
_options: ServerOptions,
|
||||
) -> Result<Option<UploaderProfile>, String> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, not(hottub_single_provider)))]
|
||||
|
||||
@@ -474,6 +474,19 @@ impl NoodlemagazineProvider {
|
||||
if normalized.is_empty() || !self.is_allowed_thumb_url(&normalized) {
|
||||
return String::new();
|
||||
}
|
||||
let Some(url) = Url::parse(&normalized).ok() else {
|
||||
return String::new();
|
||||
};
|
||||
if url
|
||||
.host_str()
|
||||
.is_some_and(|host| host.eq_ignore_ascii_case("img.pvvstream.pro"))
|
||||
{
|
||||
return crate::providers::build_proxy_url(
|
||||
_options,
|
||||
"noodlemagazine-thumb",
|
||||
&crate::providers::strip_url_scheme(&normalized),
|
||||
);
|
||||
}
|
||||
normalized
|
||||
}
|
||||
|
||||
@@ -707,7 +720,7 @@ mod tests {
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(
|
||||
items[0].thumb,
|
||||
"https://img.pvvstream.pro/preview/abc/-111_222/240/iv.okcdn.ru/getVideoPreview?id=1&type=39&fn=vid_l"
|
||||
"https://example.com/proxy/noodlemagazine-thumb/img.pvvstream.pro/preview/abc/-111_222/240/iv.okcdn.ru/getVideoPreview?id=1&type=39&fn=vid_l"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
216
src/uploaders.rs
Normal file
216
src/uploaders.rs
Normal file
@@ -0,0 +1,216 @@
|
||||
use chrono::{SecondsFormat, TimeZone, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::videos::VideoItem;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||
pub struct UploadersRequest {
|
||||
#[serde(default, alias = "uploader_id")]
|
||||
pub uploaderId: Option<String>,
|
||||
#[serde(default, alias = "uploader_name")]
|
||||
pub uploaderName: Option<String>,
|
||||
#[serde(default, alias = "profile_content")]
|
||||
pub profileContent: bool,
|
||||
#[serde(default)]
|
||||
pub query: Option<String>,
|
||||
}
|
||||
|
||||
impl UploadersRequest {
|
||||
pub fn normalized(self) -> Self {
|
||||
Self {
|
||||
uploaderId: normalize_optional_string(self.uploaderId),
|
||||
uploaderName: normalize_optional_string(self.uploaderName),
|
||||
profileContent: self.profileContent,
|
||||
query: normalize_optional_string(self.query),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||
pub struct UploaderProfile {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub url: Option<String>,
|
||||
pub channel: Option<String>,
|
||||
pub verified: bool,
|
||||
pub videoCount: u64,
|
||||
pub totalViews: u64,
|
||||
#[serde(default)]
|
||||
pub channels: Option<Vec<UploaderChannelStat>>,
|
||||
#[serde(default, alias = "profile_picture_url")]
|
||||
pub avatar: Option<String>,
|
||||
pub description: Option<String>,
|
||||
pub bio: Option<String>,
|
||||
#[serde(default)]
|
||||
pub videos: Option<Vec<UploaderVideoRef>>,
|
||||
#[serde(default)]
|
||||
pub tapes: Option<Vec<serde_json::Value>>,
|
||||
#[serde(default)]
|
||||
pub playlists: Option<Vec<serde_json::Value>>,
|
||||
#[serde(default)]
|
||||
pub layout: Option<Vec<UploaderLayoutRow>>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||
pub struct UploaderChannelStat {
|
||||
pub channel: String,
|
||||
pub videoCount: u64,
|
||||
#[serde(default, alias = "first_seen_at")]
|
||||
pub firstSeenAt: Option<String>,
|
||||
#[serde(default, alias = "last_seen_at")]
|
||||
pub lastSeenAt: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||
pub struct UploaderVideoRef {
|
||||
pub id: String,
|
||||
pub url: String,
|
||||
pub title: String,
|
||||
pub duration: u32,
|
||||
pub channel: String,
|
||||
#[serde(default, alias = "uploaded_at")]
|
||||
pub uploadedAt: Option<String>,
|
||||
pub uploader: String,
|
||||
#[serde(alias = "uploader_id")]
|
||||
pub uploaderId: String,
|
||||
pub thumb: String,
|
||||
pub preview: Option<String>,
|
||||
pub views: u32,
|
||||
pub rating: u32,
|
||||
#[serde(default, alias = "aspect_ratio")]
|
||||
pub aspectRatio: Option<f32>,
|
||||
}
|
||||
|
||||
impl UploaderVideoRef {
|
||||
pub fn from_video_item(item: &VideoItem, uploader_name: &str, uploader_id: &str) -> Self {
|
||||
Self {
|
||||
id: item.id.clone(),
|
||||
url: item.url.clone(),
|
||||
title: item.title.clone(),
|
||||
duration: item.duration,
|
||||
channel: item.channel.clone(),
|
||||
uploadedAt: iso_timestamp_from_unix(item.uploadedAt),
|
||||
uploader: item
|
||||
.uploader
|
||||
.clone()
|
||||
.unwrap_or_else(|| uploader_name.to_string()),
|
||||
uploaderId: item
|
||||
.uploaderId
|
||||
.clone()
|
||||
.unwrap_or_else(|| uploader_id.to_string()),
|
||||
thumb: item.thumb.clone(),
|
||||
preview: item.preview.clone(),
|
||||
views: item.views.unwrap_or_default(),
|
||||
rating: item.rating.map(normalize_rating).unwrap_or_default(),
|
||||
aspectRatio: item.aspectRatio,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||
pub struct UploaderLayoutRow {
|
||||
#[serde(rename = "type")]
|
||||
pub rowType: UploaderLayoutRowType,
|
||||
pub title: Option<String>,
|
||||
#[serde(default, alias = "video_ids")]
|
||||
pub videoIds: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl UploaderLayoutRow {
|
||||
pub fn horizontal(title: Option<String>, video_ids: Vec<String>) -> Self {
|
||||
Self {
|
||||
rowType: UploaderLayoutRowType::Horizontal,
|
||||
title,
|
||||
videoIds: Some(video_ids),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn videos(title: Option<String>) -> Self {
|
||||
Self {
|
||||
rowType: UploaderLayoutRowType::Videos,
|
||||
title,
|
||||
videoIds: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub enum UploaderLayoutRowType {
|
||||
#[default]
|
||||
#[serde(rename = "videos")]
|
||||
Videos,
|
||||
#[serde(rename = "horizontal", alias = "horizontal_videos")]
|
||||
Horizontal,
|
||||
}
|
||||
|
||||
pub fn normalize_optional_string(value: Option<String>) -> Option<String> {
|
||||
value.and_then(|value| {
|
||||
let trimmed = value.trim();
|
||||
(!trimmed.is_empty()).then(|| trimmed.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn iso_timestamp_from_unix(value: Option<u64>) -> Option<String> {
|
||||
let timestamp = value?;
|
||||
let dt = Utc.timestamp_opt(timestamp as i64, 0).single()?;
|
||||
Some(dt.to_rfc3339_opts(SecondsFormat::Millis, true))
|
||||
}
|
||||
|
||||
fn normalize_rating(value: f32) -> u32 {
|
||||
value.clamp(0.0, 100.0).round() as u32
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn request_accepts_snake_case_aliases() {
|
||||
let request: UploadersRequest = serde_json::from_str(
|
||||
r#"{
|
||||
"uploader_id": "hsex:xihongshiddd",
|
||||
"uploader_name": "xihongshiddd",
|
||||
"profile_content": true,
|
||||
"query": "teacher"
|
||||
}"#,
|
||||
)
|
||||
.expect("request should decode");
|
||||
|
||||
assert_eq!(request.uploaderId.as_deref(), Some("hsex:xihongshiddd"));
|
||||
assert_eq!(request.uploaderName.as_deref(), Some("xihongshiddd"));
|
||||
assert!(request.profileContent);
|
||||
assert_eq!(request.query.as_deref(), Some("teacher"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn layout_aliases_decode() {
|
||||
let row: UploaderLayoutRow = serde_json::from_str(
|
||||
r#"{
|
||||
"type": "horizontal_videos",
|
||||
"title": "For You",
|
||||
"video_ids": ["one", "two"]
|
||||
}"#,
|
||||
)
|
||||
.expect("row should decode");
|
||||
|
||||
assert_eq!(row.rowType, UploaderLayoutRowType::Horizontal);
|
||||
assert_eq!(row.videoIds.as_ref().map(Vec::len), Some(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn avatar_alias_decodes() {
|
||||
let profile: UploaderProfile = serde_json::from_str(
|
||||
r#"{
|
||||
"id": "abc",
|
||||
"name": "Example",
|
||||
"verified": false,
|
||||
"videoCount": 1,
|
||||
"totalViews": 2,
|
||||
"profile_picture_url": "https://example.com/a.jpg"
|
||||
}"#,
|
||||
)
|
||||
.expect("profile should decode");
|
||||
|
||||
assert_eq!(profile.avatar.as_deref(), Some("https://example.com/a.jpg"));
|
||||
}
|
||||
}
|
||||
@@ -100,6 +100,8 @@ pub struct VideoItem {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub uploaderUrl: Option<String>, // "https://www.youtube.com/@petcollective",
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub uploaderId: Option<String>, // "petcollective",
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub verified: Option<bool>, // false,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tags: Option<Vec<String>>, // [],
|
||||
@@ -135,6 +137,7 @@ impl VideoItem {
|
||||
thumb,
|
||||
uploader: None,
|
||||
uploaderUrl: None,
|
||||
uploaderId: None,
|
||||
verified: None,
|
||||
tags: None, // Placeholder, adjust as needed
|
||||
uploadedAt: None,
|
||||
|
||||
Reference in New Issue
Block a user