uploaders
This commit is contained in:
60
docs/uploaders-endpoint-plan.md
Normal file
60
docs/uploaders-endpoint-plan.md
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
# Uploaders Endpoint Plan
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Implement `POST /api/uploaders` using the Hot Tub uploader profile contract and ship it framework-first. The server will expose shared uploader request/response types, a provider hook for uploader lookup, endpoint routing in `src/api.rs`, and a first real provider implementation in `hsex`.
|
||||||
|
|
||||||
|
## Implementation
|
||||||
|
|
||||||
|
- Add dedicated uploader API types in `src/uploaders.rs`:
|
||||||
|
- `UploadersRequest`
|
||||||
|
- `UploaderProfile`
|
||||||
|
- `UploaderChannelStat`
|
||||||
|
- `UploaderVideoRef`
|
||||||
|
- `UploaderLayoutRow`
|
||||||
|
- Keep camelCase as the canonical serialized shape.
|
||||||
|
- Accept documented decode aliases:
|
||||||
|
- `uploader_id`
|
||||||
|
- `uploader_name`
|
||||||
|
- `profile_content`
|
||||||
|
- `profile_picture_url`
|
||||||
|
- `video_ids`
|
||||||
|
- `horizontal_videos`
|
||||||
|
- Add `POST /api/uploaders` in `src/api.rs`.
|
||||||
|
- Validate that at least one of `uploaderId` or `uploaderName` is present.
|
||||||
|
- Return:
|
||||||
|
- `400` for invalid request
|
||||||
|
- `404` for no match
|
||||||
|
- `500` for provider execution failure
|
||||||
|
- Add `Provider::get_uploader(...)` with a default `Ok(None)` implementation.
|
||||||
|
- Add a guarded uploader execution helper in `src/providers/mod.rs`.
|
||||||
|
- Use canonical uploader IDs in the format `<channel>:<provider-local-id>`.
|
||||||
|
- Implement the first provider-backed uploader profile in `src/providers/hsex.rs`.
|
||||||
|
|
||||||
|
## Hsex Strategy
|
||||||
|
|
||||||
|
- Resolve uploader lookup by canonical uploader ID or exact uploader name.
|
||||||
|
- Reuse existing uploader archive discovery and archive page fetching.
|
||||||
|
- Build uploader profile metadata from uploader archive pages.
|
||||||
|
- Populate `videos` with `UploaderVideoRef` values derived from existing `VideoItem`s.
|
||||||
|
- Always return `layout`.
|
||||||
|
- When `profileContent == true`, return:
|
||||||
|
- `videos`
|
||||||
|
- `tapes: []`
|
||||||
|
- `playlists: []`
|
||||||
|
- a `"For You"` horizontal row plus the default videos row
|
||||||
|
- When `profileContent == false`, return metadata and layout only.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
- Request alias decoding for uploader request fields.
|
||||||
|
- Response alias decoding for avatar and layout row compatibility fields.
|
||||||
|
- Endpoint helper tests for request validation and provider routing.
|
||||||
|
- Hsex uploader ID generation and uploader page parsing coverage.
|
||||||
|
|
||||||
|
## Assumptions
|
||||||
|
|
||||||
|
- The first ship focuses on the endpoint framework and one real provider implementation.
|
||||||
|
- Providers without explicit uploader support remain unsupported by `/api/uploaders`.
|
||||||
|
- Name-based resolution uses exact display-name matching.
|
||||||
|
- `videoCount` and `totalViews` are best-effort when the upstream site does not expose authoritative profile totals.
|
||||||
237
src/api.rs
237
src/api.rs
@@ -1,7 +1,9 @@
|
|||||||
use crate::providers::{
|
use crate::providers::{
|
||||||
ALL_PROVIDERS, DynProvider, build_status_response, panic_payload_to_string,
|
ALL_PROVIDERS, DynProvider, build_status_response, panic_payload_to_string,
|
||||||
report_provider_error, resolve_provider_for_build, run_provider_guarded,
|
report_provider_error, resolve_provider_for_build, run_provider_guarded,
|
||||||
|
run_uploader_provider_guarded,
|
||||||
};
|
};
|
||||||
|
use crate::uploaders::{UploaderProfile, UploadersRequest};
|
||||||
use crate::util::cache::VideoCache;
|
use crate::util::cache::VideoCache;
|
||||||
use crate::util::discord::send_discord_error_report;
|
use crate::util::discord::send_discord_error_report;
|
||||||
use crate::util::proxy::{Proxy, all_proxies_snapshot};
|
use crate::util::proxy::{Proxy, all_proxies_snapshot};
|
||||||
@@ -141,10 +143,61 @@ pub fn config(cfg: &mut web::ServiceConfig) {
|
|||||||
// .route(web::get().to(videos_get))
|
// .route(web::get().to(videos_get))
|
||||||
.route(web::post().to(videos_post)),
|
.route(web::post().to(videos_post)),
|
||||||
)
|
)
|
||||||
|
.service(web::resource("/uploaders").route(web::post().to(uploaders_post)))
|
||||||
.service(web::resource("/test").route(web::get().to(test)))
|
.service(web::resource("/test").route(web::get().to(test)))
|
||||||
.service(web::resource("/proxies").route(web::get().to(proxies)));
|
.service(web::resource("/proxies").route(web::get().to(proxies)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn uploader_request_is_valid(request: &UploadersRequest) -> bool {
|
||||||
|
request.uploaderId.is_some() || request.uploaderName.is_some()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn provider_hint_from_uploader_id(uploader_id: &str) -> Option<String> {
|
||||||
|
let (channel, _) = uploader_id.split_once(':')?;
|
||||||
|
Some(resolve_provider_for_build(channel).to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn uploader_provider_ids() -> Vec<String> {
|
||||||
|
let mut ids = ALL_PROVIDERS
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(provider_id, _)| (*provider_id != "all").then(|| (*provider_id).to_string()))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
ids.sort();
|
||||||
|
ids
|
||||||
|
}
|
||||||
|
|
||||||
|
fn uploader_match_sort_key(profile: &UploaderProfile) -> (u64, String, String) {
|
||||||
|
(
|
||||||
|
profile.videoCount,
|
||||||
|
profile.channel.clone().unwrap_or_default(),
|
||||||
|
profile.id.clone(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn lookup_uploader_with_provider(
|
||||||
|
provider_id: &str,
|
||||||
|
provider: DynProvider,
|
||||||
|
cache: VideoCache,
|
||||||
|
pool: DbPool,
|
||||||
|
request: &UploadersRequest,
|
||||||
|
options: crate::videos::ServerOptions,
|
||||||
|
) -> Result<Option<UploaderProfile>, String> {
|
||||||
|
run_uploader_provider_guarded(
|
||||||
|
provider_id,
|
||||||
|
"uploaders_post.get_uploader",
|
||||||
|
provider.get_uploader(
|
||||||
|
cache,
|
||||||
|
pool,
|
||||||
|
request.uploaderId.clone(),
|
||||||
|
request.uploaderName.clone(),
|
||||||
|
request.query.clone(),
|
||||||
|
request.profileContent,
|
||||||
|
options,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
|
async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
|
||||||
#[cfg(feature = "debug")]
|
#[cfg(feature = "debug")]
|
||||||
let trace_id = crate::util::flow_debug::next_trace_id("status");
|
let trace_id = crate::util::flow_debug::next_trace_id("status");
|
||||||
@@ -491,6 +544,144 @@ async fn videos_post(
|
|||||||
Ok(web::HttpResponse::Ok().json(&videos))
|
Ok(web::HttpResponse::Ok().json(&videos))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn uploaders_post(
|
||||||
|
uploader_request: web::types::Json<UploadersRequest>,
|
||||||
|
cache: web::types::State<VideoCache>,
|
||||||
|
pool: web::types::State<DbPool>,
|
||||||
|
requester: web::types::State<Requester>,
|
||||||
|
req: HttpRequest,
|
||||||
|
) -> Result<impl web::Responder, web::Error> {
|
||||||
|
let trace_id = crate::util::flow_debug::next_trace_id("uploaders");
|
||||||
|
let request = uploader_request.into_inner().normalized();
|
||||||
|
if !uploader_request_is_valid(&request) {
|
||||||
|
return Ok(web::HttpResponse::BadRequest().body(
|
||||||
|
"At least one of uploaderId or uploaderName must be provided",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let public_url_base = format!(
|
||||||
|
"{}://{}",
|
||||||
|
req.connection_info().scheme(),
|
||||||
|
req.connection_info().host()
|
||||||
|
);
|
||||||
|
let mut requester = requester.get_ref().clone();
|
||||||
|
requester.set_debug_trace_id(Some(trace_id.clone()));
|
||||||
|
let options = ServerOptions {
|
||||||
|
featured: None,
|
||||||
|
category: None,
|
||||||
|
sites: None,
|
||||||
|
filter: None,
|
||||||
|
language: None,
|
||||||
|
public_url_base: Some(public_url_base),
|
||||||
|
requester: Some(requester),
|
||||||
|
network: None,
|
||||||
|
stars: None,
|
||||||
|
categories: None,
|
||||||
|
duration: None,
|
||||||
|
sort: None,
|
||||||
|
sexuality: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
crate::flow_debug!(
|
||||||
|
"trace={} uploaders request uploader_id={:?} uploader_name={:?} profile_content={} query={:?}",
|
||||||
|
trace_id,
|
||||||
|
&request.uploaderId,
|
||||||
|
&request.uploaderName,
|
||||||
|
request.profileContent,
|
||||||
|
&request.query
|
||||||
|
);
|
||||||
|
|
||||||
|
if let Some(uploader_id) = request.uploaderId.as_deref() {
|
||||||
|
if let Some(provider_id) = provider_hint_from_uploader_id(uploader_id) {
|
||||||
|
let Some(provider) = get_provider(&provider_id) else {
|
||||||
|
return Ok(web::HttpResponse::NotFound().finish());
|
||||||
|
};
|
||||||
|
let result = lookup_uploader_with_provider(
|
||||||
|
&provider_id,
|
||||||
|
provider,
|
||||||
|
cache.get_ref().clone(),
|
||||||
|
pool.get_ref().clone(),
|
||||||
|
&request,
|
||||||
|
options,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return match result {
|
||||||
|
Ok(Some(profile)) => Ok(web::HttpResponse::Ok().json(&profile)),
|
||||||
|
Ok(None) => Ok(web::HttpResponse::NotFound().finish()),
|
||||||
|
Err(_error) => {
|
||||||
|
crate::flow_debug!(
|
||||||
|
"trace={} uploaders targeted provider failed provider={} error={}",
|
||||||
|
trace_id,
|
||||||
|
&provider_id,
|
||||||
|
&_error
|
||||||
|
);
|
||||||
|
Ok(web::HttpResponse::InternalServerError().finish())
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut matches = Vec::new();
|
||||||
|
let mut saw_error = false;
|
||||||
|
let requested_name = request
|
||||||
|
.uploaderName
|
||||||
|
.as_ref()
|
||||||
|
.map(|value| value.to_ascii_lowercase());
|
||||||
|
|
||||||
|
for provider_id in uploader_provider_ids() {
|
||||||
|
let Some(provider) = get_provider(&provider_id) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let result = lookup_uploader_with_provider(
|
||||||
|
&provider_id,
|
||||||
|
provider,
|
||||||
|
cache.get_ref().clone(),
|
||||||
|
pool.get_ref().clone(),
|
||||||
|
&request,
|
||||||
|
options.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
match result {
|
||||||
|
Ok(Some(profile)) => {
|
||||||
|
if let Some(requested_name) = requested_name.as_deref() {
|
||||||
|
if profile.name.to_ascii_lowercase() != requested_name {
|
||||||
|
crate::flow_debug!(
|
||||||
|
"trace={} uploaders ignoring non_exact_match provider={} requested={} returned={}",
|
||||||
|
trace_id,
|
||||||
|
&provider_id,
|
||||||
|
requested_name,
|
||||||
|
&profile.name
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
matches.push(profile);
|
||||||
|
}
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(_error) => {
|
||||||
|
saw_error = true;
|
||||||
|
crate::flow_debug!(
|
||||||
|
"trace={} uploaders provider failed provider={} error={}",
|
||||||
|
trace_id,
|
||||||
|
&provider_id,
|
||||||
|
&_error
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if matches.is_empty() {
|
||||||
|
if saw_error {
|
||||||
|
return Ok(web::HttpResponse::InternalServerError().finish());
|
||||||
|
}
|
||||||
|
return Ok(web::HttpResponse::NotFound().finish());
|
||||||
|
}
|
||||||
|
|
||||||
|
matches.sort_by(|a, b| uploader_match_sort_key(b).cmp(&uploader_match_sort_key(a)));
|
||||||
|
Ok(web::HttpResponse::Ok().json(&matches[0]))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get_provider(channel: &str) -> Option<DynProvider> {
|
pub fn get_provider(channel: &str) -> Option<DynProvider> {
|
||||||
let provider = ALL_PROVIDERS.get(channel).cloned();
|
let provider = ALL_PROVIDERS.get(channel).cloned();
|
||||||
crate::flow_debug!(
|
crate::flow_debug!(
|
||||||
@@ -539,3 +730,49 @@ pub async fn proxies() -> Result<impl web::Responder, web::Error> {
|
|||||||
}
|
}
|
||||||
Ok(web::HttpResponse::Ok().json(&by_protocol))
|
Ok(web::HttpResponse::Ok().json(&by_protocol))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uploaders_request_requires_id_or_name() {
|
||||||
|
let invalid = UploadersRequest::default();
|
||||||
|
let valid = UploadersRequest {
|
||||||
|
uploaderName: Some("Example".to_string()),
|
||||||
|
..UploadersRequest::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(!uploader_request_is_valid(&invalid));
|
||||||
|
assert!(uploader_request_is_valid(&valid));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uploader_provider_hint_uses_channel_prefix() {
|
||||||
|
assert_eq!(
|
||||||
|
provider_hint_from_uploader_id("hsex:xihongshiddd").as_deref(),
|
||||||
|
Some("hsex")
|
||||||
|
);
|
||||||
|
assert_eq!(provider_hint_from_uploader_id("plain-id"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn uploader_match_prefers_higher_video_count() {
|
||||||
|
let a = UploaderProfile {
|
||||||
|
id: "a".to_string(),
|
||||||
|
name: "Example".to_string(),
|
||||||
|
channel: Some("alpha".to_string()),
|
||||||
|
videoCount: 3,
|
||||||
|
..UploaderProfile::default()
|
||||||
|
};
|
||||||
|
let b = UploaderProfile {
|
||||||
|
id: "b".to_string(),
|
||||||
|
name: "Example".to_string(),
|
||||||
|
channel: Some("beta".to_string()),
|
||||||
|
videoCount: 9,
|
||||||
|
..UploaderProfile::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(uploader_match_sort_key(&b) > uploader_match_sort_key(&a));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ mod proxies;
|
|||||||
mod proxy;
|
mod proxy;
|
||||||
mod schema;
|
mod schema;
|
||||||
mod status;
|
mod status;
|
||||||
|
mod uploaders;
|
||||||
mod util;
|
mod util;
|
||||||
mod videos;
|
mod videos;
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,9 @@ use crate::providers::{
|
|||||||
Provider, report_provider_error, report_provider_error_background, requester_or_default,
|
Provider, report_provider_error, report_provider_error_background, requester_or_default,
|
||||||
};
|
};
|
||||||
use crate::status::*;
|
use crate::status::*;
|
||||||
|
use crate::uploaders::{
|
||||||
|
UploaderChannelStat, UploaderLayoutRow, UploaderProfile, UploaderVideoRef,
|
||||||
|
};
|
||||||
use crate::util::cache::VideoCache;
|
use crate::util::cache::VideoCache;
|
||||||
use crate::util::parse_abbreviated_number;
|
use crate::util::parse_abbreviated_number;
|
||||||
use crate::util::requester::Requester;
|
use crate::util::requester::Requester;
|
||||||
@@ -14,6 +17,7 @@ use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, Utc};
|
|||||||
use error_chain::error_chain;
|
use error_chain::error_chain;
|
||||||
use futures::stream::{self, StreamExt};
|
use futures::stream::{self, StreamExt};
|
||||||
use htmlentity::entity::{ICodedDataTrait, decode};
|
use htmlentity::entity::{ICodedDataTrait, decode};
|
||||||
|
use percent_encoding::{NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use scraper::{ElementRef, Html, Selector};
|
use scraper::{ElementRef, Html, Selector};
|
||||||
use std::sync::{Arc, RwLock};
|
use std::sync::{Arc, RwLock};
|
||||||
@@ -507,6 +511,263 @@ impl HsexProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn canonical_uploader_id(author: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"{CHANNEL_ID}:{}",
|
||||||
|
utf8_percent_encode(author, NON_ALPHANUMERIC)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn author_from_uploader_id(value: &str) -> Option<String> {
|
||||||
|
let suffix = match value.split_once(':') {
|
||||||
|
Some((channel, suffix)) if channel.eq_ignore_ascii_case(CHANNEL_ID) => suffix,
|
||||||
|
Some(_) => return None,
|
||||||
|
None => value,
|
||||||
|
};
|
||||||
|
percent_decode_str(suffix)
|
||||||
|
.decode_utf8()
|
||||||
|
.ok()
|
||||||
|
.map(|value| value.into_owned())
|
||||||
|
.and_then(|value| (!value.trim().is_empty()).then_some(value))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn author_from_uploader_href(&self, href: &str) -> Option<String> {
|
||||||
|
let url = Url::parse(&self.absolute_url(href)).ok()?;
|
||||||
|
url.query_pairs()
|
||||||
|
.find(|(key, _)| key == "author")
|
||||||
|
.map(|(_, value)| value.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pagination_last_page(html: &str) -> Option<u16> {
|
||||||
|
let regex = Regex::new(r#"user-(?P<page>\d+)\.htm\?author="#).ok()?;
|
||||||
|
regex
|
||||||
|
.captures_iter(html)
|
||||||
|
.filter_map(|captures| captures.name("page")?.as_str().parse::<u16>().ok())
|
||||||
|
.max()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn uploader_option_by_name(&self, uploader_name: &str) -> Option<FilterOption> {
|
||||||
|
let normalized = uploader_name.trim();
|
||||||
|
if normalized.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let lowered = normalized.to_lowercase();
|
||||||
|
self.uploaders
|
||||||
|
.read()
|
||||||
|
.ok()?
|
||||||
|
.iter()
|
||||||
|
.find(|option| {
|
||||||
|
option.title == normalized
|
||||||
|
|| option.title.to_lowercase() == lowered
|
||||||
|
|| option.id.eq_ignore_ascii_case(normalized)
|
||||||
|
})
|
||||||
|
.cloned()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_uploader_author(
|
||||||
|
&self,
|
||||||
|
uploader_id: Option<&str>,
|
||||||
|
uploader_name: Option<&str>,
|
||||||
|
) -> Option<String> {
|
||||||
|
if let Some(uploader_id) = uploader_id {
|
||||||
|
if let Some(author) = Self::author_from_uploader_id(uploader_id) {
|
||||||
|
return Some(author);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(uploader_name) = uploader_name {
|
||||||
|
if let Some(option) = self.uploader_option_by_name(uploader_name) {
|
||||||
|
if let Some(Target::Uploader { author }) = self.target_from_filter_id(&option.id) {
|
||||||
|
return Some(author);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let trimmed = uploader_name.trim();
|
||||||
|
if !trimmed.is_empty() {
|
||||||
|
return Some(trimmed.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn display_name_for_uploader(
|
||||||
|
&self,
|
||||||
|
author: &str,
|
||||||
|
requested_name: Option<&str>,
|
||||||
|
first_page_items: &[VideoItem],
|
||||||
|
) -> String {
|
||||||
|
if let Some(requested_name) = requested_name {
|
||||||
|
let trimmed = requested_name.trim();
|
||||||
|
if !trimmed.is_empty() {
|
||||||
|
return trimmed.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(name) = first_page_items
|
||||||
|
.iter()
|
||||||
|
.find_map(|item| item.uploader.as_deref())
|
||||||
|
.filter(|value| !value.trim().is_empty())
|
||||||
|
{
|
||||||
|
return name.to_string();
|
||||||
|
}
|
||||||
|
if let Some(option) = self
|
||||||
|
.uploaders
|
||||||
|
.read()
|
||||||
|
.ok()
|
||||||
|
.and_then(|values| {
|
||||||
|
values
|
||||||
|
.iter()
|
||||||
|
.find(|value| value.id.contains(author) || value.title == author)
|
||||||
|
.cloned()
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return option.title;
|
||||||
|
}
|
||||||
|
author.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rank_videos_for_query(
|
||||||
|
videos: &[UploaderVideoRef],
|
||||||
|
query: Option<&str>,
|
||||||
|
) -> Vec<UploaderVideoRef> {
|
||||||
|
let Some(query) = query.map(|value| value.trim()).filter(|value| !value.is_empty()) else {
|
||||||
|
return videos.to_vec();
|
||||||
|
};
|
||||||
|
let query = query.to_lowercase();
|
||||||
|
let mut ranked = videos.to_vec();
|
||||||
|
ranked.sort_by(|a, b| {
|
||||||
|
let score = |video: &UploaderVideoRef| {
|
||||||
|
let mut score = 0u8;
|
||||||
|
if video.title.to_lowercase().contains(&query) {
|
||||||
|
score += 2;
|
||||||
|
}
|
||||||
|
if video.uploader.to_lowercase().contains(&query) {
|
||||||
|
score += 1;
|
||||||
|
}
|
||||||
|
score
|
||||||
|
};
|
||||||
|
score(b)
|
||||||
|
.cmp(&score(a))
|
||||||
|
.then(b.views.cmp(&a.views))
|
||||||
|
.then_with(|| a.id.cmp(&b.id))
|
||||||
|
});
|
||||||
|
ranked
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn build_uploader_profile(
|
||||||
|
&self,
|
||||||
|
cache: VideoCache,
|
||||||
|
author: &str,
|
||||||
|
requested_name: Option<&str>,
|
||||||
|
query: Option<&str>,
|
||||||
|
profile_content: bool,
|
||||||
|
options: &ServerOptions,
|
||||||
|
) -> Result<Option<UploaderProfile>> {
|
||||||
|
let first_page_url = self.build_uploader_url(author, 1);
|
||||||
|
let first_page_items = self
|
||||||
|
.fetch_items_for_url(
|
||||||
|
cache.clone(),
|
||||||
|
first_page_url.clone(),
|
||||||
|
64,
|
||||||
|
profile_content,
|
||||||
|
options,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
if first_page_items.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut requester = requester_or_default(options, CHANNEL_ID, "get_uploader.profile_page");
|
||||||
|
let first_page_html = self
|
||||||
|
.fetch_html(&mut requester, &first_page_url, &format!("{}/", self.url))
|
||||||
|
.await?;
|
||||||
|
let last_page = Self::pagination_last_page(&first_page_html).unwrap_or(1);
|
||||||
|
let first_page_size = first_page_items.len() as u64;
|
||||||
|
|
||||||
|
let last_page_items = if last_page > 1 {
|
||||||
|
self.fetch_items_for_url(
|
||||||
|
cache,
|
||||||
|
self.build_uploader_url(author, last_page),
|
||||||
|
64,
|
||||||
|
false,
|
||||||
|
options,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default()
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
};
|
||||||
|
|
||||||
|
let display_name = self.display_name_for_uploader(author, requested_name, &first_page_items);
|
||||||
|
let canonical_id = Self::canonical_uploader_id(author);
|
||||||
|
let mut videos = first_page_items
|
||||||
|
.iter()
|
||||||
|
.map(|item| UploaderVideoRef::from_video_item(item, &display_name, &canonical_id))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let ranked_videos = Self::rank_videos_for_query(&videos, query);
|
||||||
|
let horizontal_ids = ranked_videos
|
||||||
|
.iter()
|
||||||
|
.take(12)
|
||||||
|
.map(|video| video.id.clone())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
let newest_seen = first_page_items
|
||||||
|
.iter()
|
||||||
|
.filter_map(|item| item.uploadedAt)
|
||||||
|
.max();
|
||||||
|
let oldest_seen = last_page_items
|
||||||
|
.iter()
|
||||||
|
.filter_map(|item| item.uploadedAt)
|
||||||
|
.min()
|
||||||
|
.or_else(|| first_page_items.iter().filter_map(|item| item.uploadedAt).min());
|
||||||
|
|
||||||
|
let video_count = if last_page > 1 {
|
||||||
|
((last_page as u64 - 1) * first_page_size) + last_page_items.len() as u64
|
||||||
|
} else {
|
||||||
|
first_page_size
|
||||||
|
};
|
||||||
|
let total_views = first_page_items
|
||||||
|
.iter()
|
||||||
|
.chain(last_page_items.iter())
|
||||||
|
.filter_map(|item| item.views)
|
||||||
|
.map(u64::from)
|
||||||
|
.sum();
|
||||||
|
|
||||||
|
for item in &mut videos {
|
||||||
|
item.uploader = display_name.clone();
|
||||||
|
item.uploaderId = canonical_id.clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
let layout = if horizontal_ids.is_empty() {
|
||||||
|
vec![UploaderLayoutRow::videos(None)]
|
||||||
|
} else {
|
||||||
|
vec![
|
||||||
|
UploaderLayoutRow::horizontal(Some("For You".to_string()), horizontal_ids),
|
||||||
|
UploaderLayoutRow::videos(None),
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(Some(UploaderProfile {
|
||||||
|
id: canonical_id,
|
||||||
|
name: display_name,
|
||||||
|
url: Some(first_page_url),
|
||||||
|
channel: Some(CHANNEL_ID.to_string()),
|
||||||
|
verified: false,
|
||||||
|
videoCount: video_count,
|
||||||
|
totalViews: total_views,
|
||||||
|
channels: Some(vec![UploaderChannelStat {
|
||||||
|
channel: CHANNEL_ID.to_string(),
|
||||||
|
videoCount: video_count,
|
||||||
|
firstSeenAt: crate::uploaders::iso_timestamp_from_unix(oldest_seen),
|
||||||
|
lastSeenAt: crate::uploaders::iso_timestamp_from_unix(newest_seen),
|
||||||
|
}]),
|
||||||
|
avatar: None,
|
||||||
|
description: None,
|
||||||
|
bio: None,
|
||||||
|
videos: profile_content.then_some(videos),
|
||||||
|
tapes: profile_content.then_some(Vec::new()),
|
||||||
|
playlists: profile_content.then_some(Vec::new()),
|
||||||
|
layout: Some(layout),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
fn first_video_link<'a>(&self, element: &'a ElementRef<'a>) -> Result<Option<ElementRef<'a>>> {
|
fn first_video_link<'a>(&self, element: &'a ElementRef<'a>) -> Result<Option<ElementRef<'a>>> {
|
||||||
let selector = Self::selector("a[href]")?;
|
let selector = Self::selector("a[href]")?;
|
||||||
Ok(element.select(&selector).find(|link| {
|
Ok(element.select(&selector).find(|link| {
|
||||||
@@ -638,6 +899,9 @@ impl HsexProvider {
|
|||||||
}
|
}
|
||||||
if let Some(uploader_href) = uploader.value().attr("href") {
|
if let Some(uploader_href) = uploader.value().attr("href") {
|
||||||
item.uploaderUrl = Some(self.absolute_url(uploader_href));
|
item.uploaderUrl = Some(self.absolute_url(uploader_href));
|
||||||
|
item.uploaderId = self
|
||||||
|
.author_from_uploader_href(uploader_href)
|
||||||
|
.map(|author| Self::canonical_uploader_id(&author));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -701,6 +965,9 @@ impl HsexProvider {
|
|||||||
}
|
}
|
||||||
if let Some(href) = author.value().attr("href") {
|
if let Some(href) = author.value().attr("href") {
|
||||||
item.uploaderUrl = Some(self.absolute_url(href));
|
item.uploaderUrl = Some(self.absolute_url(href));
|
||||||
|
item.uploaderId = self
|
||||||
|
.author_from_uploader_href(href)
|
||||||
|
.map(|author| Self::canonical_uploader_id(&author));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -934,6 +1201,34 @@ impl Provider for HsexProvider {
|
|||||||
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
||||||
Some(self.build_channel(clientversion))
|
Some(self.build_channel(clientversion))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn get_uploader(
|
||||||
|
&self,
|
||||||
|
cache: VideoCache,
|
||||||
|
pool: DbPool,
|
||||||
|
uploader_id: Option<String>,
|
||||||
|
uploader_name: Option<String>,
|
||||||
|
query: Option<String>,
|
||||||
|
profile_content: bool,
|
||||||
|
options: ServerOptions,
|
||||||
|
) -> std::result::Result<Option<UploaderProfile>, String> {
|
||||||
|
let _ = pool;
|
||||||
|
let Some(author) =
|
||||||
|
self.resolve_uploader_author(uploader_id.as_deref(), uploader_name.as_deref())
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
self.build_uploader_profile(
|
||||||
|
cache,
|
||||||
|
&author,
|
||||||
|
uploader_name.as_deref(),
|
||||||
|
query.as_deref(),
|
||||||
|
profile_content,
|
||||||
|
&options,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -1020,6 +1315,29 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn canonical_uploader_id_round_trips() {
|
||||||
|
let canonical = HsexProvider::canonical_uploader_id("xihongshiddd");
|
||||||
|
assert_eq!(canonical, "hsex:xihongshiddd");
|
||||||
|
assert_eq!(
|
||||||
|
HsexProvider::author_from_uploader_id(&canonical).as_deref(),
|
||||||
|
Some("xihongshiddd")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_last_page_from_pagination() {
|
||||||
|
let html = r#"
|
||||||
|
<ul class="pagination1">
|
||||||
|
<li><a href="user-1.htm?author=xihongshiddd">1</a></li>
|
||||||
|
<li><a href="user-2.htm?author=xihongshiddd">2</a></li>
|
||||||
|
<li><a href="user-7.htm?author=xihongshiddd">7</a></li>
|
||||||
|
</ul>
|
||||||
|
"#;
|
||||||
|
|
||||||
|
assert_eq!(HsexProvider::pagination_last_page(html), Some(7));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[ignore]
|
#[ignore]
|
||||||
async fn fetches_page_two_items() {
|
async fn fetches_page_two_items() {
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ use crate::{
|
|||||||
DbPool,
|
DbPool,
|
||||||
api::ClientVersion,
|
api::ClientVersion,
|
||||||
status::{Channel, ChannelGroup, ChannelView, FilterOption, Status, StatusResponse},
|
status::{Channel, ChannelGroup, ChannelView, FilterOption, Status, StatusResponse},
|
||||||
|
uploaders::UploaderProfile,
|
||||||
util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester},
|
util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester},
|
||||||
videos::{FlexibleNumber, ServerOptions, VideoItem, VideosRequest},
|
videos::{FlexibleNumber, ServerOptions, VideoItem, VideosRequest},
|
||||||
};
|
};
|
||||||
@@ -577,6 +578,53 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn run_uploader_provider_guarded<F>(
|
||||||
|
provider_name: &str,
|
||||||
|
context: &str,
|
||||||
|
fut: F,
|
||||||
|
) -> Result<Option<UploaderProfile>, String>
|
||||||
|
where
|
||||||
|
F: Future<Output = Result<Option<UploaderProfile>, String>>,
|
||||||
|
{
|
||||||
|
crate::flow_debug!(
|
||||||
|
"provider uploader guard enter provider={} context={}",
|
||||||
|
provider_name,
|
||||||
|
context
|
||||||
|
);
|
||||||
|
match AssertUnwindSafe(fut).catch_unwind().await {
|
||||||
|
Ok(result) => {
|
||||||
|
crate::flow_debug!(
|
||||||
|
"provider uploader guard exit provider={} context={} matched={}",
|
||||||
|
provider_name,
|
||||||
|
context,
|
||||||
|
result.as_ref().ok().and_then(|value| value.as_ref()).is_some()
|
||||||
|
);
|
||||||
|
result
|
||||||
|
}
|
||||||
|
Err(payload) => {
|
||||||
|
let panic_msg = panic_payload_to_string(payload);
|
||||||
|
crate::flow_debug!(
|
||||||
|
"provider uploader guard panic provider={} context={} panic={}",
|
||||||
|
provider_name,
|
||||||
|
context,
|
||||||
|
&panic_msg
|
||||||
|
);
|
||||||
|
let _ = send_discord_error_report(
|
||||||
|
format!("Provider panic: {}", provider_name),
|
||||||
|
None,
|
||||||
|
Some("Provider Guard"),
|
||||||
|
Some(&format!("context={}; panic={}", context, panic_msg)),
|
||||||
|
file!(),
|
||||||
|
line!(),
|
||||||
|
module_path!(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
schedule_provider_validation(provider_name, context, &panic_msg);
|
||||||
|
Err(panic_msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn report_provider_error(provider_name: &str, context: &str, msg: &str) {
|
pub async fn report_provider_error(provider_name: &str, context: &str, msg: &str) {
|
||||||
let _ = send_discord_error_report(
|
let _ = send_discord_error_report(
|
||||||
format!("Provider error: {}", provider_name),
|
format!("Provider error: {}", provider_name),
|
||||||
@@ -868,6 +916,19 @@ pub trait Provider: Send + Sync {
|
|||||||
cacheDuration: None,
|
cacheDuration: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn get_uploader(
|
||||||
|
&self,
|
||||||
|
_cache: VideoCache,
|
||||||
|
_pool: DbPool,
|
||||||
|
_uploader_id: Option<String>,
|
||||||
|
_uploader_name: Option<String>,
|
||||||
|
_query: Option<String>,
|
||||||
|
_profile_content: bool,
|
||||||
|
_options: ServerOptions,
|
||||||
|
) -> Result<Option<UploaderProfile>, String> {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(all(test, not(hottub_single_provider)))]
|
#[cfg(all(test, not(hottub_single_provider)))]
|
||||||
|
|||||||
@@ -474,6 +474,19 @@ impl NoodlemagazineProvider {
|
|||||||
if normalized.is_empty() || !self.is_allowed_thumb_url(&normalized) {
|
if normalized.is_empty() || !self.is_allowed_thumb_url(&normalized) {
|
||||||
return String::new();
|
return String::new();
|
||||||
}
|
}
|
||||||
|
let Some(url) = Url::parse(&normalized).ok() else {
|
||||||
|
return String::new();
|
||||||
|
};
|
||||||
|
if url
|
||||||
|
.host_str()
|
||||||
|
.is_some_and(|host| host.eq_ignore_ascii_case("img.pvvstream.pro"))
|
||||||
|
{
|
||||||
|
return crate::providers::build_proxy_url(
|
||||||
|
_options,
|
||||||
|
"noodlemagazine-thumb",
|
||||||
|
&crate::providers::strip_url_scheme(&normalized),
|
||||||
|
);
|
||||||
|
}
|
||||||
normalized
|
normalized
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -707,7 +720,7 @@ mod tests {
|
|||||||
assert_eq!(items.len(), 1);
|
assert_eq!(items.len(), 1);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
items[0].thumb,
|
items[0].thumb,
|
||||||
"https://img.pvvstream.pro/preview/abc/-111_222/240/iv.okcdn.ru/getVideoPreview?id=1&type=39&fn=vid_l"
|
"https://example.com/proxy/noodlemagazine-thumb/img.pvvstream.pro/preview/abc/-111_222/240/iv.okcdn.ru/getVideoPreview?id=1&type=39&fn=vid_l"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
216
src/uploaders.rs
Normal file
216
src/uploaders.rs
Normal file
@@ -0,0 +1,216 @@
|
|||||||
|
use chrono::{SecondsFormat, TimeZone, Utc};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::videos::VideoItem;
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||||
|
pub struct UploadersRequest {
|
||||||
|
#[serde(default, alias = "uploader_id")]
|
||||||
|
pub uploaderId: Option<String>,
|
||||||
|
#[serde(default, alias = "uploader_name")]
|
||||||
|
pub uploaderName: Option<String>,
|
||||||
|
#[serde(default, alias = "profile_content")]
|
||||||
|
pub profileContent: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
pub query: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl UploadersRequest {
|
||||||
|
pub fn normalized(self) -> Self {
|
||||||
|
Self {
|
||||||
|
uploaderId: normalize_optional_string(self.uploaderId),
|
||||||
|
uploaderName: normalize_optional_string(self.uploaderName),
|
||||||
|
profileContent: self.profileContent,
|
||||||
|
query: normalize_optional_string(self.query),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||||
|
pub struct UploaderProfile {
|
||||||
|
pub id: String,
|
||||||
|
pub name: String,
|
||||||
|
pub url: Option<String>,
|
||||||
|
pub channel: Option<String>,
|
||||||
|
pub verified: bool,
|
||||||
|
pub videoCount: u64,
|
||||||
|
pub totalViews: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub channels: Option<Vec<UploaderChannelStat>>,
|
||||||
|
#[serde(default, alias = "profile_picture_url")]
|
||||||
|
pub avatar: Option<String>,
|
||||||
|
pub description: Option<String>,
|
||||||
|
pub bio: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub videos: Option<Vec<UploaderVideoRef>>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub tapes: Option<Vec<serde_json::Value>>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub playlists: Option<Vec<serde_json::Value>>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub layout: Option<Vec<UploaderLayoutRow>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||||
|
pub struct UploaderChannelStat {
|
||||||
|
pub channel: String,
|
||||||
|
pub videoCount: u64,
|
||||||
|
#[serde(default, alias = "first_seen_at")]
|
||||||
|
pub firstSeenAt: Option<String>,
|
||||||
|
#[serde(default, alias = "last_seen_at")]
|
||||||
|
pub lastSeenAt: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||||
|
pub struct UploaderVideoRef {
|
||||||
|
pub id: String,
|
||||||
|
pub url: String,
|
||||||
|
pub title: String,
|
||||||
|
pub duration: u32,
|
||||||
|
pub channel: String,
|
||||||
|
#[serde(default, alias = "uploaded_at")]
|
||||||
|
pub uploadedAt: Option<String>,
|
||||||
|
pub uploader: String,
|
||||||
|
#[serde(alias = "uploader_id")]
|
||||||
|
pub uploaderId: String,
|
||||||
|
pub thumb: String,
|
||||||
|
pub preview: Option<String>,
|
||||||
|
pub views: u32,
|
||||||
|
pub rating: u32,
|
||||||
|
#[serde(default, alias = "aspect_ratio")]
|
||||||
|
pub aspectRatio: Option<f32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl UploaderVideoRef {
|
||||||
|
pub fn from_video_item(item: &VideoItem, uploader_name: &str, uploader_id: &str) -> Self {
|
||||||
|
Self {
|
||||||
|
id: item.id.clone(),
|
||||||
|
url: item.url.clone(),
|
||||||
|
title: item.title.clone(),
|
||||||
|
duration: item.duration,
|
||||||
|
channel: item.channel.clone(),
|
||||||
|
uploadedAt: iso_timestamp_from_unix(item.uploadedAt),
|
||||||
|
uploader: item
|
||||||
|
.uploader
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| uploader_name.to_string()),
|
||||||
|
uploaderId: item
|
||||||
|
.uploaderId
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| uploader_id.to_string()),
|
||||||
|
thumb: item.thumb.clone(),
|
||||||
|
preview: item.preview.clone(),
|
||||||
|
views: item.views.unwrap_or_default(),
|
||||||
|
rating: item.rating.map(normalize_rating).unwrap_or_default(),
|
||||||
|
aspectRatio: item.aspectRatio,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||||
|
pub struct UploaderLayoutRow {
|
||||||
|
#[serde(rename = "type")]
|
||||||
|
pub rowType: UploaderLayoutRowType,
|
||||||
|
pub title: Option<String>,
|
||||||
|
#[serde(default, alias = "video_ids")]
|
||||||
|
pub videoIds: Option<Vec<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl UploaderLayoutRow {
|
||||||
|
pub fn horizontal(title: Option<String>, video_ids: Vec<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
rowType: UploaderLayoutRowType::Horizontal,
|
||||||
|
title,
|
||||||
|
videoIds: Some(video_ids),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn videos(title: Option<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
rowType: UploaderLayoutRowType::Videos,
|
||||||
|
title,
|
||||||
|
videoIds: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
|
||||||
|
pub enum UploaderLayoutRowType {
|
||||||
|
#[default]
|
||||||
|
#[serde(rename = "videos")]
|
||||||
|
Videos,
|
||||||
|
#[serde(rename = "horizontal", alias = "horizontal_videos")]
|
||||||
|
Horizontal,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn normalize_optional_string(value: Option<String>) -> Option<String> {
|
||||||
|
value.and_then(|value| {
|
||||||
|
let trimmed = value.trim();
|
||||||
|
(!trimmed.is_empty()).then(|| trimmed.to_string())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn iso_timestamp_from_unix(value: Option<u64>) -> Option<String> {
|
||||||
|
let timestamp = value?;
|
||||||
|
let dt = Utc.timestamp_opt(timestamp as i64, 0).single()?;
|
||||||
|
Some(dt.to_rfc3339_opts(SecondsFormat::Millis, true))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_rating(value: f32) -> u32 {
|
||||||
|
value.clamp(0.0, 100.0).round() as u32
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn request_accepts_snake_case_aliases() {
|
||||||
|
let request: UploadersRequest = serde_json::from_str(
|
||||||
|
r#"{
|
||||||
|
"uploader_id": "hsex:xihongshiddd",
|
||||||
|
"uploader_name": "xihongshiddd",
|
||||||
|
"profile_content": true,
|
||||||
|
"query": "teacher"
|
||||||
|
}"#,
|
||||||
|
)
|
||||||
|
.expect("request should decode");
|
||||||
|
|
||||||
|
assert_eq!(request.uploaderId.as_deref(), Some("hsex:xihongshiddd"));
|
||||||
|
assert_eq!(request.uploaderName.as_deref(), Some("xihongshiddd"));
|
||||||
|
assert!(request.profileContent);
|
||||||
|
assert_eq!(request.query.as_deref(), Some("teacher"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn layout_aliases_decode() {
|
||||||
|
let row: UploaderLayoutRow = serde_json::from_str(
|
||||||
|
r#"{
|
||||||
|
"type": "horizontal_videos",
|
||||||
|
"title": "For You",
|
||||||
|
"video_ids": ["one", "two"]
|
||||||
|
}"#,
|
||||||
|
)
|
||||||
|
.expect("row should decode");
|
||||||
|
|
||||||
|
assert_eq!(row.rowType, UploaderLayoutRowType::Horizontal);
|
||||||
|
assert_eq!(row.videoIds.as_ref().map(Vec::len), Some(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn avatar_alias_decodes() {
|
||||||
|
let profile: UploaderProfile = serde_json::from_str(
|
||||||
|
r#"{
|
||||||
|
"id": "abc",
|
||||||
|
"name": "Example",
|
||||||
|
"verified": false,
|
||||||
|
"videoCount": 1,
|
||||||
|
"totalViews": 2,
|
||||||
|
"profile_picture_url": "https://example.com/a.jpg"
|
||||||
|
}"#,
|
||||||
|
)
|
||||||
|
.expect("profile should decode");
|
||||||
|
|
||||||
|
assert_eq!(profile.avatar.as_deref(), Some("https://example.com/a.jpg"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -100,6 +100,8 @@ pub struct VideoItem {
|
|||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub uploaderUrl: Option<String>, // "https://www.youtube.com/@petcollective",
|
pub uploaderUrl: Option<String>, // "https://www.youtube.com/@petcollective",
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub uploaderId: Option<String>, // "petcollective",
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub verified: Option<bool>, // false,
|
pub verified: Option<bool>, // false,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub tags: Option<Vec<String>>, // [],
|
pub tags: Option<Vec<String>>, // [],
|
||||||
@@ -135,6 +137,7 @@ impl VideoItem {
|
|||||||
thumb,
|
thumb,
|
||||||
uploader: None,
|
uploader: None,
|
||||||
uploaderUrl: None,
|
uploaderUrl: None,
|
||||||
|
uploaderId: None,
|
||||||
verified: None,
|
verified: None,
|
||||||
tags: None, // Placeholder, adjust as needed
|
tags: None, // Placeholder, adjust as needed
|
||||||
uploadedAt: None,
|
uploadedAt: None,
|
||||||
|
|||||||
Reference in New Issue
Block a user