Raw requester calls did not adopt the remembered FlareSolverr user agent, so a RedGifs token minted under it was rejected on every gif lookup. Tag/creator pages that mix non-RedGifs sources are refilled from later cursor batches instead of coming back empty. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi
899 lines
30 KiB
Rust
899 lines
30 KiB
Rust
use crate::DbPool;
|
|
use crate::api::ClientVersion;
|
|
use crate::providers::{Provider, report_provider_error, report_provider_error_background, requester_or_default};
|
|
use crate::status::*;
|
|
use crate::uploaders::{UploaderChannelStat, UploaderLayoutRow, UploaderProfile, UploaderVideoRef};
|
|
use crate::util::cache::VideoCache;
|
|
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
|
|
use async_trait::async_trait;
|
|
use chrono::DateTime;
|
|
use error_chain::error_chain;
|
|
use futures::stream::{self, StreamExt};
|
|
use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
|
|
use serde::Deserialize;
|
|
use std::sync::Arc;
|
|
use tokio::sync::RwLock;
|
|
|
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
|
crate::providers::ProviderChannelMetadata {
|
|
group_id: "tiktok",
|
|
tags: &["shortform", "redgifs", "amateur", "swipe"],
|
|
};
|
|
|
|
const BASE_URL: &str = "https://xxxtik.com";
|
|
const API_BASE: &str = "https://xxxtik-api-iw98m.ondigitalocean.app";
|
|
const REDGIFS_API: &str = "https://api.redgifs.com/v2";
|
|
const CHANNEL_ID: &str = "xxxtik";
|
|
const DEFAULT_PER_PAGE: usize = 20;
|
|
const MAX_PER_PAGE: usize = 30;
|
|
const RESOLVE_CONCURRENCY: usize = 8;
|
|
// xxxtik's listing endpoints are cursor-based (the numeric `id` of the last item
|
|
// returned), not page-number based, so reaching page N requires walking N
|
|
// sequential requests from the start of the feed.
|
|
const MAX_REFILL_FETCHES: usize = 4;
|
|
const MAX_PAGE_WALK: u16 = 25;
|
|
|
|
// xxxtik exposes ~60k tags through /tag/options, far too many to background-load
|
|
// as a filter catalog. This is a small curated set of popular tags for the
|
|
// `categories` shortcut option; any other tag still works via a `tag:`/`cat:`
|
|
// query prefix or the live `/search` lookup.
|
|
const CURATED_CATEGORIES: &[&str] = &[
|
|
"amateur",
|
|
"asian",
|
|
"anal",
|
|
"ass",
|
|
"big tits",
|
|
"blowjob",
|
|
"brunette",
|
|
"cumshot",
|
|
"hardcore",
|
|
"lesbian",
|
|
"milf",
|
|
"pussy",
|
|
"redhead",
|
|
"teen (18+)",
|
|
"public sex",
|
|
"latina",
|
|
"ebony",
|
|
"squirting",
|
|
"creampie",
|
|
"threesome",
|
|
];
|
|
|
|
error_chain! {
|
|
foreign_links {
|
|
Json(serde_json::Error);
|
|
}
|
|
errors {
|
|
Request(msg: String) {
|
|
description("request error")
|
|
display("request error: {}", msg)
|
|
}
|
|
NotFound(msg: String) {
|
|
description("not found")
|
|
display("not found: {}", msg)
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
enum Sort {
|
|
New,
|
|
TopWeek,
|
|
TopMonth,
|
|
TopYear,
|
|
TopAll,
|
|
}
|
|
|
|
impl Sort {
|
|
fn from_sort_id(value: &str) -> Self {
|
|
match value.trim().to_ascii_lowercase().as_str() {
|
|
"top" | "top_week" | "week" => Sort::TopWeek,
|
|
"top_month" | "month" => Sort::TopMonth,
|
|
"top_year" | "year" => Sort::TopYear,
|
|
"top_all" | "all" | "alltime" | "all_time" => Sort::TopAll,
|
|
_ => Sort::New,
|
|
}
|
|
}
|
|
|
|
fn list_path(self) -> &'static str {
|
|
match self {
|
|
Sort::New => "post/new",
|
|
Sort::TopWeek => "post/top/week",
|
|
Sort::TopMonth => "post/top/month",
|
|
Sort::TopYear => "post/top/year",
|
|
Sort::TopAll => "post/top/all",
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
enum Target {
|
|
Feed(Sort),
|
|
Tag(String),
|
|
Creator(String),
|
|
Empty,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize, Clone, Default)]
|
|
#[serde(rename_all = "camelCase")]
|
|
struct Post {
|
|
id: i64,
|
|
#[serde(default)]
|
|
uuid: String,
|
|
#[serde(default)]
|
|
description: Option<String>,
|
|
#[serde(default)]
|
|
source: Option<String>,
|
|
#[serde(default)]
|
|
status: Option<String>,
|
|
#[serde(default)]
|
|
created_at: String,
|
|
#[serde(default)]
|
|
author: PostAuthor,
|
|
#[serde(default)]
|
|
tags: Vec<PostTag>,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize, Clone, Default)]
|
|
struct PostAuthor {
|
|
#[serde(default)]
|
|
name: String,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize, Clone, Default)]
|
|
struct PostTag {
|
|
#[serde(default)]
|
|
name: String,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize, Clone, Default)]
|
|
struct SearchResult {
|
|
#[serde(default)]
|
|
name: String,
|
|
#[serde(default, rename = "type")]
|
|
kind: String,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize, Clone, Default)]
|
|
struct UserProfileResp {
|
|
#[serde(default)]
|
|
name: String,
|
|
#[serde(default)]
|
|
description: Option<String>,
|
|
#[serde(default, rename = "_count")]
|
|
count: Option<UserCount>,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize, Clone, Default)]
|
|
struct UserCount {
|
|
#[serde(default)]
|
|
posts: u64,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize, Clone, Default)]
|
|
struct TokenResponse {
|
|
#[serde(default)]
|
|
token: String,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize, Clone, Default)]
|
|
struct RedgifsResponse {
|
|
#[serde(default)]
|
|
gif: RedgifsGif,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize, Clone, Default)]
|
|
struct RedgifsGif {
|
|
#[serde(default)]
|
|
duration: f64,
|
|
#[serde(default)]
|
|
views: u32,
|
|
#[serde(default)]
|
|
width: u32,
|
|
#[serde(default)]
|
|
height: u32,
|
|
#[serde(default)]
|
|
tags: Vec<String>,
|
|
#[serde(default)]
|
|
#[serde(rename = "userName")]
|
|
user_name: String,
|
|
#[serde(default)]
|
|
urls: RedgifsUrls,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize, Clone, Default)]
|
|
struct RedgifsUrls {
|
|
#[serde(default)]
|
|
hd: Option<String>,
|
|
#[serde(default)]
|
|
sd: Option<String>,
|
|
#[serde(default)]
|
|
poster: Option<String>,
|
|
#[serde(default)]
|
|
thumbnail: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct XxxtikProvider {
|
|
redgifs_token: Arc<RwLock<Option<String>>>,
|
|
}
|
|
|
|
impl XxxtikProvider {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
redgifs_token: Arc::new(RwLock::new(None)),
|
|
}
|
|
}
|
|
|
|
fn title_case(value: &str) -> String {
|
|
value
|
|
.split(' ')
|
|
.map(|word| {
|
|
let mut chars = word.chars();
|
|
match chars.next() {
|
|
Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
|
|
None => String::new(),
|
|
}
|
|
})
|
|
.collect::<Vec<_>>()
|
|
.join(" ")
|
|
}
|
|
|
|
fn normalize_tag(value: &str) -> String {
|
|
value.trim().to_ascii_lowercase()
|
|
}
|
|
|
|
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
|
let category_titles = CURATED_CATEGORIES
|
|
.iter()
|
|
.map(|name| Self::title_case(name))
|
|
.collect::<Vec<_>>();
|
|
let category_options = CURATED_CATEGORIES
|
|
.iter()
|
|
.map(|name| FilterOption {
|
|
id: name.to_string(),
|
|
title: Self::title_case(name),
|
|
})
|
|
.collect::<Vec<_>>();
|
|
|
|
Channel {
|
|
id: CHANNEL_ID.to_string(),
|
|
name: "xxxtik".to_string(),
|
|
description: "xxxtik short-form TikTok-style clips, an aggregator of moderated RedGifs reposts with tag and creator browsing.".to_string(),
|
|
premium: false,
|
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=xxxtik.com".to_string(),
|
|
status: "active".to_string(),
|
|
categories: category_titles,
|
|
options: vec![
|
|
ChannelOption {
|
|
id: "sort".to_string(),
|
|
title: "Sort".to_string(),
|
|
description: "Browse xxxtik by latest or top-rated over a time period.".to_string(),
|
|
systemImage: "arrow.up.arrow.down".to_string(),
|
|
colorName: "blue".to_string(),
|
|
options: vec![
|
|
FilterOption {
|
|
id: "new".to_string(),
|
|
title: "Latest".to_string(),
|
|
},
|
|
FilterOption {
|
|
id: "top_week".to_string(),
|
|
title: "Top This Week".to_string(),
|
|
},
|
|
FilterOption {
|
|
id: "top_month".to_string(),
|
|
title: "Top This Month".to_string(),
|
|
},
|
|
FilterOption {
|
|
id: "top_year".to_string(),
|
|
title: "Top This Year".to_string(),
|
|
},
|
|
FilterOption {
|
|
id: "top_all".to_string(),
|
|
title: "Top All Time".to_string(),
|
|
},
|
|
],
|
|
multiSelect: false,
|
|
},
|
|
ChannelOption {
|
|
id: "categories".to_string(),
|
|
title: "Tags".to_string(),
|
|
description: "Open a popular xxxtik tag feed directly.".to_string(),
|
|
systemImage: "tag".to_string(),
|
|
colorName: "orange".to_string(),
|
|
options: category_options,
|
|
multiSelect: false,
|
|
},
|
|
],
|
|
nsfw: true,
|
|
cacheDuration: Some(1800),
|
|
ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string())
|
|
}
|
|
}
|
|
|
|
fn resolve_query_target(query: &str) -> Option<Target> {
|
|
let trimmed = query.trim();
|
|
let (kind, value) = trimmed.split_once(':')?;
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
return None;
|
|
}
|
|
match kind.trim().to_ascii_lowercase().as_str() {
|
|
"tag" | "category" | "cat" => Some(Target::Tag(Self::normalize_tag(value))),
|
|
"user" | "uploader" | "creator" => Some(Target::Creator(value.to_string())),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
async fn search_target(&self, query: &str, options: &ServerOptions) -> Result<Target> {
|
|
let encoded = utf8_percent_encode(query.trim(), NON_ALPHANUMERIC).to_string();
|
|
let url = format!("{API_BASE}/search?query={encoded}");
|
|
let mut requester = requester_or_default(options, CHANNEL_ID, "search_target");
|
|
let text = requester
|
|
.get_with_headers(&url, vec![("Referer".to_string(), format!("{BASE_URL}/"))], None)
|
|
.await
|
|
.map_err(|error| Error::from(format!("request failed for {url}: {error}")))?;
|
|
let results: Vec<SearchResult> = serde_json::from_str(&text)?;
|
|
|
|
let normalized_query = query.trim().to_ascii_lowercase();
|
|
|
|
if let Some(exact_tag) = results
|
|
.iter()
|
|
.find(|item| item.kind == "tag" && item.name.to_ascii_lowercase() == normalized_query)
|
|
{
|
|
return Ok(Target::Tag(Self::normalize_tag(&exact_tag.name)));
|
|
}
|
|
if let Some(exact_profile) = results
|
|
.iter()
|
|
.find(|item| item.kind == "profile" && item.name.to_ascii_lowercase() == normalized_query)
|
|
{
|
|
return Ok(Target::Creator(exact_profile.name.clone()));
|
|
}
|
|
if let Some(first_tag) = results.iter().find(|item| item.kind == "tag") {
|
|
return Ok(Target::Tag(Self::normalize_tag(&first_tag.name)));
|
|
}
|
|
if let Some(first_profile) = results.iter().find(|item| item.kind == "profile") {
|
|
return Ok(Target::Creator(first_profile.name.clone()));
|
|
}
|
|
|
|
Ok(Target::Empty)
|
|
}
|
|
|
|
async fn pick_target(&self, query: Option<&str>, sort: &str, options: &ServerOptions) -> Result<Target> {
|
|
if let Some(query) = query {
|
|
if !query.trim().is_empty() {
|
|
if let Some(target) = Self::resolve_query_target(query) {
|
|
return Ok(target);
|
|
}
|
|
return self.search_target(query, options).await;
|
|
}
|
|
}
|
|
|
|
if let Some(category) = options.categories.as_deref() {
|
|
let category = category.trim();
|
|
if !category.is_empty() && category != "all" {
|
|
return Ok(Target::Tag(Self::normalize_tag(category)));
|
|
}
|
|
}
|
|
|
|
Ok(Target::Feed(Sort::from_sort_id(sort)))
|
|
}
|
|
|
|
fn parse_timestamp(value: &str) -> Option<u64> {
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
return None;
|
|
}
|
|
DateTime::parse_from_rfc3339(value)
|
|
.ok()
|
|
.map(|parsed| parsed.timestamp().max(0) as u64)
|
|
}
|
|
|
|
fn extract_gif_id(source: &str) -> Option<String> {
|
|
let without_query = source.split(['?', '#']).next().unwrap_or(source);
|
|
let id = without_query.trim_end_matches('/').rsplit('/').next()?;
|
|
if id.is_empty() { None } else { Some(id.to_string()) }
|
|
}
|
|
|
|
async fn fetch_one_page(
|
|
&self,
|
|
path: &str,
|
|
cursor: i64,
|
|
per_page: usize,
|
|
options: &ServerOptions,
|
|
) -> Result<Vec<Post>> {
|
|
let url = format!("{API_BASE}/{path}?limit={per_page}&cursor={cursor}");
|
|
let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_one_page");
|
|
let text = requester
|
|
.get_with_headers(&url, vec![("Referer".to_string(), format!("{BASE_URL}/"))], None)
|
|
.await
|
|
.map_err(|error| Error::from(format!("request failed for {url}: {error}")))?;
|
|
Ok(serde_json::from_str(&text)?)
|
|
}
|
|
|
|
async fn fetch_cursor_page(
|
|
&self,
|
|
path: &str,
|
|
page: u16,
|
|
per_page: usize,
|
|
options: &ServerOptions,
|
|
) -> Result<Vec<Post>> {
|
|
let page = page.max(1).min(MAX_PAGE_WALK);
|
|
let mut cursor: i64 = 0;
|
|
let mut items = Vec::new();
|
|
|
|
for _ in 0..page {
|
|
// Tag and creator pages mix in posts whose source is not RedGifs
|
|
// (erome links, empty sources); those cannot be resolved here and
|
|
// used to leave whole pages empty ("girl" returned nothing). Keep
|
|
// pulling raw batches, bounded, until a page has enough
|
|
// resolvable posts.
|
|
let mut collected: Vec<Post> = Vec::new();
|
|
let mut exhausted = false;
|
|
for _ in 0..MAX_REFILL_FETCHES {
|
|
let batch = self.fetch_one_page(path, cursor, per_page, options).await?;
|
|
if batch.is_empty() {
|
|
exhausted = true;
|
|
break;
|
|
}
|
|
cursor = batch.last().map(|post| post.id).unwrap_or(cursor);
|
|
let short_batch = batch.len() < per_page;
|
|
collected.extend(batch.into_iter().filter(Self::is_redgifs_post));
|
|
if collected.len() >= per_page || short_batch {
|
|
break;
|
|
}
|
|
}
|
|
if exhausted && collected.is_empty() {
|
|
items = Vec::new();
|
|
break;
|
|
}
|
|
items = collected;
|
|
}
|
|
|
|
Ok(items)
|
|
}
|
|
|
|
fn is_redgifs_post(post: &Post) -> bool {
|
|
post.source
|
|
.as_deref()
|
|
.map(|source| source.contains("redgifs.com/"))
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
async fn fetch_target_posts(
|
|
&self,
|
|
target: &Target,
|
|
page: u16,
|
|
per_page: usize,
|
|
options: &ServerOptions,
|
|
) -> Result<Vec<Post>> {
|
|
match target {
|
|
Target::Feed(sort) => self.fetch_cursor_page(sort.list_path(), page, per_page, options).await,
|
|
Target::Tag(name) => {
|
|
let encoded = utf8_percent_encode(name, NON_ALPHANUMERIC).to_string();
|
|
let path = format!("post/tag/{encoded}");
|
|
self.fetch_cursor_page(&path, page, per_page, options).await
|
|
}
|
|
Target::Creator(name) => {
|
|
let encoded = utf8_percent_encode(name, NON_ALPHANUMERIC).to_string();
|
|
let path = format!("post/creator/{encoded}");
|
|
self.fetch_cursor_page(&path, page, per_page, options).await
|
|
}
|
|
Target::Empty => Ok(Vec::new()),
|
|
}
|
|
}
|
|
|
|
async fn redgifs_token(&self, options: &ServerOptions, force_refresh: bool) -> Result<String> {
|
|
if !force_refresh {
|
|
if let Some(token) = self.redgifs_token.read().await.clone() {
|
|
return Ok(token);
|
|
}
|
|
}
|
|
|
|
let mut guard = self.redgifs_token.write().await;
|
|
if !force_refresh {
|
|
if let Some(token) = guard.clone() {
|
|
return Ok(token);
|
|
}
|
|
}
|
|
|
|
let url = format!("{REDGIFS_API}/auth/temporary");
|
|
let mut requester = requester_or_default(options, CHANNEL_ID, "redgifs_token");
|
|
let text = requester
|
|
.get_with_headers(&url, vec![("Referer".to_string(), format!("{BASE_URL}/"))], None)
|
|
.await
|
|
.map_err(|error| Error::from(format!("request failed for {url}: {error}")))?;
|
|
let parsed: TokenResponse = serde_json::from_str(&text)?;
|
|
if parsed.token.is_empty() {
|
|
return Err(Error::from(format!("empty redgifs token response for {url}")));
|
|
}
|
|
*guard = Some(parsed.token.clone());
|
|
Ok(parsed.token)
|
|
}
|
|
|
|
async fn resolve_redgifs(&self, gif_id: &str, options: &ServerOptions) -> Result<RedgifsGif> {
|
|
for attempt in 0..2 {
|
|
let force_refresh = attempt > 0;
|
|
let token = self.redgifs_token(options, force_refresh).await?;
|
|
let url = format!("{REDGIFS_API}/gifs/{gif_id}");
|
|
let mut requester = requester_or_default(options, CHANNEL_ID, "resolve_redgifs");
|
|
let response = requester
|
|
.get_raw_with_headers(
|
|
&url,
|
|
vec![
|
|
("Authorization".to_string(), format!("Bearer {token}")),
|
|
("Referer".to_string(), "https://www.redgifs.com/".to_string()),
|
|
],
|
|
)
|
|
.await
|
|
.map_err(|error| Error::from(format!("request failed for {url}: {error}")))?;
|
|
|
|
if response.status().as_u16() == 401 && attempt == 0 {
|
|
continue;
|
|
}
|
|
if !response.status().is_success() {
|
|
return Err(Error::from(format!(
|
|
"redgifs resolve failed for {gif_id}: status={}",
|
|
response.status()
|
|
)));
|
|
}
|
|
let text = response
|
|
.text()
|
|
.await
|
|
.map_err(|error| Error::from(format!("response read failed for {url}: {error}")))?;
|
|
let parsed: RedgifsResponse = serde_json::from_str(&text)?;
|
|
return Ok(parsed.gif);
|
|
}
|
|
Err(Error::from(format!("redgifs resolve failed for {gif_id}: exhausted retries")))
|
|
}
|
|
|
|
async fn build_video_item(&self, post: Post, options: &ServerOptions) -> Option<VideoItem> {
|
|
if let Some(status) = post.status.as_deref() {
|
|
let status = status.trim().to_ascii_lowercase();
|
|
if !status.is_empty() && status != "approved" {
|
|
return None;
|
|
}
|
|
}
|
|
if post.uuid.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
let source = post.source.as_deref()?;
|
|
let gif_id = Self::extract_gif_id(source)?;
|
|
let gif = match self.resolve_redgifs(&gif_id, options).await {
|
|
Ok(gif) => gif,
|
|
Err(error) => {
|
|
report_provider_error_background(CHANNEL_ID, "build_video_item.resolve_redgifs", &error.to_string());
|
|
return None;
|
|
}
|
|
};
|
|
|
|
let media_url = gif.urls.hd.clone().or_else(|| gif.urls.sd.clone())?;
|
|
let thumb = gif.urls.poster.clone().or_else(|| gif.urls.thumbnail.clone())?;
|
|
|
|
let title = post
|
|
.description
|
|
.as_deref()
|
|
.map(str::trim)
|
|
.filter(|value| !value.is_empty())
|
|
.map(ToOwned::to_owned)
|
|
.unwrap_or_else(|| {
|
|
if !post.author.name.trim().is_empty() {
|
|
format!("xxxtik clip by {}", post.author.name.trim())
|
|
} else {
|
|
format!("xxxtik clip {}", post.uuid)
|
|
}
|
|
});
|
|
|
|
let url = format!("{BASE_URL}/feed/{}", post.uuid);
|
|
let duration = gif.duration.round().max(0.0) as u32;
|
|
|
|
let mut item = VideoItem::new(post.uuid.clone(), title, url, CHANNEL_ID.to_string(), thumb, duration);
|
|
item.views = Some(gif.views);
|
|
item.uploadedAt = Self::parse_timestamp(&post.created_at);
|
|
|
|
let uploader_name = if !post.author.name.trim().is_empty() {
|
|
post.author.name.trim().to_string()
|
|
} else {
|
|
gif.user_name.trim().to_string()
|
|
};
|
|
if !uploader_name.is_empty() {
|
|
item.uploader = Some(uploader_name.clone());
|
|
item.uploaderUrl = Some(format!("{BASE_URL}/profile/{uploader_name}"));
|
|
item.uploaderId = Some(format!("{CHANNEL_ID}:{uploader_name}"));
|
|
}
|
|
|
|
let mut tags = post
|
|
.tags
|
|
.iter()
|
|
.map(|tag| tag.name.clone())
|
|
.filter(|name| !name.trim().is_empty())
|
|
.collect::<Vec<_>>();
|
|
for tag in &gif.tags {
|
|
let lower = tag.to_ascii_lowercase();
|
|
if !tags.iter().any(|existing| existing.to_ascii_lowercase() == lower) {
|
|
tags.push(tag.clone());
|
|
}
|
|
}
|
|
item.tags = (!tags.is_empty()).then_some(tags);
|
|
|
|
if gif.height > 0 {
|
|
item.aspectRatio = Some(gif.width as f32 / gif.height as f32);
|
|
}
|
|
|
|
item.formats = Some(vec![VideoFormat::new(media_url, "hd".to_string(), "mp4".to_string())]);
|
|
|
|
Some(item)
|
|
}
|
|
|
|
async fn fetch_target_items(
|
|
&self,
|
|
target: Target,
|
|
page: u16,
|
|
per_page: usize,
|
|
options: &ServerOptions,
|
|
) -> Result<Vec<VideoItem>> {
|
|
let posts = self.fetch_target_posts(&target, page, per_page, options).await?;
|
|
|
|
let items = stream::iter(posts.into_iter().map(|post| {
|
|
let options = options.clone();
|
|
async move { self.build_video_item(post, &options).await }
|
|
}))
|
|
.buffer_unordered(RESOLVE_CONCURRENCY)
|
|
.filter_map(async move |v| v)
|
|
.collect::<Vec<_>>()
|
|
.await;
|
|
|
|
Ok(items)
|
|
}
|
|
|
|
async fn fetch_user_profile(&self, username: &str, options: &ServerOptions) -> Result<Option<UserProfileResp>> {
|
|
let encoded = utf8_percent_encode(username, NON_ALPHANUMERIC).to_string();
|
|
let url = format!("{API_BASE}/user/by-username/{encoded}");
|
|
let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_user_profile");
|
|
let text = match requester
|
|
.get_with_headers(&url, vec![("Referer".to_string(), format!("{BASE_URL}/"))], None)
|
|
.await
|
|
{
|
|
Ok(text) => text,
|
|
Err(_) => return Ok(None),
|
|
};
|
|
match serde_json::from_str::<UserProfileResp>(&text) {
|
|
Ok(profile) if !profile.name.trim().is_empty() => Ok(Some(profile)),
|
|
_ => Ok(None),
|
|
}
|
|
}
|
|
|
|
async fn build_uploader_profile(
|
|
&self,
|
|
username: &str,
|
|
query: Option<&str>,
|
|
profile_content: bool,
|
|
options: &ServerOptions,
|
|
) -> Result<Option<UploaderProfile>> {
|
|
let Some(profile) = self.fetch_user_profile(username, options).await? else {
|
|
return Ok(None);
|
|
};
|
|
|
|
let canonical_id = format!("{CHANNEL_ID}:{}", profile.name);
|
|
let video_count = profile.count.as_ref().map(|count| count.posts).unwrap_or(0);
|
|
let mut videos = None;
|
|
let mut total_views: u64 = 0;
|
|
|
|
if profile_content {
|
|
let items = self
|
|
.fetch_target_items(Target::Creator(profile.name.clone()), 1, 24, options)
|
|
.await?;
|
|
|
|
let filtered_items = if let Some(query) = query.filter(|value| !value.trim().is_empty()) {
|
|
let normalized_query = query.to_ascii_lowercase();
|
|
items
|
|
.into_iter()
|
|
.filter(|item| {
|
|
let haystack = format!(
|
|
"{} {}",
|
|
item.title,
|
|
item.tags.as_ref().map(|values| values.join(" ")).unwrap_or_default()
|
|
)
|
|
.to_ascii_lowercase();
|
|
haystack.contains(&normalized_query)
|
|
})
|
|
.collect::<Vec<_>>()
|
|
} else {
|
|
items
|
|
};
|
|
|
|
total_views = filtered_items.iter().filter_map(|item| item.views).map(|v| v as u64).sum();
|
|
|
|
let refs = filtered_items
|
|
.iter()
|
|
.map(|item| UploaderVideoRef::from_video_item(item, &profile.name, &canonical_id))
|
|
.collect::<Vec<_>>();
|
|
videos = Some(refs);
|
|
}
|
|
|
|
Ok(Some(UploaderProfile {
|
|
id: canonical_id,
|
|
name: profile.name.clone(),
|
|
url: Some(format!("{BASE_URL}/profile/{}", profile.name)),
|
|
channel: Some(CHANNEL_ID.to_string()),
|
|
verified: false,
|
|
videoCount: video_count,
|
|
totalViews: total_views,
|
|
channels: Some(vec![UploaderChannelStat {
|
|
channel: CHANNEL_ID.to_string(),
|
|
videoCount: video_count,
|
|
firstSeenAt: None,
|
|
lastSeenAt: None,
|
|
}]),
|
|
avatar: None,
|
|
description: profile.description.clone(),
|
|
bio: profile.description,
|
|
videos,
|
|
tapes: Some(vec![]),
|
|
playlists: Some(vec![]),
|
|
layout: Some(vec![UploaderLayoutRow::videos(Some("Posts".to_string()))]),
|
|
}))
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl Provider for XxxtikProvider {
|
|
async fn get_videos(
|
|
&self,
|
|
cache: VideoCache,
|
|
pool: DbPool,
|
|
sort: String,
|
|
query: Option<String>,
|
|
page: String,
|
|
per_page: String,
|
|
options: ServerOptions,
|
|
) -> Vec<VideoItem> {
|
|
let _ = cache;
|
|
let _ = pool;
|
|
|
|
let page = page.parse::<u16>().unwrap_or(1).max(1);
|
|
let per_page = per_page
|
|
.parse::<usize>()
|
|
.unwrap_or(DEFAULT_PER_PAGE)
|
|
.clamp(1, MAX_PER_PAGE);
|
|
let normalized_query = query
|
|
.as_deref()
|
|
.map(str::trim)
|
|
.filter(|value| !value.is_empty());
|
|
|
|
let target = match self.pick_target(normalized_query, &sort, &options).await {
|
|
Ok(target) => target,
|
|
Err(error) => {
|
|
report_provider_error(CHANNEL_ID, "get_videos.pick_target", &error.to_string()).await;
|
|
return vec![];
|
|
}
|
|
};
|
|
|
|
match self.fetch_target_items(target, page, per_page, &options).await {
|
|
Ok(items) => items,
|
|
Err(error) => {
|
|
report_provider_error(CHANNEL_ID, "get_videos", &error.to_string()).await;
|
|
vec![]
|
|
}
|
|
}
|
|
}
|
|
|
|
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
|
Some(self.build_channel(clientversion))
|
|
}
|
|
|
|
async fn get_uploader(
|
|
&self,
|
|
cache: VideoCache,
|
|
pool: DbPool,
|
|
uploader_id: Option<String>,
|
|
uploader_name: Option<String>,
|
|
query: Option<String>,
|
|
profile_content: bool,
|
|
options: ServerOptions,
|
|
) -> std::result::Result<Option<UploaderProfile>, String> {
|
|
let _ = cache;
|
|
let _ = pool;
|
|
|
|
let username = uploader_id
|
|
.as_deref()
|
|
.and_then(|id| id.strip_prefix(&format!("{CHANNEL_ID}:")))
|
|
.map(ToOwned::to_owned)
|
|
.or(uploader_name.clone())
|
|
.map(|value| value.trim().to_string())
|
|
.filter(|value| !value.is_empty());
|
|
|
|
let Some(username) = username else {
|
|
return Ok(None);
|
|
};
|
|
|
|
self.build_uploader_profile(&username, query.as_deref(), profile_content, &options)
|
|
.await
|
|
.map_err(|error| error.to_string())
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
#[test]
|
|
fn only_redgifs_sources_are_resolvable() {
|
|
let mk = |source: Option<&str>| super::Post {
|
|
id: 1,
|
|
uuid: "u".to_string(),
|
|
description: None,
|
|
source: source.map(str::to_string),
|
|
status: None,
|
|
created_at: String::new(),
|
|
author: Default::default(),
|
|
tags: vec![],
|
|
};
|
|
assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://www.redgifs.com/watch/abc"))));
|
|
assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://redgifs.com/watch/abc"))));
|
|
assert!(!XxxtikProvider::is_redgifs_post(&mk(Some("https://www.erome.com/a/CE5z46ne#39"))));
|
|
assert!(!XxxtikProvider::is_redgifs_post(&mk(Some(""))));
|
|
assert!(!XxxtikProvider::is_redgifs_post(&mk(None)));
|
|
}
|
|
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn resolves_query_shortcuts() {
|
|
match XxxtikProvider::resolve_query_target("tag:Big Tits") {
|
|
Some(Target::Tag(name)) => assert_eq!(name, "big tits"),
|
|
other => panic!("expected tag target, got {other:?}"),
|
|
}
|
|
|
|
match XxxtikProvider::resolve_query_target("user:SomeCreator") {
|
|
Some(Target::Creator(name)) => assert_eq!(name, "SomeCreator"),
|
|
other => panic!("expected creator target, got {other:?}"),
|
|
}
|
|
|
|
assert!(XxxtikProvider::resolve_query_target("blonde teen").is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn maps_sort_ids() {
|
|
assert_eq!(Sort::from_sort_id("new"), Sort::New);
|
|
assert_eq!(Sort::from_sort_id("top_week"), Sort::TopWeek);
|
|
assert_eq!(Sort::from_sort_id("month"), Sort::TopMonth);
|
|
assert_eq!(Sort::from_sort_id("unknown"), Sort::New);
|
|
}
|
|
|
|
#[test]
|
|
fn extracts_gif_id_from_source() {
|
|
assert_eq!(
|
|
XxxtikProvider::extract_gif_id("https://www.redgifs.com/watch/fastlightsteelbluexenotarsosaurus"),
|
|
Some("fastlightsteelbluexenotarsosaurus".to_string())
|
|
);
|
|
assert_eq!(
|
|
XxxtikProvider::extract_gif_id("https://v3.redgifs.com/watch/790203707073529740"),
|
|
Some("790203707073529740".to_string())
|
|
);
|
|
assert_eq!(
|
|
XxxtikProvider::extract_gif_id("https://www.redgifs.com/ifr/lastgrimycrossbill"),
|
|
Some("lastgrimycrossbill".to_string())
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parses_post_payload() {
|
|
let json = r#"{
|
|
"id": 123,
|
|
"uuid": "abc-uuid",
|
|
"description": "Test post",
|
|
"source": "https://www.redgifs.com/watch/someid",
|
|
"status": "approved",
|
|
"createdAt": "2026-01-01T00:00:00.000Z",
|
|
"author": {"name": "creator"},
|
|
"tags": [{"name": "asian"}]
|
|
}"#;
|
|
|
|
let post: Post = serde_json::from_str(json).expect("parses");
|
|
assert_eq!(post.id, 123);
|
|
assert_eq!(post.author.name, "creator");
|
|
assert_eq!(post.tags.len(), 1);
|
|
}
|
|
}
|