freeuseporn

build.rs

@@ -214,6 +214,11 @@ const PROVIDERS: &[ProviderDef] = &[
         module: "freepornvideosxxx",
         ty: "FreepornvideosxxxProvider",
     },
+    ProviderDef {
+        id: "freeuseporn",
+        module: "freeuseporn",
+        ty: "FreeusepornProvider",
+    },
     ProviderDef {
         id: "heavyfetish",
         module: "heavyfetish",

src/api.rs

@@ -554,9 +554,8 @@ async fn uploaders_post(
     let trace_id = crate::util::flow_debug::next_trace_id("uploaders");
     let request = uploader_request.into_inner().normalized();
     if !uploader_request_is_valid(&request) {
-        return Ok(web::HttpResponse::BadRequest().body(
+        return Ok(web::HttpResponse::BadRequest()
-            "At least one of uploaderId or uploaderName must be provided",
+            .body("At least one of uploaderId or uploaderName must be provided"));
-        ));
     }
 
     let public_url_base = format!(

src/providers/freeuseporn.rs

@@ -0,0 +1,562 @@
+use crate::DbPool;
+use crate::api::ClientVersion;
+use crate::providers::{Provider, report_provider_error, requester_or_default};
+use crate::status::*;
+use crate::util::cache::VideoCache;
+use crate::util::parse_abbreviated_number;
+use crate::util::time::parse_time_to_seconds;
+use crate::videos::{ServerOptions, VideoFormat, VideoItem};
+use async_trait::async_trait;
+use error_chain::error_chain;
+use htmlentity::entity::{ICodedDataTrait, decode};
+use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
+use scraper::{Html, Selector};
+use std::collections::HashSet;
+use std::vec;
+
+pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
+    crate::providers::ProviderChannelMetadata {
+        group_id: "fetish-kink",
+        tags: &["freeuse", "hypno", "mind-control"],
+    };
+
+error_chain! {
+    foreign_links {
+        Io(std::io::Error);
+        HttpRequest(wreq::Error);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct FreeusepornProvider {
+    url: String,
+}
+
+impl FreeusepornProvider {
+    pub fn new() -> Self {
+        Self {
+            url: "https://www.freeuseporn.com".to_string(),
+        }
+    }
+
+    fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
+        Channel {
+            id: "freeuseporn".to_string(),
+            name: "FreeusePorn".to_string(),
+            description: "FreeusePorn streams freeuse, hypno, mind control, ignored sex, and related fetish videos.".to_string(),
+            premium: false,
+            favicon: "https://www.google.com/s2/favicons?sz=64&domain=freeuseporn.com".to_string(),
+            status: "active".to_string(),
+            categories: vec![],
+            options: vec![
+                ChannelOption {
+                    id: "sort".to_string(),
+                    title: "Sort".to_string(),
+                    description: "Sort the videos".to_string(),
+                    systemImage: "list.number".to_string(),
+                    colorName: "blue".to_string(),
+                    options: vec![
+                        FilterOption {
+                            id: "recent".to_string(),
+                            title: "Most Recent".to_string(),
+                        },
+                        FilterOption {
+                            id: "viewed".to_string(),
+                            title: "Most Viewed".to_string(),
+                        },
+                        FilterOption {
+                            id: "rated".to_string(),
+                            title: "Top Rated".to_string(),
+                        },
+                        FilterOption {
+                            id: "favorites".to_string(),
+                            title: "Top Favorites".to_string(),
+                        },
+                        FilterOption {
+                            id: "watched".to_string(),
+                            title: "Being Watched".to_string(),
+                        },
+                    ],
+                    multiSelect: false,
+                },
+                ChannelOption {
+                    id: "category".to_string(),
+                    title: "Category".to_string(),
+                    description: "Filter by category".to_string(),
+                    systemImage: "square.grid.2x2".to_string(),
+                    colorName: "orange".to_string(),
+                    options: vec![
+                        FilterOption {
+                            id: "all".to_string(),
+                            title: "All".to_string(),
+                        },
+                        FilterOption {
+                            id: "mind-control".to_string(),
+                            title: "Mind Control".to_string(),
+                        },
+                        FilterOption {
+                            id: "general-freeuse".to_string(),
+                            title: "General Freeuse".to_string(),
+                        },
+                        FilterOption {
+                            id: "free-service".to_string(),
+                            title: "Free Service".to_string(),
+                        },
+                        FilterOption {
+                            id: "forced".to_string(),
+                            title: "Forced".to_string(),
+                        },
+                        FilterOption {
+                            id: "japanese".to_string(),
+                            title: "Japanese".to_string(),
+                        },
+                        FilterOption {
+                            id: "time-stop".to_string(),
+                            title: "Time Stop".to_string(),
+                        },
+                        FilterOption {
+                            id: "ignored-sex".to_string(),
+                            title: "Ignored Sex".to_string(),
+                        },
+                        FilterOption {
+                            id: "glory-hole".to_string(),
+                            title: "Glory Hole".to_string(),
+                        },
+                    ],
+                    multiSelect: false,
+                },
+            ],
+            nsfw: true,
+            cacheDuration: Some(1800),
+        }
+    }
+
+    fn absolute_url(&self, url: &str) -> String {
+        if url.starts_with("http://") || url.starts_with("https://") {
+            url.to_string()
+        } else if url.starts_with('/') {
+            format!("{}{}", self.url, url)
+        } else {
+            format!("{}/{}", self.url, url.trim_start_matches('/'))
+        }
+    }
+
+    fn sort_param(sort: &str) -> &'static str {
+        match sort {
+            "viewed" => "mv",
+            "rated" => "tr",
+            "favorites" => "tf",
+            "watched" => "bw",
+            _ => "mr",
+        }
+    }
+
+    fn build_list_url(
+        &self,
+        sort: &str,
+        page: u8,
+        query: Option<&str>,
+        category: Option<&str>,
+    ) -> String {
+        let path = if let Some(query) = query.map(str::trim).filter(|value| !value.is_empty()) {
+            format!(
+                "/search/videos/{}",
+                utf8_percent_encode(query, NON_ALPHANUMERIC)
+            )
+        } else if let Some(category) = category
+            .map(str::trim)
+            .filter(|value| !value.is_empty() && *value != "all")
+        {
+            format!("/videos/{}", category)
+        } else {
+            "/videos".to_string()
+        };
+
+        let mut params = vec![format!("o={}", Self::sort_param(sort))];
+        if page > 1 {
+            params.push(format!("page={page}"));
+        }
+
+        format!("{}{}?{}", self.url, path, params.join("&"))
+    }
+
+    fn build_formats(&self, id: &str) -> Vec<VideoFormat> {
+        let hd = VideoFormat::new(
+            format!("{}/media/videos/h264/{}_720p.mp4", self.url, id),
+            "720p".to_string(),
+            "video/mp4".to_string(),
+        )
+        .format_id("720p".to_string())
+        .format_note("720p".to_string());
+        let sd = VideoFormat::new(
+            format!("{}/media/videos/h264/{}_480p.mp4", self.url, id),
+            "480p".to_string(),
+            "video/mp4".to_string(),
+        )
+        .format_id("480p".to_string())
+        .format_note("480p".to_string());
+        vec![hd, sd]
+    }
+
+    fn normalized_text(text: &str) -> String {
+        text.split_whitespace().collect::<Vec<_>>().join(" ")
+    }
+
+    fn decode_text(value: &str) -> String {
+        decode(value.as_bytes())
+            .to_string()
+            .unwrap_or_else(|_| value.to_string())
+    }
+
+    fn parse_views(value: &str) -> Option<u32> {
+        let digits = value
+            .chars()
+            .filter(|character| character.is_ascii_digit() || *character == '.' || *character == 'K' || *character == 'M' || *character == 'B' || *character == 'k' || *character == 'm' || *character == 'b')
+            .collect::<String>();
+        if digits.is_empty() {
+            return None;
+        }
+        parse_abbreviated_number(&digits).map(|views| views as u32)
+    }
+
+    fn parse_rating(value: &str) -> Option<f32> {
+        value
+            .trim()
+            .trim_end_matches('%')
+            .parse::<f32>()
+            .ok()
+    }
+
+    fn parse_video_item_from_anchor(
+        &self,
+        anchor: scraper::ElementRef<'_>,
+        selectors: &FreeusepornSelectors,
+    ) -> Option<VideoItem> {
+        let href = anchor.value().attr("href")?;
+        if !href.contains("/video/") {
+            return None;
+        }
+
+        let absolute_url = self.absolute_url(href);
+        let id = absolute_url.split('/').nth(4)?.to_string();
+        if id.is_empty() {
+            return None;
+        }
+
+        let title_raw = anchor
+            .select(&selectors.title)
+            .next()
+            .map(|element| Self::normalized_text(&element.text().collect::<Vec<_>>().join(" ")))
+            .filter(|value| !value.is_empty())
+            .or_else(|| anchor.value().attr("title").map(Self::normalized_text))
+            .or_else(|| {
+                anchor
+                    .select(&selectors.image)
+                    .next()
+                    .and_then(|element| element.value().attr("alt"))
+                    .map(Self::normalized_text)
+            })?;
+        let title = Self::decode_text(&title_raw);
+
+        let thumb = anchor
+            .select(&selectors.image)
+            .next()
+            .and_then(|element| element.value().attr("src"))
+            .map(|src| self.absolute_url(src))
+            .unwrap_or_default();
+
+        let duration = anchor
+            .select(&selectors.duration)
+            .next()
+            .map(|element| Self::normalized_text(&element.text().collect::<Vec<_>>().join(" ")))
+            .and_then(|value| parse_time_to_seconds(&value))
+            .unwrap_or(0) as u32;
+
+        let mut stats = anchor
+            .select(&selectors.video_stat)
+            .map(|element| Self::normalized_text(&element.text().collect::<Vec<_>>().join(" ")))
+            .collect::<Vec<_>>();
+        stats.retain(|value| !value.is_empty());
+        let views = stats.first().and_then(|value| Self::parse_views(value));
+        let rating = stats.get(1).and_then(|value| Self::parse_rating(value));
+
+        let mut item = VideoItem::new(
+            id.clone(),
+            title,
+            absolute_url,
+            "freeuseporn".to_string(),
+            thumb,
+            duration,
+        )
+        .views(views.unwrap_or(0))
+        .formats(self.build_formats(&id));
+
+        if views.is_none() {
+            item.views = None;
+        }
+        item.rating = rating;
+
+        Some(item)
+    }
+
+    fn get_video_items_from_html(&self, html: &str) -> Vec<VideoItem> {
+        if html.trim().is_empty() {
+            return vec![];
+        }
+
+        let document = Html::parse_document(html);
+        let selectors = FreeusepornSelectors::new();
+        let primary_anchors = document
+            .select(&selectors.list_anchor)
+            .collect::<Vec<_>>();
+        let anchors = if primary_anchors.is_empty() {
+            document
+                .select(&selectors.fallback_anchor)
+                .collect::<Vec<_>>()
+        } else {
+            primary_anchors
+        };
+
+        let mut seen = HashSet::new();
+        let mut items = Vec::new();
+
+        for anchor in anchors {
+            let Some(item) = self.parse_video_item_from_anchor(anchor, &selectors) else {
+                continue;
+            };
+            if seen.insert(item.id.clone()) {
+                items.push(item);
+            }
+        }
+
+        items
+    }
+
+    async fn fetch_listing(
+        &self,
+        cache: VideoCache,
+        url: String,
+        options: ServerOptions,
+        error_context: &str,
+    ) -> Result<Vec<VideoItem>> {
+        let old_items = match cache.get(&url) {
+            Some((time, items)) => {
+                if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
+                    return Ok(items.clone());
+                }
+                items.clone()
+            }
+            None => vec![],
+        };
+
+        let mut requester = requester_or_default(&options, module_path!(), "missing_requester");
+        let text = match requester.get(&url, None).await {
+            Ok(text) => text,
+            Err(error) => {
+                report_provider_error(
+                    "freeuseporn",
+                    error_context,
+                    &format!("url={url}; error={error}"),
+                )
+                .await;
+                return Ok(old_items);
+            }
+        };
+
+        let items = self.get_video_items_from_html(&text);
+        if items.is_empty() {
+            return Ok(old_items);
+        }
+
+        cache.remove(&url);
+        cache.insert(url, items.clone());
+        Ok(items)
+    }
+
+    async fn get(
+        &self,
+        cache: VideoCache,
+        page: u8,
+        sort: &str,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
+        let url = self.build_list_url(sort, page, None, options.category.as_deref());
+        self.fetch_listing(cache, url, options, "get.request").await
+    }
+
+    async fn query(
+        &self,
+        cache: VideoCache,
+        page: u8,
+        query: &str,
+        sort: &str,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
+        let url = self.build_list_url(sort, page, Some(query), None);
+        self.fetch_listing(cache, url, options, "query.request").await
+    }
+}
+
+struct FreeusepornSelectors {
+    list_anchor: Selector,
+    fallback_anchor: Selector,
+    title: Selector,
+    image: Selector,
+    duration: Selector,
+    video_stat: Selector,
+}
+
+impl FreeusepornSelectors {
+    fn new() -> Self {
+        Self {
+            list_anchor: Selector::parse("#videos-list a[href]").expect("valid freeuseporn list selector"),
+            fallback_anchor: Selector::parse("a[href]").expect("valid freeuseporn fallback selector"),
+            title: Selector::parse(".v-name").expect("valid freeuseporn title selector"),
+            image: Selector::parse("img").expect("valid freeuseporn image selector"),
+            duration: Selector::parse(".duration").expect("valid freeuseporn duration selector"),
+            video_stat: Selector::parse(".video-stats li").expect("valid freeuseporn stats selector"),
+        }
+    }
+}
+
+#[async_trait]
+impl Provider for FreeusepornProvider {
+    async fn get_videos(
+        &self,
+        cache: VideoCache,
+        _pool: DbPool,
+        sort: String,
+        query: Option<String>,
+        page: String,
+        _per_page: String,
+        options: ServerOptions,
+    ) -> Vec<VideoItem> {
+        let page = page.parse::<u8>().unwrap_or(1);
+        let videos = match query {
+            Some(query) => self.query(cache, page, &query, &sort, options).await,
+            None => self.get(cache, page, &sort, options).await,
+        };
+
+        match videos {
+            Ok(items) => items,
+            Err(error) => {
+                eprintln!("freeuseporn provider error: {error}");
+                vec![]
+            }
+        }
+    }
+
+    fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
+        Some(self.build_channel(clientversion))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn provider() -> FreeusepornProvider {
+        FreeusepornProvider::new()
+    }
+
+    #[test]
+    fn builds_listing_urls_for_sort_category_and_search() {
+        let provider = provider();
+
+        assert_eq!(
+            provider.build_list_url("recent", 1, None, None),
+            "https://www.freeuseporn.com/videos?o=mr"
+        );
+        assert_eq!(
+            provider.build_list_url("viewed", 2, None, Some("mind-control")),
+            "https://www.freeuseporn.com/videos/mind-control?o=mv&page=2"
+        );
+        assert_eq!(
+            provider.build_list_url("favorites", 3, Some("mind control"), None),
+            "https://www.freeuseporn.com/search/videos/mind%20control?o=tf&page=3"
+        );
+    }
+
+    #[test]
+    fn parses_listing_items_and_builds_formats() {
+        let provider = provider();
+        let html = r#"
+            <ul class="grid" id="videos-list">
+                <li>
+                    <div class="item">
+                        <div class="thumbnail">
+                            <div class="embed">
+                                <iframe src="https://ads.example"></iframe>
+                            </div>
+                        </div>
+                    </div>
+                </li>
+                <li>
+                    <a href="/video/9579/nicole-kitt-shady-slut-keeps-confessing" class="thumb-wrap-link">
+                        <div class="item">
+                            <div class="thumbnail overlay" id="playvthumb_9579">
+                                <div class="sub-data">
+                                    <span class="duration">59:09</span>
+                                </div>
+                                <img src="https://www.freeuseporn.com/media/videos/tmb/9579/1.jpg" alt="Nicole Kitt &amp; The Truth"/>
+                            </div>
+                            <div class="info">
+                                <span class="v-name">Nicole Kitt &amp; The Truth</span>
+                                <ul class="video-stats">
+                                    <li><i class="far fa-eye"></i>52180</li>
+                                    <li><i class="far fa-heart"></i>100%</li>
+                                </ul>
+                            </div>
+                        </div>
+                    </a>
+                </li>
+                <li>
+                    <a href="https://www.freeuseporn.com/video/9578/lollipop-time-stop-2">
+                        <div class="item">
+                            <div class="thumbnail overlay">
+                                <div class="sub-data">
+                                    <span class="duration">16:27</span>
+                                </div>
+                                <img src="https://www.freeuseporn.com/media/videos/tmb/9578/1.jpg" alt="Lollipop time stop 2"/>
+                            </div>
+                            <div class="info">
+                                <span class="v-name">Lollipop time stop 2</span>
+                                <ul class="video-stats">
+                                    <li><i class="far fa-eye"></i>35058</li>
+                                    <li><i class="far fa-heart"></i>88%</li>
+                                </ul>
+                            </div>
+                        </div>
+                    </a>
+                </li>
+            </ul>
+        "#;
+
+        let items = provider.get_video_items_from_html(html);
+        assert_eq!(items.len(), 2);
+        assert_eq!(items[0].id, "9579");
+        assert_eq!(items[0].title, "Nicole Kitt & The Truth");
+        assert_eq!(
+            items[0].url,
+            "https://www.freeuseporn.com/video/9579/nicole-kitt-shady-slut-keeps-confessing"
+        );
+        assert_eq!(
+            items[0].thumb,
+            "https://www.freeuseporn.com/media/videos/tmb/9579/1.jpg"
+        );
+        assert_eq!(items[0].duration, 3549);
+        assert_eq!(items[0].views, Some(52180));
+        assert_eq!(items[0].rating, Some(100.0));
+        assert_eq!(items[0].formats.as_ref().map(|formats| formats.len()), Some(2));
+        assert_eq!(
+            items[0]
+                .formats
+                .as_ref()
+                .and_then(|formats| formats.first())
+                .map(|format| format.url.as_str()),
+            Some("https://www.freeuseporn.com/media/videos/h264/9579_720p.mp4")
+        );
+        assert_eq!(items[1].id, "9578");
+        assert_eq!(items[1].rating, Some(88.0));
+    }
+}

src/providers/mod.rs

@@ -597,7 +597,11 @@ where
                 "provider uploader guard exit provider={} context={} matched={}",
                 provider_name,
                 context,
-                result.as_ref().ok().and_then(|value| value.as_ref()).is_some()
+                result
+                    .as_ref()
+                    .ok()
+                    .and_then(|value| value.as_ref())
+                    .is_some()
             );
             result
         }
@@ -1262,7 +1266,8 @@ mod tests {
         let now = Instant::now();
         let mut counted = 0;
         for step in 0..VALIDATION_FAILURES_FOR_ERROR {
-            counted = record_validation_failure(provider_id, now + VALIDATION_COOLDOWN * step as u32);
+            counted =
+                record_validation_failure(provider_id, now + VALIDATION_COOLDOWN * step as u32);
         }
         assert_eq!(counted, VALIDATION_FAILURES_FOR_ERROR);
 

src/proxies/doodstream.rs

@@ -60,9 +60,13 @@ impl DoodstreamProxy {
     }
 
     fn request_headers(detail_url: &str) -> Vec<(String, String)> {
-        let origin = Self::request_origin(detail_url).unwrap_or_else(|| "https://turboplayers.xyz".to_string());
+        let origin = Self::request_origin(detail_url)
+            .unwrap_or_else(|| "https://turboplayers.xyz".to_string());
         vec![
-            ("Referer".to_string(), format!("{}/", origin.trim_end_matches('/'))),
+            (
+                "Referer".to_string(),
+                format!("{}/", origin.trim_end_matches('/')),
+            ),
             ("Origin".to_string(), origin),
             (
                 "Accept".to_string(),
@@ -224,9 +228,11 @@ impl DoodstreamProxy {
 
     fn extract_pass_md5_url(text: &str, detail_url: &str) -> Option<String> {
         let decoded = text.replace("\\/", "/");
-        let absolute_regex =
+        let absolute_regex = Self::regex(r#"https?://[^\s"'<>]+/pass_md5/[^\s"'<>]+"#)?;
-            Self::regex(r#"https?://[^\s"'<>]+/pass_md5/[^\s"'<>]+"#)?;
+        if let Some(url) = absolute_regex
-        if let Some(url) = absolute_regex.find(&decoded).map(|value| value.as_str().to_string()) {
+            .find(&decoded)
+            .map(|value| value.as_str().to_string())
+        {
             return Some(url);
         }
 
@@ -276,7 +282,8 @@ impl DoodstreamProxy {
         requester: &mut Requester,
     ) -> Option<String> {
         let pass_md5_url = Self::extract_pass_md5_url(html, detail_url).or_else(|| {
-            Self::unpack_packer(html).and_then(|unpacked| Self::extract_pass_md5_url(&unpacked, detail_url))
+            Self::unpack_packer(html)
+                .and_then(|unpacked| Self::extract_pass_md5_url(&unpacked, detail_url))
         })?;
 
         let headers = vec![
@@ -311,7 +318,9 @@ impl crate::proxies::Proxy for DoodstreamProxy {
             return url;
         }
 
-        if let Some(url) = Self::resolve_stream_from_pass_md5(&detail_url, &html, &mut requester).await {
+        if let Some(url) =
+            Self::resolve_stream_from_pass_md5(&detail_url, &html, &mut requester).await
+        {
             return url;
         }
 
@@ -370,7 +379,8 @@ mod tests {
 
     #[test]
     fn composes_media_url_from_pass_md5_response() {
-        let pass_md5_url = "https://trailerhg.xyz/pass_md5/abc123/def456?token=t0k3n&expiry=1775000000";
+        let pass_md5_url =
+            "https://trailerhg.xyz/pass_md5/abc123/def456?token=t0k3n&expiry=1775000000";
         let body = "https://g4vsrqvtrj.pinebrookproductionlab.shop/1ghkpx2e8jnal/hls3/01/08534/syyzvotfnhaa_l/master.txt";
         assert_eq!(
             DoodstreamProxy::compose_pass_md5_media_url(pass_md5_url, body).as_deref(),