From 3e56bbc23ba507dd8d0d19ab0d60f56d2bd8b951 Mon Sep 17 00:00:00 2001 From: vdbhb59 Date: Tue, 8 Sep 2026 17:29:31 +0530 Subject: [PATCH] Synced throwie's docker debug instructions + more shit (https://git.lolcat.ca/lolcat/4get/commit/01cea014d78280960bc97efbda9f5a8fe9a26530) (https://git.lolcat.ca/lolcat/4get/commit/7d391fe3ef8720a77aefa9e858f01c7ee52101ef) (https://git.lolcat.ca/lolcat/4get/commit/9f7f294f391c90dc180b9999ea4120fb8b1b7a7d) (https://git.lolcat.ca/lolcat/4get/commit/eaf66511f5aefd55d8037bc9900cc0d7eb714184) (https://git.lolcat.ca/lolcat/4get/commit/bd04cf25d723a14095026aaef4d57f1ab1aeedde) (https://git.lolcat.ca/lolcat/4get/commit/c3b450eed967c700ce83f1fd388d3eab042c24e7) detect yt captcha, added a shit ton of archive shortcuts to hidden menu (https://git.lolcat.ca/lolcat/4get/commit/311c99f2989bcb82d6a72dd66be9d5dc6602b2a6) (https://git.lolcat.ca/lolcat/4get/commit/c0823c84f36d72ae9170e0add7ce8fc7360358b1) (https://git.lolcat.ca/lolcat/4get/commit/e2e76fd007985ee9944090f0390efd7833ab1c95) (https://git.lolcat.ca/lolcat/4get/commit/13c335dbdbae66d0e9f30e264291447bdddd133c) (https://git.lolcat.ca/lolcat/4get/commit/b37802e8f10579ee5ecfedd5a19de7f7674757cc) (https://git.lolcat.ca/lolcat/4get/commit/b3285309517f1e6a123e0ee4230e4603d654d71e) (https://git.lolcat.ca/lolcat/4get/commit/bdc3cffc7b508effb3561de0c8a43007ffdbeffb) (https://git.lolcat.ca/lolcat/4get/commit/a210f77c99fef93cbb1b6620428c7a4bc14096ed) (https://git.lolcat.ca/lolcat/4get/commit/31a9ef4e1380f32507d9e51d3fdb01f72395e30f) --- docs/develop.md | 33 +++- lib/frontend.php | 498 ++++++++++++++++++++++++++++++++++++++++++++++- scraper/yt.php | 34 ++++ 3 files changed, 555 insertions(+), 10 deletions(-) diff --git a/docs/develop.md b/docs/develop.md index e51df0e..7da812b 100644 --- a/docs/develop.md +++ b/docs/develop.md @@ -1,7 +1,36 @@ # Develop - + Start the PHP webserver like this, assuming you have curl-impersonate installed already ```sh LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000 -``` \ No newline at end of file +``` + +Make sure to add `.php` to paths, the embedded PHP webserver doesnt support `mod_rewrite`-like directives. + +If, for some reason, you're in an environment where accessing the debug logs is impossible (like those shitty shared webhosts), you can set `DISPLAY_ERRORS` to `true` in `data/config.php`. + +# Debug guide for docker + +Install: +```sh +git clone https://git.lolcat.ca/lolcat/4get +# make changes to docker/apache/http/httpd.conf (comment out the lines that disable logging) +docker build -t luuul/4get . +``` + +Run: +```sh +docker run --name 4get -p 80:80 -p 443:443 luuul/4get +``` + +Get debug logs: +```sh +docker exec -it `docker ps -qf ancestor=luuul/4get:latest` sh # browse the container +docker exec -it `docker ps -qf ancestor=luuul/4get:latest` tail -f /var/log/apache2/error_log +``` + +Restart the container: +```sh +docker restart 4get +``` diff --git a/lib/frontend.php b/lib/frontend.php index 88ac1a0..cedf9c4 100644 --- a/lib/frontend.php +++ b/lib/frontend.php @@ -535,14 +535,490 @@ class frontend{ '' . '
'; - $payload .= - 'arArchive.org' . - 'arArchive.is' . - 'ghGhostarchive' . - 'arArquivo.pt' . - 'biBing cache' . - 'meMegalodon' . - '
'; + $archives = []; + + $fansites = [ + [ + "url" => "https://cum.st/creators?cq=", + "favicon" => "cum.st", + "favicon_alt" => "cu", + "title" => "OnlyHaven" + ], + [ + "url" => "https://pawchive.pw/artists?q=", + "favicon" => "pawchive.pw", + "favicon_alt" => "pa", + "title" => "Pawchive" + ], + [ + "url" => "https://bakemono.app/creators?q=", + "favicon" => "bakemono.app", + "favicon_alt" => "ba", + "title" => "Bakemono" + ], + [ + "url" => "https://kemono.cr/artists?q=", + "favicon" => "kemono.cr", + "favicon_alt" => "ke", + "title" => "Kemono" + ], + [ + "url" => "https://coomer.st/artists?q=", + "favicon" => "coomer.st", + "favicon_alt" => "co", + "title" => "Coomer" + ] + ]; + + // add website-specific archives + switch($host["host"]){ + + case "www.youtube.com": + case "music.youtube.com": + case "youtube.com": + case "m.youtube.com": + case "youtu.be": + + if( + ( + $host["host"] == "youtu.be" && + isset($host["path"]) && + preg_match( + '/^\/([A-Za-z0-9_-]+)/', + $host["path"], + $slug + ) + ) || + ( + isset($host["query"]) && + preg_match( + '/v=([A-Za-z0-9_-]+)/', + $host["query"], + $slug + ) + ) + ){ + + // for watch?v=slug + $archives[] = [ + "url" => "https://findyoutubevideo.thetechrobo.ca/?q=" . $slug[1], + "favicon" => "findyoutubevideo.thetechrobo.ca", + "favicon_alt" => "fi", + "title" => "FindYouTubeVideo" + ]; + + $archives[] = [ + "url" => "https://filmot.com/video/" . $slug[1] . "/", + "favicon" => "filmot.com", + "favicon_alt" => "fi", + "title" => "Filmot (metadata only)" + ]; + }elseif( + preg_match( + '/^\/(?:channel)\/@?([A-Za-z0-9_-]+)/', + $host["path"], + $username + ) + ){ + + $archives[] = [ + "url" => "https://filmot.com/channel/" . $username[1] . "/", + "favicon" => "filmot.com", + "favicon_alt" => "fi", + "title" => "Filmot (metadata only)" + ]; + } + break; + + case "m.twitch.tv": + case "twitch.tv": + case "player.twitch.tv": + case "www.twitch.tv": + if( + isset($host["path"]) && + preg_match( + '/^\/([A-Za-z0-9_.]+)/', + $host["path"], + $username + ) && + $username[1] != "videos" + ){ + + $archives[] = [ + "url" => "https://vodvod.top/channels/@" . $username[1], + "favicon" => "vodvod.top", + "favicon_alt" => "vo", + "title" => "vodvod" + ]; + + $archives[] = [ + "url" => "https://twitchtracker.com/" . $username[1], + "favicon" => "twitchtracker.com", + "favicon_alt" => "tw", + "title" => "TwitchTracker" + ]; + } + + break; + + case "kick.com": + case "player.kick.com": + if( + isset($host["path"]) && + preg_match( + '/^\/([A-Za-z0-9_.]+)/', + $host["path"], + $username + ) + ){ + + $archives[] = [ + "url" => "https://lick.lolcat.ca/channel?name=" . $username[1], + "favicon" => "lick.lolcat.ca", + "favicon_alt" => "li", + "title" => "lick" + ]; + + $archives[] = [ + "url" => "https://kicktracker.net/" . $username[1], + "favicon" => "kicktracker.net", + "favicon_alt" => "ki", + "title" => "Kick tracker" + ]; + } + break; + + case "x.com": + case "twitter.com": + if( + isset($host["path"]) && + preg_match( + '/^\/([A-Za-z0-9_.]+)/', + $host["path"], + $username + ) + ){ + + $archives[] = [ + "url" => "https://web.archive.org/web/*/https://x.com/" . $username[1] . "/status*", + "favicon" => "archive.org", + "favicon_alt" => "ar", + "title" => "Archive.org: Tweets >2023" + ]; + + $archives[] = [ + "url" => "https://web.archive.org/web/*/https://twitter.com/" . $username[1] . "/status*", + "favicon" => "archive.org", + "favicon_alt" => "ar", + "title" => "Archive.org: Tweets <2023" + ]; + } + break; + + case "www.instagram.com": + case "instagram.com": + if( + isset($host["path"]) && + preg_match( + '/^\/([A-Za-z0-9_.]+)/', + $host["path"], + $username + ) && + $username[1] != "p" + ){ + + $archives[] = [ + "url" => "https://instarchiver.net/users?q=" . $username[1], + "favicon" => "instarchiver.net", + "favicon_alt" => "in", + "title" => "Instarchiver" + ]; + + $archives[] = [ + "url" => "https://www.storiesdb.ch/" . $username[1], + "favicon" => "www.storiesdb.ch", + "favicon_alt" => "st", + "title" => "StoriesDB" + ]; + } + break; + + case "reddit.com": + case "old.reddit.com": + case "www.reddit.com": + if(isset($host["path"])){ + // direct thread lookup + // https://ihsoyct.github.io/r/selfhosted/comments/16emfv0/FOSS-Q_a_proxy_search_engine_that_doesnt_suck/?backend=pullpush + // https://ihsoyct.github.io/r/selfhosted/comments/16emfv0/FOSS-Q_a_proxy_search_engine_that_doesnt_suck/?backend=artic_shift + if( + preg_match( + '/^\/r\/[^\/]+\/comments\/[^?&]+/', + $host["path"], + $slug + ) + ){ + + $archives[] = [ + "url" => "https://ihsoyct.github.io{$slug[0]}?backend=artic_shift", + "favicon" => "reddit.com", + "favicon_alt" => "re", + "title" => "Artic Shift" + ]; + + $archives[] = [ + "url" => "https://ihsoyct.github.io{$slug[0]}?backend=pullpush", + "favicon" => "reddit.com", + "favicon_alt" => "re", + "title" => "PullPush" + ]; + } + + // subreddit thread search + // https://ihsoyct.github.io/?subreddit=selfhosted&backend=artic_shift&mode=submissions&sort=desc + // https://ihsoyct.github.io/?subreddit=selfhosted&backend=pullpush&mode=submissions&sort=desc + // subreddit comment search + // https://ihsoyct.github.io/?subreddit=selfhosted&backend=artic_shift&mode=comments&sort=desc + // https://ihsoyct.github.io/?subreddit=selfhosted&backend=pullpush&mode=comments&sort=desc + elseif( + preg_match( + '/^\/r\/([^\/]+)\/(?:(?:search|wiki|about)\/?)?(?:$|\?|&)/', + $host["path"], + $slug + ) + ){ + + $archives[] = [ + "url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=artic_shift&mode=submissions&sort=desc", + "favicon" => "reddit.com", + "favicon_alt" => "re", + "title" => "Artic Shift (Search threads)" + ]; + + $archives[] = [ + "url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=pullpush&mode=submissions&sort=desc", + "favicon" => "reddit.com", + "favicon_alt" => "re", + "title" => "PullPush (Search threads)" + ]; + + $archives[] = [ + "url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=artic_shift&mode=comments&sort=desc", + "favicon" => "reddit.com", + "favicon_alt" => "re", + "title" => "Artic Shift (Search comments)" + ]; + + $archives[] = [ + "url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=pullpush&mode=comments&sort=desc", + "favicon" => "reddit.com", + "favicon_alt" => "re", + "title" => "PullPush (Search comments)" + ]; + } + + // user thread lookup + // https://ihsoyct.github.io/index.html?author=google&mode=submissions&backend=artic_shift + // https://ihsoyct.github.io/index.html?author=google&mode=submissions&backend=pullpush + // user comment lookup + // https://ihsoyct.github.io/index.html?author=google&mode=comments&backend=artic_shift + // https://ihsoyct.github.io/index.html?author=google&mode=comments&backend=pullpush + elseif( + preg_match( + '/^\/(?:u|user)\/([^\/]+)\/(?:(?:comments|submitted)\/?)?(?:$|\?|&)/', + $host["path"], + $slug + ) + ){ + + $archives[] = [ + "url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=submissions&backend=artic_shift", + "favicon" => "reddit.com", + "favicon_alt" => "re", + "title" => "Artic Shift (Search threads)" + ]; + + $archives[] = [ + "url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=submissions&backend=pullpush", + "favicon" => "reddit.com", + "favicon_alt" => "re", + "title" => "PullPush (Search threads)" + ]; + + $archives[] = [ + "url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=comments&backend=artic_shift", + "favicon" => "reddit.com", + "favicon_alt" => "re", + "title" => "Artic Shift (Search comments)" + ]; + + $archives[] = [ + "url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=comments&backend=pullpush", + "favicon" => "reddit.com", + "favicon_alt" => "re", + "title" => "PullPush (Search comments)" + ]; + } + } + break; + + // https://cum.st/creators?cq= onlyfans fansly patreon + // https://pawchive.pw/artists?q= patreon pixiv/fanbox discord + // https://bakemono.app/creators?q= pixiv/fanbox fansly onlyfans patreon + // https://kemono.cr/artists?q= patreon pixiv/fanbox fantia afdian boosty gumroad subscribestar dlsite + // https://coomer.st/artists?q= onlyfans fansly candfans + + // cant look up + /* + case "fantia.jp": // https://fantia.jp/fanclubs/ + */ + + // normal + case "onlyfans.com": // https://onlyfans.com/ + case "fansly.com": // https://fansly.com/ + case "fanbox.cc": + case "patreon.com": // https://www.patreon.com/ + case "www.patreon.com": + case "boosty.to": // https://boosty.to/ + case "www.subscribestar.com": // https://www.subscribestar.com/ + case "subscribestar.com": + case "www.dlsite.com": // https://www.dlsite.com/ / + case "dlsite.com": + case "candfans.com": // https://candfans.com/ + case "afdian.com": // https://afdian.com/a/ + if( + isset($host["path"]) && + ( + ( + $host["host"] == "afdian.com" && + preg_match( + '/^\/a\/([A-Za-z0-9-_.]+)/', + $host["path"], + $username + ) + ) || + preg_match( + '/^\/([A-Za-z0-9-_.]+)/', + $host["path"], + $username + ) + ) + ){ + + foreach($fansites as $fansite){ + + $archives[] = [ + "url" => $fansite["url"] . $this->sanitize_slug($username[1]), + "favicon" => $fansite["favicon"], + "favicon_alt" => $fansite["favicon_alt"], + "title" => $fansite["title"] + ]; + } + } + break; + } + + // special fansite cases + if( + preg_match( + '/^([A-Za-z0-9_]+)\.(?:fanbox\.cc|gumroad\.com)/', + $host["host"], + $username + ) + ){ + + foreach($fansites as $fansite){ + + $archives[] = [ + "url" => $fansite["url"] . $this->sanitize_slug($username[1]), + "favicon" => $fansite["favicon"], + "favicon_alt" => $fansite["favicon_alt"], + "title" => $fansite["title"] + ]; + } + } + + // detect git + if( + ( + stripos( + $host["host"], + "git." + ) !== false || + $host["host"] == "codeberg.org" || + $host["host"] == "sourceforge.net" || + $host["host"] == "github.com" + ) && + isset($host["path"]) && + preg_match( + '/^(\/[^\/]+\/[^\/]+\/?)/', + $host["path"], + $edge + ) + ){ + + $archives[] = [ + "url" => "https://archive.softwareheritage.org/browse/origin/directory/?origin_url=" . urlencode($host["scheme"] . "://" . $host["host"] . $edge[1]), + "favicon" => "www.softwareheritage.org", + "favicon_alt" => "so", + "title" => "SoftwareHeritage" + ]; + } + + $archives = + array_merge( + $archives, + [ + [ + "url" => "https://web.archive.org/web/" . $urlencode, + "favicon" => "archive.org", + "favicon_alt" => "ar", + "title" => "Archive.org" + ], + [ + "url" => "https://archive.ph/newest/" . htmlspecialchars($link), + "favicon" => "archive.ph", + "favicon_alt" => "ar", + "title" => "Archive.ph" + ], + [ + "url" => "https://yandex.com/search/?text=" . urlencode("url:" . $link), + "favicon" => "yandex.com", + "favicon_alt" => "ya", + "title" => "Yandex cache" + ], + [ + "url" => "https://ghostarchive.org/search?term=" . $urlencode, + "favicon" => "ghostarchive.org", + "favicon_alt" => "gh", + "title" => "Ghostarchive" + ], + [ + "url" => "https://arquivo.pt/wayback/" . htmlspecialchars($link), + "favicon" => "arquivo.pt", + "favicon_alt" => "ar", + "title" => "Arquivo.pt" + ], + [ + "url" => "https://megalodon.jp/?url=" . $urlencode, + "favicon" => "megalodon.jp", + "favicon_alt" => "me", + "title" => "Megalodon" + ], + [ + "url" => "https://www.webcitation.org/query?url=" . $urlencode, + "favicon" => "webcitation.org", + "favicon_alt" => "we", + "title" => "Webcitation" + ] + ] + ); + + foreach($archives as $archive){ + + $payload .= '' . $archive[' . $archive["title"] . ''; + } + + $payload .= ''; } /* @@ -701,6 +1177,7 @@ class frontend{ "option" => [ "yt" => "YouTube", //"archiveorg" => "Archive.org", + //"dailymotion" => "Dailymotion", "vimeo" => "Vimeo", //"odysee" => "Odysee", "sepiasearch" => "Sepia Search", @@ -1061,6 +1538,11 @@ class frontend{ return http_build_query($out); } + private function sanitize_slug($username){ + + return urlencode(preg_replace('/[-_]/', " ", $username)); + } + public function increment_real_reqs($scraper){ apcu_inc(intdiv(time(), 3600) . ".real_requests", 1, $s, 262800); diff --git a/scraper/yt.php b/scraper/yt.php index da38a3c..061ccb9 100644 --- a/scraper/yt.php +++ b/scraper/yt.php @@ -6,6 +6,9 @@ class yt{ include "lib/backend.php"; $this->backend = new backend("yt"); + + include "lib/fuckhtml.php"; + $this->fuckhtml = new fuckhtml(); } public function getfilters($page){ @@ -496,6 +499,8 @@ class yt{ throw new Exception("Could not fetch search page"); } + $this->detect_sorry($json); + //$json = file_get_contents("scraper/yt.html"); // get next page data, unique to first page @@ -1402,4 +1407,33 @@ class yt{ return $text; } + + private function detect_sorry($html){ + + $this->fuckhtml->load($html); + + $as = + $this->fuckhtml + ->getElementsByTagName( + "a" + ); + + foreach($as as $a){ + + // https://www.google.com/sorry/index?continue=https://www.youtube.com/results%3Fsearch_query%3Dhigurashi&q=EhAmBm0ADxn8AAAAAAAAAAEBGOuP_tQGIjDfNp10wvoKdz1WVrgH0F9RjWm2DrJNrPEJl642dSbds6QXWka0rtTlh2s8o0_c5nwyBz5qY25kclJaAUM + + if( + stripos( + $this->fuckhtml + ->getTextContent( + $a["attributes"]["href"] + ), + "https://www.google.com/sorry/index?continue=" + ) !== false + ){ + + throw new Exception("YouTube returned a captcha"); + } + } + } }