mirror of
https://git.bakhai.co.in/FbIN/4Get.git
synced 2025-11-05 20:31:30 +05:30
Synced
f73b5f0298/scraper/yandex.phpa2bc1e6190/lib/anubis.phpa2bc1e6190/scraper/marginalia.php
This commit is contained in:
parent
a7650f0ad0
commit
27b91e5124
3 changed files with 291 additions and 14 deletions
|
|
@ -3,7 +3,10 @@
|
|||
class marginalia{
|
||||
public function __construct(){
|
||||
|
||||
include "lib/fuckhtml.php";
|
||||
include "lib/anubis.php";
|
||||
$this->anubis = new anubis();
|
||||
|
||||
include_once "lib/fuckhtml.php";
|
||||
$this->fuckhtml = new fuckhtml();
|
||||
|
||||
include "lib/backend.php";
|
||||
|
|
@ -102,7 +105,40 @@ class marginalia{
|
|||
);
|
||||
}
|
||||
|
||||
private function get($proxy, $url, $get = []){
|
||||
private function get($proxy, $url, $get = [], $get_cookies = 1){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
switch($get_cookies){
|
||||
|
||||
case 0:
|
||||
$cookies = "";
|
||||
$cookies_tmp = [];
|
||||
curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
|
||||
|
||||
$length = strlen($header);
|
||||
|
||||
$header = explode(":", $header, 2);
|
||||
|
||||
if(trim(strtolower($header[0])) == "set-cookie"){
|
||||
|
||||
$cookie_tmp = explode("=", trim($header[1]), 2);
|
||||
|
||||
$cookies_tmp[trim($cookie_tmp[0])] =
|
||||
explode(";", $cookie_tmp[1], 2)[0];
|
||||
}
|
||||
|
||||
return $length;
|
||||
});
|
||||
break;
|
||||
|
||||
case 1:
|
||||
$cookies = "";
|
||||
break;
|
||||
|
||||
default:
|
||||
$cookies = "Cookie: " . $get_cookies;
|
||||
}
|
||||
|
||||
$headers = [
|
||||
"User-Agent: " . config::USER_AGENT,
|
||||
|
|
@ -110,6 +146,7 @@ class marginalia{
|
|||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip",
|
||||
"DNT: 1",
|
||||
$cookies,
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
|
|
@ -118,8 +155,6 @@ class marginalia{
|
|||
"Sec-Fetch-User: ?1"
|
||||
];
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
if($get !== []){
|
||||
$get = http_build_query($get);
|
||||
$url .= "?" . $get;
|
||||
|
|
@ -145,7 +180,19 @@ class marginalia{
|
|||
throw new Exception(curl_error($curlproc));
|
||||
}
|
||||
|
||||
curl_close($curlproc);
|
||||
if($get_cookies === 0){
|
||||
|
||||
$cookie = [];
|
||||
|
||||
foreach($cookies_tmp as $key => $value){
|
||||
|
||||
$cookie[] = $key . "=" . $value;
|
||||
}
|
||||
|
||||
curl_close($curlproc);
|
||||
return implode(";", $cookie);
|
||||
}
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
|
|
@ -267,6 +314,55 @@ class marginalia{
|
|||
// HTML parser
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
//
|
||||
// Bypass anubis check
|
||||
//
|
||||
if(($anubis_key = apcu_fetch("marginalia_cookie")) === false){
|
||||
|
||||
try{
|
||||
$html =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://old-search.marginalia.nu/"
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to get anubis challenge");
|
||||
}
|
||||
|
||||
try{
|
||||
|
||||
$anubis_data = $this->anubis->scrape($html);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception($error);
|
||||
}
|
||||
|
||||
// send anubis response & get cookies
|
||||
// https://old-search.marginalia.nu/.within.website/x/cmd/anubis/api/pass-challenge?response=0000018966b086834f738bacba6031028adb5aa875974ead197a8b75778baf3a&nonce=39947&redir=https%3A%2F%2Fold-search.marginalia.nu%2F&elapsedTime=1164
|
||||
|
||||
try{
|
||||
|
||||
$anubis_key =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://old-search.marginalia.nu/.within.website/x/cmd/anubis/api/pass-challenge",
|
||||
[
|
||||
"response" => $anubis_data["response"],
|
||||
"nonce" => $anubis_data["nonce"],
|
||||
"redir" => "https://old-search.marginalia.nu/",
|
||||
"elapsedTime" => random_int(1000, 2000)
|
||||
],
|
||||
0
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to submit anubis challenge");
|
||||
}
|
||||
|
||||
apcu_store("marginalia_cookie", $anubis_key);
|
||||
}
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
[$params, $proxy] =
|
||||
|
|
@ -279,7 +375,9 @@ class marginalia{
|
|||
$html =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://old-search.marginalia.nu/search?" . $params
|
||||
"https://old-search.marginalia.nu/search?" . $params,
|
||||
[],
|
||||
$anubis_key
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
|
|
@ -309,7 +407,8 @@ class marginalia{
|
|||
$this->get(
|
||||
$proxy,
|
||||
"https://old-search.marginalia.nu/search",
|
||||
$params
|
||||
$params,
|
||||
$anubis_key
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
|
|
@ -472,5 +571,4 @@ class marginalia{
|
|||
|
||||
return $out;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@ class yandex{
|
|||
// backend included in the scraper functions
|
||||
}
|
||||
|
||||
private function get($proxy, $url, $get = [], $nsfw){
|
||||
private function get($proxy, $url, $get = [], $nsfw, $get_cookie = 1){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
|
|
@ -25,19 +25,55 @@ class yandex{
|
|||
|
||||
curl_setopt($curlproc, CURLOPT_URL, $url);
|
||||
|
||||
// extract "i" cookie
|
||||
if($get_cookie === 0){
|
||||
|
||||
$cookies_tmp = [];
|
||||
curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
|
||||
|
||||
$length = strlen($header);
|
||||
|
||||
$header = explode(":", $header, 2);
|
||||
|
||||
if(trim(strtolower($header[0])) == "set-cookie"){
|
||||
|
||||
$cookie_tmp = explode("=", trim($header[1]), 2);
|
||||
|
||||
$cookies_tmp[trim($cookie_tmp[0])] =
|
||||
explode(";", $cookie_tmp[1], 2)[0];
|
||||
}
|
||||
|
||||
return $length;
|
||||
});
|
||||
}
|
||||
|
||||
switch($nsfw){
|
||||
case "yes": $nsfw = "0"; break;
|
||||
case "maybe": $nsfw = "1"; break;
|
||||
case "no": $nsfw = "2"; break;
|
||||
}
|
||||
|
||||
switch($get_cookie){
|
||||
|
||||
case 0:
|
||||
$cookie = "";
|
||||
break;
|
||||
|
||||
case 1:
|
||||
$cookie = "Cookie: yp=" . (time() - 4000033) . ".szm.1:1920x1080:876x1000#" . time() . ".sp.family:" . $nsfw;
|
||||
break;
|
||||
|
||||
default:
|
||||
$cookie = "Cookie: i=" . $get_cookie;
|
||||
}
|
||||
|
||||
$headers =
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Encoding: gzip",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"DNT: 1",
|
||||
"Cookie: yp=" . (time() - 4000033) . ".szm.1:1920x1080:876x1000#" . time() . ".sp.family:" . $nsfw,
|
||||
$cookie,
|
||||
"Referer: https://yandex.com/images/search",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
|
|
@ -59,6 +95,17 @@ class yandex{
|
|||
|
||||
$data = curl_exec($curlproc);
|
||||
|
||||
if($get_cookie === 0){
|
||||
|
||||
if(isset($cookies_tmp["i"])){
|
||||
|
||||
return $cookies_tmp["i"];
|
||||
}else{
|
||||
|
||||
throw new Exception("Failed to get Yandex clearance cookie");
|
||||
}
|
||||
}
|
||||
|
||||
if(curl_errno($curlproc)){
|
||||
|
||||
throw new Exception(curl_error($curlproc));
|
||||
|
|
@ -217,6 +264,23 @@ class yandex{
|
|||
// https://yandex.com/search/site/?text=minecraft&web=1&frame=1&v=2.0&searchid=3131712
|
||||
// &within=777&from_day=26&from_month=8&from_year=2023&to_day=26&to_month=8&to_year=2023
|
||||
|
||||
// get clearance cookie
|
||||
if(($cookie = apcu_fetch("yandexweb_cookie")) === false){
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$cookie =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://yandex.ru/support2/smart-captcha/ru/",
|
||||
[],
|
||||
false,
|
||||
0
|
||||
);
|
||||
|
||||
apcu_store("yandexweb_cookie", $cookie);
|
||||
}
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
[$npt, $proxy] = $this->backend->get($get["npt"], "web");
|
||||
|
|
@ -226,7 +290,8 @@ class yandex{
|
|||
$proxy,
|
||||
"https://yandex.com" . $npt,
|
||||
[],
|
||||
"yes"
|
||||
"yes",
|
||||
$cookie
|
||||
);
|
||||
}else{
|
||||
|
||||
|
|
@ -236,7 +301,7 @@ class yandex{
|
|||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
$proxy = !isset($proxy) ? $this->backend->get_ip() : $proxy;
|
||||
$lang = $get["lang"];
|
||||
$older = $get["older"];
|
||||
$newer = $get["newer"];
|
||||
|
|
@ -283,7 +348,8 @@ class yandex{
|
|||
$proxy,
|
||||
"https://yandex.com/search/site/",
|
||||
$params,
|
||||
"yes"
|
||||
"yes",
|
||||
$cookie
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
|
|
@ -314,6 +380,19 @@ class yandex{
|
|||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
// Scrape page blocked error
|
||||
$title =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName("title");
|
||||
|
||||
if(
|
||||
count($title) !== 0 &&
|
||||
$title[0]["innerHTML"] == "403"
|
||||
){
|
||||
|
||||
throw new Exception("Yandex blocked this proxy or 4get instance.");
|
||||
}
|
||||
|
||||
// get nextpage
|
||||
$npt =
|
||||
$this->fuckhtml
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue