vdbhb59
4e9e613df1
fix dmca links not showing up on google (d598a560ae) dumbass moment (28ba8422b2) deny access to resolve (06ee6878ca) scrape google sublinks, which are encrypted pieces of shit (90232152e9)
2919 lines
55 KiB
PHP
2919 lines
55 KiB
PHP
<?php
|
|
|
|
class google{
|
|
|
|
public function __construct(){
|
|
|
|
include "lib/fuckhtml.php";
|
|
$this->fuckhtml = new fuckhtml();
|
|
|
|
include "lib/backend.php";
|
|
$this->backend = new backend("google");
|
|
}
|
|
|
|
public function getfilters($page){
|
|
|
|
$base = [
|
|
"country" => [ // gl=<country> (image: cr=countryAF)
|
|
"display" => "Country",
|
|
"option" => [
|
|
"any" => "Instance's country",
|
|
"af" => "Afghanistan",
|
|
"al" => "Albania",
|
|
"dz" => "Algeria",
|
|
"as" => "American Samoa",
|
|
"ad" => "Andorra",
|
|
"ao" => "Angola",
|
|
"ai" => "Anguilla",
|
|
"aq" => "Antarctica",
|
|
"ag" => "Antigua and Barbuda",
|
|
"ar" => "Argentina",
|
|
"am" => "Armenia",
|
|
"aw" => "Aruba",
|
|
"au" => "Australia",
|
|
"at" => "Austria",
|
|
"az" => "Azerbaijan",
|
|
"bs" => "Bahamas",
|
|
"bh" => "Bahrain",
|
|
"bd" => "Bangladesh",
|
|
"bb" => "Barbados",
|
|
"by" => "Belarus",
|
|
"be" => "Belgium",
|
|
"bz" => "Belize",
|
|
"bj" => "Benin",
|
|
"bm" => "Bermuda",
|
|
"bt" => "Bhutan",
|
|
"bo" => "Bolivia",
|
|
"ba" => "Bosnia and Herzegovina",
|
|
"bw" => "Botswana",
|
|
"bv" => "Bouvet Island",
|
|
"br" => "Brazil",
|
|
"io" => "British Indian Ocean Territory",
|
|
"bn" => "Brunei Darussalam",
|
|
"bg" => "Bulgaria",
|
|
"bf" => "Burkina Faso",
|
|
"bi" => "Burundi",
|
|
"kh" => "Cambodia",
|
|
"cm" => "Cameroon",
|
|
"ca" => "Canada",
|
|
"cv" => "Cape Verde",
|
|
"ky" => "Cayman Islands",
|
|
"cf" => "Central African Republic",
|
|
"td" => "Chad",
|
|
"cl" => "Chile",
|
|
"cn" => "China",
|
|
"cx" => "Christmas Island",
|
|
"cc" => "Cocos (Keeling) Islands",
|
|
"co" => "Colombia",
|
|
"km" => "Comoros",
|
|
"cg" => "Congo",
|
|
"cd" => "Congo, the Democratic Republic",
|
|
"ck" => "Cook Islands",
|
|
"cr" => "Costa Rica",
|
|
"ci" => "Cote D'ivoire",
|
|
"hr" => "Croatia",
|
|
"cu" => "Cuba",
|
|
"cy" => "Cyprus",
|
|
"cz" => "Czech Republic",
|
|
"dk" => "Denmark",
|
|
"dj" => "Djibouti",
|
|
"dm" => "Dominica",
|
|
"do" => "Dominican Republic",
|
|
"ec" => "Ecuador",
|
|
"eg" => "Egypt",
|
|
"sv" => "El Salvador",
|
|
"gq" => "Equatorial Guinea",
|
|
"er" => "Eritrea",
|
|
"ee" => "Estonia",
|
|
"et" => "Ethiopia",
|
|
"fk" => "Falkland Islands (Malvinas)",
|
|
"fo" => "Faroe Islands",
|
|
"fj" => "Fiji",
|
|
"fi" => "Finland",
|
|
"fr" => "France",
|
|
"gf" => "French Guiana",
|
|
"pf" => "French Polynesia",
|
|
"tf" => "French Southern Territories",
|
|
"ga" => "Gabon",
|
|
"gm" => "Gambia",
|
|
"ge" => "Georgia",
|
|
"de" => "Germany",
|
|
"gh" => "Ghana",
|
|
"gi" => "Gibraltar",
|
|
"gr" => "Greece",
|
|
"gl" => "Greenland",
|
|
"gd" => "Grenada",
|
|
"gp" => "Guadeloupe",
|
|
"gu" => "Guam",
|
|
"gt" => "Guatemala",
|
|
"gn" => "Guinea",
|
|
"gw" => "Guinea-Bissau",
|
|
"gy" => "Guyana",
|
|
"ht" => "Haiti",
|
|
"hm" => "Heard Island and Mcdonald Islands",
|
|
"va" => "Holy See (Vatican City State)",
|
|
"hn" => "Honduras",
|
|
"hk" => "Hong Kong",
|
|
"hu" => "Hungary",
|
|
"is" => "Iceland",
|
|
"in" => "India",
|
|
"id" => "Indonesia",
|
|
"ir" => "Iran, Islamic Republic",
|
|
"iq" => "Iraq",
|
|
"ie" => "Ireland",
|
|
"il" => "Israel",
|
|
"it" => "Italy",
|
|
"jm" => "Jamaica",
|
|
"jp" => "Japan",
|
|
"jo" => "Jordan",
|
|
"kz" => "Kazakhstan",
|
|
"ke" => "Kenya",
|
|
"ki" => "Kiribati",
|
|
"kp" => "Korea, Democratic People's Republic",
|
|
"kr" => "Korea, Republic",
|
|
"kw" => "Kuwait",
|
|
"kg" => "Kyrgyzstan",
|
|
"la" => "Lao People's Democratic Republic",
|
|
"lv" => "Latvia",
|
|
"lb" => "Lebanon",
|
|
"ls" => "Lesotho",
|
|
"lr" => "Liberia",
|
|
"ly" => "Libyan Arab Jamahiriya",
|
|
"li" => "Liechtenstein",
|
|
"lt" => "Lithuania",
|
|
"lu" => "Luxembourg",
|
|
"mo" => "Macao",
|
|
"mk" => "Macedonia, the Former Yugosalv Republic",
|
|
"mg" => "Madagascar",
|
|
"mw" => "Malawi",
|
|
"my" => "Malaysia",
|
|
"mv" => "Maldives",
|
|
"ml" => "Mali",
|
|
"mt" => "Malta",
|
|
"mh" => "Marshall Islands",
|
|
"mq" => "Martinique",
|
|
"mr" => "Mauritania",
|
|
"mu" => "Mauritius",
|
|
"yt" => "Mayotte",
|
|
"mx" => "Mexico",
|
|
"fm" => "Micronesia, Federated States",
|
|
"md" => "Moldova, Republic",
|
|
"mc" => "Monaco",
|
|
"mn" => "Mongolia",
|
|
"ms" => "Montserrat",
|
|
"ma" => "Morocco",
|
|
"mz" => "Mozambique",
|
|
"mm" => "Myanmar",
|
|
"na" => "Namibia",
|
|
"nr" => "Nauru",
|
|
"np" => "Nepal",
|
|
"nl" => "Netherlands",
|
|
"an" => "Netherlands Antilles",
|
|
"nc" => "New Caledonia",
|
|
"nz" => "New Zealand",
|
|
"ni" => "Nicaragua",
|
|
"ne" => "Niger",
|
|
"ng" => "Nigeria",
|
|
"nu" => "Niue",
|
|
"nf" => "Norfolk Island",
|
|
"mp" => "Northern Mariana Islands",
|
|
"no" => "Norway",
|
|
"om" => "Oman",
|
|
"pk" => "Pakistan",
|
|
"pw" => "Palau",
|
|
"ps" => "Palestinian Territory, Occupied",
|
|
"pa" => "Panama",
|
|
"pg" => "Papua New Guinea",
|
|
"py" => "Paraguay",
|
|
"pe" => "Peru",
|
|
"ph" => "Philippines",
|
|
"pn" => "Pitcairn",
|
|
"pl" => "Poland",
|
|
"pt" => "Portugal",
|
|
"pr" => "Puerto Rico",
|
|
"qa" => "Qatar",
|
|
"re" => "Reunion",
|
|
"ro" => "Romania",
|
|
"ru" => "Russian Federation",
|
|
"rw" => "Rwanda",
|
|
"sh" => "Saint Helena",
|
|
"kn" => "Saint Kitts and Nevis",
|
|
"lc" => "Saint Lucia",
|
|
"pm" => "Saint Pierre and Miquelon",
|
|
"vc" => "Saint Vincent and the Grenadines",
|
|
"ws" => "Samoa",
|
|
"sm" => "San Marino",
|
|
"st" => "Sao Tome and Principe",
|
|
"sa" => "Saudi Arabia",
|
|
"sn" => "Senegal",
|
|
"cs" => "Serbia and Montenegro",
|
|
"sc" => "Seychelles",
|
|
"sl" => "Sierra Leone",
|
|
"sg" => "Singapore",
|
|
"sk" => "Slovakia",
|
|
"si" => "Slovenia",
|
|
"sb" => "Solomon Islands",
|
|
"so" => "Somalia",
|
|
"za" => "South Africa",
|
|
"gs" => "South Georgia and the South Sandwich Islands",
|
|
"es" => "Spain",
|
|
"lk" => "Sri Lanka",
|
|
"sd" => "Sudan",
|
|
"sr" => "Suriname",
|
|
"sj" => "Svalbard and Jan Mayen",
|
|
"sz" => "Swaziland",
|
|
"se" => "Sweden",
|
|
"ch" => "Switzerland",
|
|
"sy" => "Syrian Arab Republic",
|
|
"tw" => "Taiwan, Province of China",
|
|
"tj" => "Tajikistan",
|
|
"tz" => "Tanzania, United Republic",
|
|
"th" => "Thailand",
|
|
"tl" => "Timor-Leste",
|
|
"tg" => "Togo",
|
|
"tk" => "Tokelau",
|
|
"to" => "Tonga",
|
|
"tt" => "Trinidad and Tobago",
|
|
"tn" => "Tunisia",
|
|
"tr" => "Turkey",
|
|
"tm" => "Turkmenistan",
|
|
"tc" => "Turks and Caicos Islands",
|
|
"tv" => "Tuvalu",
|
|
"ug" => "Uganda",
|
|
"ua" => "Ukraine",
|
|
"ae" => "United Arab Emirates",
|
|
"uk" => "United Kingdom",
|
|
"us" => "United States",
|
|
"um" => "United States Minor Outlying Islands",
|
|
"uy" => "Uruguay",
|
|
"uz" => "Uzbekistan",
|
|
"vu" => "Vanuatu",
|
|
"ve" => "Venezuela",
|
|
"vn" => "Viet Nam",
|
|
"vg" => "Virgin Islands, British",
|
|
"vi" => "Virgin Islands, U.S.",
|
|
"wf" => "Wallis and Futuna",
|
|
"eh" => "Western Sahara",
|
|
"ye" => "Yemen",
|
|
"zm" => "Zambia",
|
|
"zw" => "Zimbabwe"
|
|
]
|
|
],
|
|
"nsfw" => [
|
|
"display" => "NSFW",
|
|
"option" => [
|
|
"yes" => "Yes", // safe=active
|
|
"no" => "No" // safe=off
|
|
]
|
|
]
|
|
];
|
|
|
|
switch($page){
|
|
|
|
case "web":
|
|
return array_merge(
|
|
$base,
|
|
[
|
|
"lang" => [ // lr=<lang> (prefix lang with "lang_")
|
|
"display" => "Language",
|
|
"option" => [
|
|
"any" => "Any language",
|
|
"ar" => "Arabic",
|
|
"bg" => "Bulgarian",
|
|
"ca" => "Catalan",
|
|
"cs" => "Czech",
|
|
"da" => "Danish",
|
|
"de" => "German",
|
|
"el" => "Greek",
|
|
"en" => "English",
|
|
"es" => "Spanish",
|
|
"et" => "Estonian",
|
|
"fi" => "Finnish",
|
|
"fr" => "French",
|
|
"hr" => "Croatian",
|
|
"hu" => "Hungarian",
|
|
"id" => "Indonesian",
|
|
"is" => "Icelandic",
|
|
"it" => "Italian",
|
|
"iw" => "Hebrew",
|
|
"ja" => "Japanese",
|
|
"ko" => "Korean",
|
|
"lt" => "Lithuanian",
|
|
"lv" => "Latvian",
|
|
"nl" => "Dutch",
|
|
"no" => "Norwegian",
|
|
"pl" => "Polish",
|
|
"pt" => "Portuguese",
|
|
"ro" => "Romanian",
|
|
"ru" => "Russian",
|
|
"sk" => "Slovak",
|
|
"sl" => "Slovenian",
|
|
"sr" => "Serbian",
|
|
"sv" => "Swedish",
|
|
"tr" => "Turkish",
|
|
"zh-CN" => "Chinese (Simplified)",
|
|
"zh-TW" => "Chinese (Traditional)"
|
|
]
|
|
],
|
|
"newer" => [ // tbs
|
|
"display" => "Newer than",
|
|
"option" => "_DATE"
|
|
],
|
|
"older" => [
|
|
"display" => "Older than",
|
|
"option" => "_DATE"
|
|
],
|
|
"spellcheck" => [
|
|
"display" => "Spellcheck",
|
|
"option" => [
|
|
"yes" => "Yes",
|
|
"no" => "No"
|
|
]
|
|
]
|
|
]
|
|
);
|
|
break;
|
|
|
|
case "images":
|
|
return array_merge(
|
|
$base,
|
|
[
|
|
"time" => [ // tbs=qdr:<time>
|
|
"display" => "Time posted",
|
|
"option" => [
|
|
"any" => "Any time",
|
|
"d" => "Past 24 hours",
|
|
"w" => "Past week",
|
|
"m" => "Past month",
|
|
"y" => "Past year"
|
|
]
|
|
],
|
|
"size" => [ // imgsz
|
|
"display" => "Size",
|
|
"option" => [
|
|
"any" => "Any size",
|
|
"l" => "Large",
|
|
"m" => "Medium",
|
|
"i" => "Icon",
|
|
"qsvga" => "Larger than 400x300",
|
|
"vga" => "Larger than 640x480",
|
|
"svga" => "Larger than 800x600",
|
|
"xga" => "Larger than 1024x768",
|
|
"2mp" => "Larger than 2MP",
|
|
"4mp" => "Larger than 4MP",
|
|
"6mp" => "Larger than 6MP",
|
|
"8mp" => "Larger than 8MP",
|
|
"10mp" => "Larger than 10MP",
|
|
"12mp" => "Larger than 12MP",
|
|
"15mp" => "Larger than 15MP",
|
|
"20mp" => "Larger than 20MP",
|
|
"40mp" => "Larger than 40MP",
|
|
"70mp" => "Larger than 70MP"
|
|
]
|
|
],
|
|
"ratio" => [ // imgar
|
|
"display" => "Aspect ratio",
|
|
"option" => [
|
|
"any" => "Any ratio",
|
|
"t|xt" => "Tall",
|
|
"s" => "Square",
|
|
"w" => "Wide",
|
|
"xw" => "Panoramic"
|
|
]
|
|
],
|
|
"color" => [ // imgc
|
|
"display" => "Color",
|
|
"option" => [
|
|
"any" => "Any color",
|
|
"color" => "Full color",
|
|
"bnw" => "Black & white",
|
|
"trans" => "Transparent",
|
|
// from here, imgcolor
|
|
"red" => "Red",
|
|
"orange" => "Orange",
|
|
"yellow" => "Yellow",
|
|
"green" => "Green",
|
|
"teal" => "Teal",
|
|
"blue" => "Blue",
|
|
"purple" => "Purple",
|
|
"pink" => "Pink",
|
|
"white" => "White",
|
|
"gray" => "Gray",
|
|
"black" => "Black",
|
|
"brown" => "Brown"
|
|
]
|
|
],
|
|
"type" => [ // tbs=itp:<type>
|
|
"display" => "Type",
|
|
"option" => [
|
|
"any" => "Any type",
|
|
"clipart" => "Clip Art",
|
|
"lineart" => "Line Drawing",
|
|
"animated" => "Animated"
|
|
]
|
|
],
|
|
"format" => [ // as_filetype
|
|
"display" => "Format",
|
|
"option" => [
|
|
"any" => "Any format",
|
|
"jpg" => "JPG",
|
|
"gif" => "GIF",
|
|
"png" => "PNG",
|
|
"bmp" => "BMP",
|
|
"svg" => "SVG",
|
|
"webp" => "WEBP",
|
|
"ico" => "ICO",
|
|
"craw" => "RAW"
|
|
]
|
|
],
|
|
"rights" => [ // tbs=sur:<rights>
|
|
"display" => "Usage rights",
|
|
"option" => [
|
|
"any" => "Any license",
|
|
"cl" => "Creative Commons licenses",
|
|
"ol" => "Commercial & other licenses"
|
|
]
|
|
]
|
|
]
|
|
);
|
|
break;
|
|
|
|
case "videos":
|
|
return array_merge(
|
|
$base,
|
|
[
|
|
"newer" => [ // tbs
|
|
"display" => "Newer than",
|
|
"option" => "_DATE"
|
|
],
|
|
"older" => [
|
|
"display" => "Older than",
|
|
"option" => "_DATE"
|
|
],
|
|
"duration" => [
|
|
"display" => "Duration",
|
|
"option" => [
|
|
"any" => "Any duration",
|
|
"s" => "Short (0-4min)", // tbs=dur:s
|
|
"m" => "Medium (4-20min)", // tbs=dur:m
|
|
"l" => "Long (20+ min)" // tbs=dur:l
|
|
]
|
|
],
|
|
"quality" => [
|
|
"display" => "Quality",
|
|
"option" => [
|
|
"any" => "Any quality",
|
|
"h" => "High quality" // tbs=hq:h
|
|
]
|
|
],
|
|
"captions" => [
|
|
"display" => "Captions",
|
|
"option" => [
|
|
"any" => "No preference",
|
|
"yes" => "Closed captioned" // tbs=cc:1
|
|
]
|
|
]
|
|
]
|
|
);
|
|
break;
|
|
|
|
case "news":
|
|
return array_merge(
|
|
$base,
|
|
[
|
|
"newer" => [ // tbs
|
|
"display" => "Newer than",
|
|
"option" => "_DATE"
|
|
],
|
|
"older" => [
|
|
"display" => "Older than",
|
|
"option" => "_DATE"
|
|
],
|
|
"sort" => [
|
|
"display" => "Sort",
|
|
"option" => [
|
|
"relevance" => "Relevance",
|
|
"date" => "Date" // sbd:1
|
|
]
|
|
]
|
|
]
|
|
);
|
|
break;
|
|
}
|
|
}
|
|
|
|
private function get($proxy, $url, $get = [], $container = null){
|
|
|
|
if(config::FPLAY_EXTERNAL_ENDPOINT === null){
|
|
|
|
throw new Exception("This scraper requires 4play, and the instance maintainer did not configure it. Please refer to the documentation for more information");
|
|
}
|
|
|
|
try{
|
|
$curlproc = curl_init();
|
|
|
|
if($get !== []){
|
|
$get = http_build_query($get);
|
|
$url .= "?" . $get;
|
|
}
|
|
|
|
if($container === null){
|
|
|
|
$url =
|
|
config::FPLAY_EXTERNAL_ENDPOINT .
|
|
"?url=" . urlencode($url) .
|
|
"&pw=" . urlencode(config::FPLAY_PASSWORD) .
|
|
"&container=" .// urlencode($container) .
|
|
"&proxy=" . urlencode($proxy) .
|
|
"&url_match=" . urlencode("&sei=") .
|
|
"&fail_url_match=" . urlencode("\/sorry\/");
|
|
}else{
|
|
|
|
$url =
|
|
config::FPLAY_EXTERNAL_ENDPOINT .
|
|
"?url=" . urlencode($url) .
|
|
"&pw=" . urlencode(config::FPLAY_PASSWORD) .
|
|
"&container=" . urlencode($container) .
|
|
"&proxy=" . urlencode($proxy);
|
|
}
|
|
|
|
$header_out = [];
|
|
curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$header_out){
|
|
|
|
$length = strlen($header);
|
|
$header_out[] = trim($header);
|
|
|
|
return $length;
|
|
});
|
|
|
|
curl_setopt($curlproc, CURLOPT_URL, $url);
|
|
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
|
|
|
curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
|
|
curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
|
|
"User-Agent: " . config::USER_AGENT_FRIENDLY,
|
|
"Accept: text/plain"
|
|
]);
|
|
|
|
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
|
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
|
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
|
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
|
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
|
|
|
curl_setopt($curlproc, CURLOPT_FOLLOWLOCATION, true);
|
|
|
|
//$this->backend->assign_proxy($curlproc, $proxy);
|
|
|
|
$data = curl_exec($curlproc);
|
|
|
|
if(curl_errno($curlproc)){
|
|
|
|
throw new Exception(curl_error($curlproc));
|
|
}
|
|
|
|
curl_close($curlproc);
|
|
}catch(Exception $error){
|
|
|
|
throw new Exception("4play API is currently unreachable. Render server offline (probably) woopsies");
|
|
}
|
|
|
|
// check for failstate
|
|
if(
|
|
str_contains(
|
|
$header_out[0],
|
|
"400"
|
|
)
|
|
){
|
|
|
|
throw new Exception("Page renderer returned an error: " . $data);
|
|
}
|
|
|
|
// parse headers
|
|
$container = null;
|
|
|
|
foreach($header_out as $header){
|
|
|
|
$header = explode(":", $header, 2);
|
|
|
|
if(strtolower($header[0]) == "x-container-id"){
|
|
|
|
$container = trim($header[1]);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if($container === null){
|
|
|
|
throw new Exception("The 4play API did not return a valid container ID. Make sure your configuration forwards X-Container-ID headers properly");
|
|
}
|
|
|
|
return [
|
|
"data" => $data,
|
|
"container" => $container
|
|
];
|
|
}
|
|
|
|
|
|
public function web($get){
|
|
|
|
if($get["npt"]){
|
|
|
|
[$get, $proxy] = $this->backend->get($get["npt"], "web");
|
|
|
|
$get = json_decode($get, true);
|
|
$container = $get["c"];
|
|
$get = $get["u"];
|
|
|
|
$data =
|
|
$this->get(
|
|
$proxy,
|
|
"https://www.google.com" . $get,
|
|
[],
|
|
$container
|
|
);
|
|
|
|
$html = &$data["data"];
|
|
$container = &$data["container"];
|
|
|
|
}else{
|
|
|
|
$search = $get["s"];
|
|
if(strlen($search) === 0){
|
|
|
|
throw new Exception("Search term is empty!");
|
|
}
|
|
/*
|
|
$session = $this->backend->session_get();
|
|
|
|
if(config::FPLAY_ENABLE_API === false){
|
|
|
|
throw new Exception("This scraper requires the 4play session generator. Check the documentation for more information");
|
|
}
|
|
|
|
if($session === false){
|
|
|
|
throw new Exception("Session database is empty");
|
|
}
|
|
|
|
$proxy = &$session["proxy"];
|
|
$headers = &$session["headers"];
|
|
*/
|
|
|
|
$proxy = $this->backend->get_ip();
|
|
|
|
//
|
|
// Generate params
|
|
//
|
|
|
|
$params = [
|
|
"q" => $get["s"],
|
|
"hl" => "en",
|
|
"udm" => 14
|
|
];
|
|
|
|
// country
|
|
if($get["country"] != "any"){
|
|
|
|
$params["gl"] = $country;
|
|
}
|
|
|
|
// nsfw
|
|
$params["safe"] = $get["nsfw"] == "yes" ? "off" : "active";
|
|
|
|
// language
|
|
if($get["lang"] != "any"){
|
|
|
|
$params["lr"] = "lang_" . $lang;
|
|
}
|
|
|
|
// generate tbs
|
|
$tbs = [];
|
|
|
|
// get date
|
|
$get["older"] = $get["older"] === false ? null : date("m/d/Y", $older);
|
|
$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $newer);
|
|
|
|
if(
|
|
$get["older"] !== null ||
|
|
$get["newer"] !== null
|
|
){
|
|
|
|
$tbs["cdr"] = "1";
|
|
$tbs["cd_min"] = $get["newer"];
|
|
$tbs["cd_max"] = $get["older"];
|
|
}
|
|
|
|
// spellcheck filter
|
|
if($get["spellcheck"] == "no"){
|
|
|
|
$params["nfpr"] = "1";
|
|
}
|
|
|
|
if(count($tbs) !== 0){
|
|
|
|
$params["tbs"] = "";
|
|
|
|
foreach($tbs as $key => $value){
|
|
|
|
$params["tbs"] .= $key . ":" . $value . ",";
|
|
}
|
|
|
|
$params["tbs"] = rtrim($params["tbs"], ",");
|
|
}
|
|
|
|
$data = $this->get(
|
|
$proxy,
|
|
"https://www.google.com/search",
|
|
$params,
|
|
null
|
|
);
|
|
|
|
$html = &$data["data"];
|
|
$container = &$data["container"];
|
|
|
|
//file_put_contents("scraper/google-jp.html", $html);
|
|
//$html = file_get_contents("scraper/sublink.html");
|
|
//$container = "";
|
|
}
|
|
|
|
$out = [
|
|
"status" => "ok",
|
|
"spelling" => [
|
|
"type" => "no_correction",
|
|
"using" => null,
|
|
"correction" => null
|
|
],
|
|
"npt" => null,
|
|
"answer" => [],
|
|
"web" => [],
|
|
"image" => [],
|
|
"video" => [],
|
|
"news" => [],
|
|
"related" => []
|
|
];
|
|
|
|
$this->fuckhtml->load($html);
|
|
|
|
// solve softcaptcha
|
|
$softcaptcha = $this->detect_sorry();
|
|
|
|
if($softcaptcha !== false){
|
|
|
|
$params["sei"] = $softcaptcha["sei"];
|
|
|
|
$data2 = $this->get(
|
|
$proxy,
|
|
"https://www.google.com/search",
|
|
$params,
|
|
$container
|
|
);
|
|
|
|
$html = &$data2["data"];
|
|
|
|
$this->fuckhtml->load($html);
|
|
}
|
|
|
|
$this->parsestyles();
|
|
|
|
$this->scrape_dimg($html);
|
|
$this->scrape_imagearr($html);
|
|
|
|
$content =
|
|
$this->fuckhtml
|
|
->getElementById(
|
|
"rso"
|
|
);
|
|
|
|
if($content === null){
|
|
|
|
throw new Exception("Failed to grep search div");
|
|
}
|
|
|
|
// get encoded links
|
|
$encoded_links =
|
|
explode(
|
|
"function(){var m={",
|
|
$html,
|
|
2
|
|
);
|
|
|
|
if(count($encoded_links) === 2){
|
|
|
|
$encoded_links =
|
|
json_decode(
|
|
$this->fuckhtml
|
|
->extract_json(
|
|
"{" . $encoded_links[1]
|
|
),
|
|
true
|
|
);
|
|
}else{
|
|
|
|
$encoded_links = [];
|
|
}
|
|
|
|
// get the "did you mean" bullshit
|
|
$taw =
|
|
$this->fuckhtml
|
|
->getElementById("taw");
|
|
|
|
if($taw !== null){
|
|
$this->fuckhtml->load($taw);
|
|
|
|
$as =
|
|
$this->fuckhtml
|
|
->getElementsByTagName("a");
|
|
|
|
$links = [];
|
|
|
|
foreach($as as $a){
|
|
|
|
$href =
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$a["attributes"]["href"]
|
|
);
|
|
|
|
if(preg_match('/^\/search\?/', $href)){
|
|
|
|
$links[] =
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$a
|
|
);
|
|
}
|
|
}
|
|
|
|
switch(count($links)){
|
|
|
|
case 1:
|
|
$out["spelling"] = [
|
|
"type" => "including",
|
|
"using" => $params["q"],
|
|
"correction" => $links[0]
|
|
];
|
|
break;
|
|
|
|
case 2:
|
|
$out["spelling"] = [
|
|
"type" => "including",
|
|
"using" => $links[0],
|
|
"correction" => $links[1]
|
|
];
|
|
break;
|
|
}
|
|
|
|
// reset
|
|
$this->fuckhtml->load($html);
|
|
}
|
|
|
|
$bottom =
|
|
$this->fuckhtml
|
|
->getElementById("botstuff");
|
|
|
|
if($bottom !== null){
|
|
|
|
$this->fuckhtml->load($bottom);
|
|
|
|
// get next page thingy
|
|
$npt =
|
|
$this->fuckhtml
|
|
->getElementById(
|
|
"pnnext"
|
|
);
|
|
|
|
if($npt !== false){
|
|
|
|
$out["npt"] =
|
|
$this->backend->store(
|
|
json_encode([
|
|
"u" =>
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$npt["attributes"]["href"]
|
|
),
|
|
"c" => $container
|
|
]),
|
|
"web",
|
|
$proxy
|
|
);
|
|
}
|
|
|
|
// get lumendatabase notices
|
|
$notices = [];
|
|
$as =
|
|
$this->fuckhtml
|
|
->getElementsByTagName("a");
|
|
|
|
foreach($as as $a){
|
|
|
|
if(!isset($a["attributes"]["href"])){ continue; }
|
|
|
|
$link =
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$a["attributes"]["href"]
|
|
);
|
|
|
|
$link_text =
|
|
strtolower(
|
|
$this->fuckhtml
|
|
->getTextContent($a)
|
|
);
|
|
|
|
if(
|
|
str_contains(
|
|
$link_text,
|
|
"complaint"
|
|
)
|
|
){
|
|
|
|
// sometimes the link is an encrypted redirect, handle redirection securely
|
|
if(
|
|
preg_match(
|
|
'/^\/goto\?url=(.*)/',
|
|
$link,
|
|
$payload
|
|
)
|
|
){
|
|
|
|
$notices[] = "/resolver?scraper=google&target=" . $payload[1];
|
|
}else{
|
|
|
|
// normal link
|
|
$notices[] = $link;
|
|
}
|
|
}
|
|
}
|
|
|
|
// push notices to dom
|
|
if(count($notices) !== 0){
|
|
|
|
$notices_dom = [];
|
|
|
|
$i = 0;
|
|
$c = count($notices);
|
|
|
|
$notices_dom = [
|
|
[
|
|
"type" => "text",
|
|
"value" => "Google removed results from this page, but you can get them anyways by consulting these takedown notices: "
|
|
]
|
|
];
|
|
|
|
foreach($notices as $n){
|
|
|
|
$i++;
|
|
|
|
$notices_dom[] = [
|
|
"type" => "link",
|
|
"value" => "Notice #{$i}",
|
|
"url" => $n
|
|
];
|
|
|
|
if($i !== count($notices)){
|
|
|
|
$notices_dom[] = [
|
|
"type" => "text",
|
|
"value" => ", "
|
|
];
|
|
}
|
|
}
|
|
|
|
$out["answer"][] = [
|
|
"title" => "LumenDB notices",
|
|
"description" => $notices_dom,
|
|
"url" => null,
|
|
"thumb" => null,
|
|
"table" => [],
|
|
"sublink" => []
|
|
];
|
|
}
|
|
}
|
|
|
|
$this->fuckhtml->load($content);
|
|
|
|
// get search results
|
|
$results =
|
|
$this->fuckhtml
|
|
->getElementsByAttributeName(
|
|
"data-rpos",
|
|
"div"
|
|
);
|
|
|
|
foreach($results as $result){
|
|
|
|
$this->fuckhtml->load($result);
|
|
|
|
// get title
|
|
$title =
|
|
$this->fuckhtml
|
|
->getElementsByTagName("h3");
|
|
|
|
if(count($title) === 0){
|
|
|
|
// should not happen
|
|
continue;
|
|
}
|
|
|
|
$title =
|
|
$this->titledots(
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$title[0]
|
|
)
|
|
);
|
|
|
|
$url =
|
|
$this->fuckhtml
|
|
->getElementsByTagName(
|
|
"a"
|
|
);
|
|
|
|
if(
|
|
count($url) === 0 ||
|
|
!isset($url[0]["attributes"]["href"])
|
|
){
|
|
|
|
continue;
|
|
}
|
|
|
|
$url =
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$url[0]["attributes"]["href"]
|
|
);
|
|
|
|
if(
|
|
preg_match(
|
|
'/^\/goto\?url/',
|
|
$url
|
|
)
|
|
){
|
|
|
|
// encrypted url detected
|
|
// get reference div
|
|
$refdivs =
|
|
$this->fuckhtml
|
|
->getElementsByAttributeName(
|
|
"id",
|
|
"div"
|
|
);
|
|
|
|
foreach($refdivs as $r){
|
|
|
|
if(
|
|
preg_match(
|
|
'/^atritem-/',
|
|
$r["attributes"]["id"]
|
|
) &&
|
|
isset($r["attributes"]["jsdata"])
|
|
){
|
|
|
|
$ref = explode(";", $r["attributes"]["jsdata"]);
|
|
$ref = $ref[count($ref) - 1];
|
|
|
|
if(isset($encoded_links[$ref][32][3][0])){
|
|
|
|
$url = $encoded_links[$ref][32][3][0];
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// get citation text. contains table elements
|
|
$table = [];
|
|
$date = null;
|
|
|
|
$cite =
|
|
$this->fuckhtml
|
|
->getElementsByTagName("cite");
|
|
|
|
if(isset($cite[0])){
|
|
|
|
$texts =
|
|
explode(
|
|
" · ",
|
|
$this->fuckhtml
|
|
->getTextContent($cite[0])
|
|
);
|
|
|
|
foreach($texts as $text){
|
|
|
|
$text = trim($text);
|
|
|
|
preg_match(
|
|
'/([0-9+kmb,.]+) +(likes?|views?|comments?|followers?|subscribers?|(?:hours?|days?|months?|years?) +ago)/i',
|
|
$text,
|
|
$probe
|
|
);
|
|
|
|
if(isset($probe[1])){
|
|
|
|
// is it date?
|
|
if(
|
|
preg_match(
|
|
'/ago$/i',
|
|
$text
|
|
)
|
|
){
|
|
|
|
$time = strtotime($text);
|
|
|
|
if($time !== false){ $date = $time; }
|
|
continue;
|
|
}
|
|
|
|
// anything else?
|
|
$table[ucfirst($probe[2])] = trim($probe[1]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// get description/image wrapper
|
|
$desc_img_wrappers =
|
|
$this->fuckhtml
|
|
->getElementsByAttributeName(
|
|
"data-sncf",
|
|
"div"
|
|
);
|
|
|
|
$description = null;
|
|
$thumb = [
|
|
"url" => null,
|
|
"ratio" => null
|
|
];
|
|
|
|
foreach($desc_img_wrappers as $wrapper){
|
|
|
|
$this->fuckhtml->load($wrapper);
|
|
|
|
// probe for image
|
|
$image =
|
|
$this->fuckhtml
|
|
->getElementsByTagName(
|
|
"img"
|
|
);
|
|
|
|
if(count($image) !== 0){
|
|
|
|
// handle thumbnails
|
|
$i = null;
|
|
|
|
if(isset($image[0]["attributes"]["id"])){
|
|
|
|
$i = $this->getdimg($image[0]["attributes"]["id"]);
|
|
}
|
|
|
|
if($i !== null){
|
|
|
|
$thumb = [
|
|
"ratio" => "1:1",
|
|
"url" => $i
|
|
];
|
|
}
|
|
}else{
|
|
|
|
if($description === null){
|
|
|
|
// parse it as description
|
|
$description =
|
|
$this->titledots(
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$wrapper
|
|
)
|
|
);
|
|
}else{
|
|
|
|
// parse table elements
|
|
$ts =
|
|
explode(
|
|
" · ",
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$wrapper
|
|
)
|
|
);
|
|
|
|
foreach($ts as $t){
|
|
|
|
$t = trim(html_entity_decode($t));
|
|
|
|
if(preg_match("/\p{Sc}/u", $t)){
|
|
|
|
$table["Price"] = $t;
|
|
continue;
|
|
}
|
|
|
|
if(str_contains($t, "stock")){
|
|
|
|
$table["Stock"] = $t;
|
|
continue;
|
|
}
|
|
|
|
if(str_contains($t, "return")){
|
|
|
|
$table["Returns"] = $t;
|
|
continue;
|
|
}
|
|
|
|
if(str_contains($t, "delivery")){
|
|
|
|
$table["Delivery"] = $t;
|
|
continue;
|
|
}
|
|
|
|
// 5.0(590,004)
|
|
if(preg_match('/^([0-9kmb\/%.,]+) *(\([0-9kmb,.]+\))?/', $t, $match)){
|
|
|
|
if(isset($match[2])){
|
|
|
|
$table["Ratings"] = "{$match[1]} {$match[2]}";
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if(str_contains($t, "rating")){
|
|
|
|
$table["Ratings"] = $t;
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// extract date from description
|
|
// Nov 6, 2017 — Keira Hagai from Jak and Daxter. I took off her belt/suspenders/goggles. Originally posted by DemonBoy on Models Resource, and edited and posted to Sketchfab
|
|
if($date === null){
|
|
$date_probe =
|
|
explode(
|
|
" — ",
|
|
$description,
|
|
2
|
|
);
|
|
|
|
if(count($date_probe) === 2){
|
|
|
|
$time = strtotime($date_probe[0]);
|
|
|
|
if($time !== false){
|
|
|
|
$description = $date_probe[1];
|
|
$date = $time;
|
|
}
|
|
}
|
|
}
|
|
|
|
$sublinks = [];
|
|
|
|
// get sublinks (rare)
|
|
$as =
|
|
$this->fuckhtml
|
|
->getElementsByClassName(
|
|
$this->getstyle([
|
|
"padding-right" => "6px",
|
|
"margin-top" => "4px",
|
|
"display" => "inline-block"
|
|
]),
|
|
"a"
|
|
);
|
|
|
|
foreach($as as $sublink){
|
|
|
|
$sublink_url =
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$sublink["attributes"]["href"]
|
|
);
|
|
|
|
if(
|
|
preg_match(
|
|
'/^\/goto\?url=(.*)/',
|
|
$sublink_url,
|
|
$piece_of_shit
|
|
)
|
|
){
|
|
|
|
// encrypted piece of shit url
|
|
$sublink_url = "/resolver?scraper=google&target=" . $piece_of_shit[1];
|
|
}
|
|
|
|
$sublinks[] = [
|
|
"title" =>
|
|
$this->fuckhtml
|
|
->getTextContent($sublink),
|
|
"date" => null,
|
|
"url" => $sublink_url
|
|
];
|
|
}
|
|
|
|
$out["web"][] = [
|
|
"title" => $title,
|
|
"description" => $description,
|
|
"url" => $url,
|
|
"date" => $date,
|
|
"type" => "web",
|
|
"thumb" => $thumb,
|
|
"sublink" => $sublinks,
|
|
"table" => $table
|
|
];
|
|
}
|
|
|
|
return $out;
|
|
}
|
|
|
|
|
|
|
|
public function image($get){
|
|
|
|
// generate parameters
|
|
if($get["npt"]){
|
|
|
|
[$params, $proxy] =
|
|
$this->backend->get(
|
|
$get["npt"],
|
|
"images"
|
|
);
|
|
|
|
$params = json_decode($params, true);
|
|
$container = $params["c"];
|
|
unset($params["c"]);
|
|
}else{
|
|
|
|
$search = $get["s"];
|
|
if(strlen($search) === 0){
|
|
|
|
throw new Exception("Search term is empty!");
|
|
}
|
|
|
|
$proxy = $this->backend->get_ip();
|
|
$country = $get["country"];
|
|
$nsfw = $get["nsfw"];
|
|
$time = $get["time"];
|
|
$size = $get["size"];
|
|
$ratio = $get["ratio"];
|
|
$color = $get["color"];
|
|
$type = $get["type"];
|
|
$format = $get["format"];
|
|
$rights = $get["rights"];
|
|
|
|
$params = [
|
|
"q" => $search,
|
|
"udm" => "2", // get images
|
|
"start" => 0
|
|
];
|
|
|
|
// country (image search uses cr instead of gl)
|
|
if($country != "any"){
|
|
|
|
$params["cr"] = "country" . strtoupper($country);
|
|
}
|
|
|
|
// nsfw
|
|
$params["safe"] = $nsfw == "yes" ? "off" : "active";
|
|
|
|
// generate tbs
|
|
$tbs = [];
|
|
|
|
// time
|
|
if($time != "any"){
|
|
|
|
$tbs["qdr"] = $time;
|
|
}
|
|
|
|
// size
|
|
if($size != "any"){
|
|
|
|
$params["imgsz"] = $size;
|
|
}
|
|
|
|
// ratio
|
|
if($ratio != "any"){
|
|
|
|
$params["imgar"] = $ratio;
|
|
}
|
|
|
|
// color
|
|
if($color != "any"){
|
|
|
|
if(
|
|
$color == "color" ||
|
|
$color == "trans"
|
|
){
|
|
|
|
$params["imgc"] = $color;
|
|
}elseif($color == "bnw"){
|
|
|
|
$params["imgc"] = "gray";
|
|
}else{
|
|
|
|
$tbs["ic"] = "specific";
|
|
$tbs["isc"] = $color;
|
|
}
|
|
}
|
|
|
|
// type
|
|
if($type != "any"){
|
|
|
|
$tbs["itp"] = $type;
|
|
}
|
|
|
|
// format
|
|
if($format != "any"){
|
|
|
|
$params["as_filetype"] = $format;
|
|
}
|
|
|
|
// rights (tbs)
|
|
if($rights != "any"){
|
|
|
|
$tbs["sur"] = $rights;
|
|
}
|
|
|
|
// append tbs
|
|
if(count($tbs) !== 0){
|
|
|
|
$params["tbs"] = "";
|
|
|
|
foreach($tbs as $key => $value){
|
|
|
|
$params["tbs"] .= $key . ":" . $value . ",";
|
|
}
|
|
|
|
$params["tbs"] = rtrim($params["tbs"], ",");
|
|
}
|
|
}
|
|
|
|
//$html = file_get_contents("scraper/google.html");
|
|
|
|
$data = $this->get(
|
|
$proxy,
|
|
"https://www.google.com/search",
|
|
$params
|
|
);
|
|
|
|
$html = &$data["data"];
|
|
$container = &$data["container"];
|
|
|
|
$this->fuckhtml->load($html);
|
|
|
|
// parse all <style> tags
|
|
$this->parsestyles();
|
|
|
|
// solve softcaptcha
|
|
$softcaptcha = $this->detect_sorry();
|
|
|
|
if($softcaptcha !== false){
|
|
|
|
$params["sei"] = $softcaptcha["sei"];
|
|
|
|
$data2 = $this->get(
|
|
$proxy,
|
|
"https://www.google.com/search",
|
|
$params,
|
|
$container
|
|
);
|
|
|
|
$html = &$data2["data"];
|
|
|
|
$this->fuckhtml->load($html);
|
|
$this->parsestyles();
|
|
}
|
|
|
|
// get javascript images
|
|
$this->scrape_imagearr($html);
|
|
|
|
$out = [
|
|
"status" => "ok",
|
|
"npt" => null,
|
|
"image" => []
|
|
];
|
|
|
|
$images =
|
|
$this->fuckhtml
|
|
->getElementsByClassName(
|
|
"ivg-i",
|
|
"div"
|
|
);
|
|
|
|
foreach($images as $div){
|
|
|
|
$this->fuckhtml->load($div);
|
|
|
|
$image =
|
|
$this->fuckhtml
|
|
->getElementsByTagName("img")[0];
|
|
|
|
// make sure we dont attempt to show an image we dont have data for
|
|
if(
|
|
isset($div["attributes"]["data-docid"]) &&
|
|
isset($this->image_arr[$div["attributes"]["data-docid"]])
|
|
){
|
|
|
|
$source =
|
|
$this->image_arr[
|
|
$div["attributes"]["data-docid"]
|
|
];
|
|
}else{
|
|
|
|
continue;
|
|
}
|
|
|
|
$out["image"][] = [
|
|
"title" =>
|
|
$this->titledots(
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$image["attributes"]["alt"]
|
|
)
|
|
),
|
|
"source" => $source,
|
|
"url" =>
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$div["attributes"]["data-lpage"]
|
|
)
|
|
];
|
|
}
|
|
|
|
// as usual, no way to check if there is a next page reliably
|
|
if(count($out["image"]) > 50){
|
|
|
|
if(!isset($params["start"])){
|
|
|
|
$params["start"] = 10;
|
|
}else{
|
|
|
|
$params["start"] += 10;
|
|
}
|
|
|
|
$out["npt"] =
|
|
$this->backend
|
|
->store(
|
|
json_encode($params),
|
|
"image",
|
|
$proxy
|
|
);
|
|
}
|
|
|
|
return $out;
|
|
}
|
|
|
|
|
|
|
|
public function video($get){
|
|
|
|
if($get["npt"]){
|
|
|
|
[$params, $proxy] = $this->backend->get($get["npt"], "video");
|
|
$params = json_decode($params, true);
|
|
|
|
$container = $params["c"];
|
|
unset($params["c"]);
|
|
|
|
}else{
|
|
$search = $get["s"];
|
|
$country = $get["country"];
|
|
$nsfw = $get["nsfw"];
|
|
$older = $get["older"];
|
|
$newer = $get["newer"];
|
|
$duration = $get["duration"];
|
|
$quality = $get["quality"];
|
|
$captions = $get["captions"];
|
|
|
|
$container = null;
|
|
$proxy = $this->backend->get_ip();
|
|
|
|
$params = [
|
|
"q" => $search,
|
|
"udm" => "7",
|
|
"hl" => "en"
|
|
];
|
|
|
|
// country
|
|
if($country != "any"){
|
|
|
|
$params["gl"] = $country;
|
|
}
|
|
|
|
// nsfw
|
|
$params["safe"] = $nsfw == "yes" ? "off" : "active";
|
|
|
|
$tbs = [];
|
|
|
|
// get date
|
|
$older = $older === false ? null : date("m/d/Y", $older);
|
|
$newer = $newer === false ? null : date("m/d/Y", $newer);
|
|
|
|
if(
|
|
$older !== null ||
|
|
$newer !== null
|
|
){
|
|
|
|
$tbs["cdr"] = "1";
|
|
$tbs["cd_min"] = $newer;
|
|
$tbs["cd_max"] = $older;
|
|
}
|
|
|
|
// duration
|
|
if($duration != "any"){
|
|
|
|
$tbs[] = "dur:" . $duration;
|
|
}
|
|
|
|
// quality
|
|
if($quality != "any"){
|
|
|
|
$tbs[] = "hq:" . $quality;
|
|
}
|
|
|
|
// captions
|
|
if($captions != "any"){
|
|
|
|
$tbs[] = "cc:" . $captions;
|
|
}
|
|
|
|
// append tbs
|
|
if(count($tbs) !== 0){
|
|
|
|
$params["tbs"] =
|
|
implode(",", $tbs);
|
|
}
|
|
}
|
|
|
|
$data = $this->get(
|
|
$proxy,
|
|
"https://www.google.com/search",
|
|
$params,
|
|
$container
|
|
);
|
|
|
|
$html = &$data["data"];
|
|
$container = &$data["container"];
|
|
|
|
//$html = file_get_contents("scraper/google.html");
|
|
|
|
if(!isset($params["start"])){
|
|
|
|
$params["start"] = 0;
|
|
}
|
|
$params["start"] += 20;
|
|
|
|
$this->fuckhtml->load($html);
|
|
|
|
// parse all <style> tags
|
|
$this->parsestyles();
|
|
|
|
// solve softcaptcha
|
|
$softcaptcha = $this->detect_sorry();
|
|
|
|
if($softcaptcha !== false){
|
|
|
|
$params["sei"] = $softcaptcha["sei"];
|
|
|
|
$data2 = $this->get(
|
|
$proxy,
|
|
"https://www.google.com/search",
|
|
$params,
|
|
$container
|
|
);
|
|
|
|
$html = &$data2["data"];
|
|
|
|
$this->fuckhtml->load($html);
|
|
$this->parsestyles();
|
|
}
|
|
|
|
//
|
|
// Parse web video page
|
|
//
|
|
|
|
// get javascript images
|
|
$this->scrape_dimg($html);
|
|
|
|
$this->scrape_imagearr($html);
|
|
|
|
$params["c"] = $container;
|
|
|
|
// get encoded links
|
|
$encoded_links =
|
|
explode(
|
|
"function(){var m={",
|
|
$html,
|
|
2
|
|
);
|
|
|
|
if(count($encoded_links) === 2){
|
|
|
|
$encoded_links =
|
|
json_decode(
|
|
$this->fuckhtml
|
|
->extract_json(
|
|
"{" . $encoded_links[1]
|
|
),
|
|
true
|
|
);
|
|
}else{
|
|
|
|
$encoded_links = [];
|
|
}
|
|
|
|
$out = [
|
|
"status" => "ok",
|
|
"npt" =>
|
|
$this->backend->store(
|
|
json_encode($params),
|
|
"videos",
|
|
$proxy
|
|
),
|
|
"video" => [],
|
|
"author" => [],
|
|
"livestream" => [],
|
|
"playlist" => [],
|
|
"reel" => []
|
|
];
|
|
|
|
$search_div =
|
|
$this->fuckhtml
|
|
->getElementById(
|
|
"center_col"
|
|
);
|
|
|
|
if($search_div === false){
|
|
|
|
throw new Exception("Failed to grep search div");
|
|
}
|
|
|
|
$this->fuckhtml->load($search_div);
|
|
|
|
$results =
|
|
$this->fuckhtml
|
|
->getElementsByClassName(
|
|
$this->getstyle([
|
|
"margin" => "0px 0px 30px"
|
|
]),
|
|
"div"
|
|
);
|
|
|
|
foreach($results as $result){
|
|
|
|
$this->fuckhtml->load($result);
|
|
|
|
$url =
|
|
$this->fuckhtml
|
|
->getElementsByTagName(
|
|
"a"
|
|
);
|
|
|
|
if(count($url) === 0){
|
|
|
|
// no url, weird, continue
|
|
continue;
|
|
}
|
|
|
|
$title =
|
|
$this->fuckhtml
|
|
->getElementsByTagName(
|
|
"h3"
|
|
);
|
|
|
|
if(count($title) === 0){
|
|
|
|
// no title, weird, continue
|
|
continue;
|
|
}
|
|
|
|
// get description
|
|
$description =
|
|
$this->fuckhtml
|
|
->getElementsByClassName(
|
|
$this->getstyle([
|
|
"-webkit-box-orient" => "vertical",
|
|
"display" => "-webkit-box",
|
|
"-webkit-line-clamp" => "2",
|
|
"overflow" => "hidden",
|
|
"word-break" => "break-word"
|
|
]),
|
|
"div"
|
|
);
|
|
|
|
if(count($description) === 0){
|
|
|
|
$description = null;
|
|
}else{
|
|
|
|
$description =
|
|
html_entity_decode(
|
|
$this->titledots(
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$description[0]
|
|
)
|
|
)
|
|
);
|
|
}
|
|
|
|
// get author + date posted
|
|
$metadiv =
|
|
$this->fuckhtml
|
|
->getElementsByClassName(
|
|
$this->getstyle([
|
|
"margin-top" => "12px"
|
|
]),
|
|
"div"
|
|
);
|
|
|
|
$author = null;
|
|
$date = null;
|
|
|
|
if(count($metadiv) !== 0){
|
|
|
|
$metadiv =
|
|
explode(
|
|
"·",
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$metadiv[0]
|
|
)
|
|
);
|
|
|
|
if(count($metadiv) === 3){
|
|
|
|
$author = trim($metadiv[1]);
|
|
$date = strtotime(trim($metadiv[2]));
|
|
}elseif(count($metadiv) === 2){
|
|
|
|
$author = trim($metadiv[0]);
|
|
$date = strtotime(trim($metadiv[1]));
|
|
}
|
|
}
|
|
|
|
$thumb = [
|
|
"url" => null,
|
|
"ratio" => null
|
|
];
|
|
|
|
$image =
|
|
$this->fuckhtml
|
|
->getElementsByTagName(
|
|
"img"
|
|
);
|
|
|
|
$duration = null;
|
|
|
|
if(
|
|
count($image) !== 0 &&
|
|
isset($image[0]["attributes"]["id"])
|
|
){
|
|
|
|
$thumb = [
|
|
"url" => $this->getdimg($image[0]["attributes"]["id"]),
|
|
"ratio" => "16:9"
|
|
];
|
|
|
|
// get duration
|
|
$duration =
|
|
$this->fuckhtml
|
|
->getElementsByClassName(
|
|
$this->getstyle([
|
|
"background-color" => "rgba(0,0,0,0.6)",
|
|
"color" => "#fff",
|
|
"fill" => "#fff"
|
|
])
|
|
);
|
|
|
|
if(count($duration) !== 0){
|
|
|
|
$duration =
|
|
$this->hms2int(
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$duration[0]
|
|
));
|
|
}else{
|
|
|
|
$duration = null;
|
|
}
|
|
}
|
|
|
|
$url =
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$url[0]["attributes"]["href"]
|
|
);
|
|
|
|
if(
|
|
preg_match(
|
|
'/^\/goto\?url/',
|
|
$url
|
|
)
|
|
){
|
|
|
|
// encrypted url detected
|
|
// get reference div
|
|
$refdivs =
|
|
$this->fuckhtml
|
|
->getElementsByAttributeName(
|
|
"id",
|
|
"div"
|
|
);
|
|
|
|
foreach($refdivs as $r){
|
|
|
|
if(
|
|
preg_match(
|
|
'/^atritem-/',
|
|
$r["attributes"]["id"]
|
|
) &&
|
|
isset($r["attributes"]["jsdata"])
|
|
){
|
|
|
|
$ref = explode(";", $r["attributes"]["jsdata"]);
|
|
$ref = $ref[count($ref) - 1];
|
|
|
|
if(isset($encoded_links[$ref][32][11][7])){
|
|
|
|
$url = $encoded_links[$ref][32][11][7];
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
$out["video"][] = [
|
|
"title" =>
|
|
$this->titledots(
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$title[0]
|
|
)
|
|
),
|
|
"description" => $description,
|
|
"author" => [
|
|
"name" => $author,
|
|
"url" => null,
|
|
"avatar" => null
|
|
],
|
|
"date" => $date,
|
|
"duration" => $duration,
|
|
"views" => null,
|
|
"thumb" => $thumb,
|
|
"url" => $url
|
|
];
|
|
}
|
|
|
|
return $out;
|
|
}
|
|
|
|
|
|
public function news($get){
|
|
|
|
if($get["npt"]){
|
|
|
|
[$req, $proxy] = $this->backend->get($get["npt"], "news");
|
|
|
|
$req = json_decode($req, true);
|
|
$container = $req["c"];
|
|
$req = $req["u"];
|
|
|
|
$data = $this->get(
|
|
$proxy,
|
|
"https://www.google.com" . $req,
|
|
[],
|
|
$container
|
|
);
|
|
|
|
$html = &$data["data"];
|
|
$container = &$data["container"];
|
|
|
|
}else{
|
|
$search = $get["s"];
|
|
$country = $get["country"];
|
|
$nsfw = $get["nsfw"];
|
|
$older = $get["older"];
|
|
$newer = $get["newer"];
|
|
$sort = $get["sort"];
|
|
|
|
$container = null;
|
|
$proxy = $this->backend->get_ip();
|
|
|
|
$params = [
|
|
"q" => $search,
|
|
"tbm" => "nws",
|
|
"hl" => "en"
|
|
];
|
|
|
|
// country
|
|
if($country != "any"){
|
|
|
|
$params["gl"] = $country;
|
|
}
|
|
|
|
// nsfw
|
|
$params["safe"] = $nsfw == "yes" ? "off" : "active";
|
|
|
|
$tbs = [];
|
|
|
|
// get date
|
|
$older = $older === false ? null : date("m/d/Y", $older);
|
|
$newer = $newer === false ? null : date("m/d/Y", $newer);
|
|
|
|
if(
|
|
$older !== null ||
|
|
$newer !== null
|
|
){
|
|
|
|
$tbs["cdr"] = "1";
|
|
$tbs["cd_min"] = $newer;
|
|
$tbs["cd_max"] = $older;
|
|
}
|
|
|
|
// relevance
|
|
if($sort == "date"){
|
|
|
|
$tbs["sbd"] = "1";
|
|
}
|
|
|
|
// append tbs
|
|
if(count($tbs) !== 0){
|
|
|
|
$params["tbs"] = "";
|
|
|
|
foreach($tbs as $key => $value){
|
|
|
|
$params["tbs"] .= $key . ":" . $value . ",";
|
|
}
|
|
|
|
$params["tbs"] = rtrim($params["tbs"], ",");
|
|
}
|
|
|
|
//$html = file_get_contents("scraper/google.html");
|
|
|
|
$data = $this->get(
|
|
$proxy,
|
|
"https://www.google.com/search",
|
|
$params
|
|
);
|
|
|
|
$html = &$data["data"];
|
|
$container = &$data["container"];
|
|
}
|
|
|
|
$out = [
|
|
"status" => "ok",
|
|
"npt" => null,
|
|
"news" => []
|
|
];
|
|
|
|
$this->fuckhtml->load($html);
|
|
|
|
// parse styles
|
|
$this->parsestyles();
|
|
|
|
// solve softcaptcha
|
|
$softcaptcha = $this->detect_sorry();
|
|
|
|
if($softcaptcha !== false){
|
|
|
|
$params["sei"] = $softcaptcha["sei"];
|
|
|
|
$data2 = $this->get(
|
|
$proxy,
|
|
"https://www.google.com/search",
|
|
$params,
|
|
$container
|
|
);
|
|
|
|
$html = &$data2["data"];
|
|
|
|
$this->fuckhtml->load($html);
|
|
$this->parsestyles();
|
|
}
|
|
|
|
// get images
|
|
$this->scrape_dimg($html);
|
|
|
|
$center_col =
|
|
$this->fuckhtml
|
|
->getElementById(
|
|
"center_col",
|
|
"div"
|
|
);
|
|
|
|
if($center_col === null){
|
|
|
|
throw new Exception("Could not grep result div");
|
|
}
|
|
|
|
$this->fuckhtml->load($center_col);
|
|
|
|
// get next page
|
|
$npt =
|
|
$this->fuckhtml
|
|
->getElementById(
|
|
"pnnext",
|
|
"a"
|
|
);
|
|
|
|
if($npt !== false){
|
|
|
|
$out["npt"] =
|
|
$this->backend->store(
|
|
json_encode([
|
|
"u" =>
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$npt["attributes"]
|
|
["href"]
|
|
),
|
|
"c" => $container
|
|
]),
|
|
"news",
|
|
$proxy
|
|
);
|
|
}
|
|
|
|
$as =
|
|
$this->fuckhtml
|
|
->getElementsByAttributeName(
|
|
"jsname",
|
|
"a"
|
|
);
|
|
|
|
foreach($as as $a){
|
|
|
|
$this->fuckhtml->load($a);
|
|
|
|
// get title
|
|
$title =
|
|
$this->fuckhtml
|
|
->getElementsByAttributeValue(
|
|
"role",
|
|
"heading",
|
|
"div"
|
|
);
|
|
|
|
if(count($title) === 0){
|
|
|
|
continue;
|
|
}
|
|
|
|
$title =
|
|
$this->titledots(
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$title[0]
|
|
)
|
|
);
|
|
|
|
// get thumbnail
|
|
$image =
|
|
$this->fuckhtml
|
|
->getElementsByAttributeName(
|
|
"id",
|
|
"img"
|
|
);
|
|
|
|
// check for padded title node, if found, we're inside a carousel
|
|
$probe =
|
|
$this->fuckhtml
|
|
->getElementsByClassName(
|
|
$this->getstyle(
|
|
[
|
|
"padding" => "16px 16px 40px 16px"
|
|
]
|
|
),
|
|
"div"
|
|
);
|
|
|
|
if(count($probe) !== 0){
|
|
|
|
$probe = true;
|
|
}else{
|
|
|
|
$probe = false;
|
|
}
|
|
|
|
if(
|
|
count($image) !== 0 &&
|
|
!isset($image[0]["attributes"]["width"])
|
|
){
|
|
|
|
$thumb = [
|
|
"url" =>
|
|
$this->getdimg(
|
|
$image[0]["attributes"]["id"]
|
|
),
|
|
"ratio" => $probe === true ? "16:9" : "1:1"
|
|
];
|
|
}else{
|
|
|
|
$thumb = [
|
|
"url" => null,
|
|
"ratio" => null
|
|
];
|
|
}
|
|
|
|
$description = null;
|
|
|
|
if($probe === false){
|
|
|
|
$desc_divs =
|
|
$this->fuckhtml
|
|
->getElementsByAttributeName(
|
|
"style",
|
|
"div"
|
|
);
|
|
|
|
foreach($desc_divs as $desc){
|
|
|
|
if(
|
|
strpos(
|
|
$desc["attributes"]["style"],
|
|
"margin-top:"
|
|
) !== false
|
|
){
|
|
|
|
$description =
|
|
$this->titledots(
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$desc
|
|
)
|
|
);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// get author
|
|
$author =
|
|
$this->fuckhtml
|
|
->getElementsByClassName(
|
|
$this->getstyle(
|
|
[
|
|
"overflow" => "hidden",
|
|
"text-align" => "left",
|
|
"text-overflow" => "ellipsis",
|
|
"white-space" => "nowrap",
|
|
"margin-bottom" => "8px"
|
|
]
|
|
),
|
|
"div"
|
|
);
|
|
|
|
if(count($author) !== 0){
|
|
|
|
$author =
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$author[0]
|
|
);
|
|
}else{
|
|
|
|
$author = null;
|
|
}
|
|
|
|
// get date
|
|
$date = null;
|
|
|
|
$date_div =
|
|
$this->fuckhtml
|
|
->getElementsByAttributeName(
|
|
"style",
|
|
"div"
|
|
);
|
|
|
|
foreach($date_div as $d){
|
|
|
|
$this->fuckhtml->load($d);
|
|
|
|
$span =
|
|
$this->fuckhtml
|
|
->getElementsByTagName(
|
|
"span"
|
|
);
|
|
|
|
if(
|
|
strpos(
|
|
$d["attributes"]["style"],
|
|
"bottom:"
|
|
) !== false
|
|
){
|
|
|
|
$date =
|
|
strtotime(
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$span[count($span) - 1]
|
|
)
|
|
);
|
|
break;
|
|
}
|
|
}
|
|
|
|
$out["news"][] = [
|
|
"title" => $title,
|
|
"author" => $author,
|
|
"description" => $description,
|
|
"date" => $date,
|
|
"thumb" => $thumb,
|
|
"url" =>
|
|
$this->unshiturl(
|
|
$a["attributes"]
|
|
["href"]
|
|
)
|
|
];
|
|
}
|
|
|
|
return $out;
|
|
}
|
|
|
|
|
|
|
|
private function scrape_dimg($html){
|
|
|
|
// get images loaded through javascript
|
|
$this->dimg = [];
|
|
|
|
preg_match_all(
|
|
'/function\(\){google\.ldi=({.*?});/',
|
|
$html,
|
|
$dimg
|
|
);
|
|
|
|
if(isset($dimg[1])){
|
|
|
|
foreach($dimg[1] as $i){
|
|
|
|
$tmp = json_decode($i, true);
|
|
foreach($tmp as $key => $value){
|
|
|
|
$this->dimg[$key] =
|
|
$this->unshit_thumb(
|
|
$value
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// get additional javascript base64 images
|
|
preg_match_all(
|
|
'/var s=\'(data:image\/[^\']+)\';var ii=\[((?:\'[^\']+\',?)+)\];/',
|
|
$html,
|
|
$dimg
|
|
);
|
|
|
|
if(isset($dimg[1])){
|
|
|
|
for($i=0; $i<count($dimg[1]); $i++){
|
|
|
|
$delims = explode(",", $dimg[2][$i]);
|
|
$string =
|
|
$this->fuckhtml
|
|
->parseJsString(
|
|
$dimg[1][$i]
|
|
);
|
|
|
|
foreach($delims as $delim){
|
|
|
|
$this->dimg[trim($delim, "'")] = $string;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
private function scrape_imagearr($html){
|
|
// get image links arrays
|
|
preg_match_all(
|
|
'/\[[0-9]+,"([^"]+)",\["([^"]+)\",([0-9]+),([0-9]+)\],\["([^"]+)",([0-9]+),([0-9]+)\]/',
|
|
$html,
|
|
$image_arr
|
|
);
|
|
|
|
$this->image_arr = [];
|
|
if(isset($image_arr[1])){
|
|
|
|
for($i=0; $i<count($image_arr[1]); $i++){
|
|
|
|
$original =
|
|
$this->fuckhtml
|
|
->parseJsString(
|
|
$image_arr[5][$i]
|
|
);
|
|
|
|
if(
|
|
preg_match(
|
|
'/^x-raw-image/',
|
|
$original
|
|
)
|
|
){
|
|
|
|
// only add thumbnail, google doesnt have OG resolution
|
|
$this->image_arr[$image_arr[1][$i]] = [
|
|
[
|
|
"url" =>
|
|
$this->unshit_thumb(
|
|
$this->fuckhtml
|
|
->parseJsString(
|
|
$image_arr[2][$i]
|
|
)
|
|
),
|
|
"width" => (int)$image_arr[7][$i], // pass the OG image width & height
|
|
"height" => (int)$image_arr[6][$i]
|
|
]
|
|
];
|
|
|
|
continue;
|
|
}
|
|
|
|
$this->image_arr[$image_arr[1][$i]] =
|
|
[
|
|
[
|
|
"url" => $original,
|
|
"width" => (int)$image_arr[7][$i],
|
|
"height" => (int)$image_arr[6][$i]
|
|
],
|
|
[
|
|
"url" =>
|
|
$this->unshit_thumb(
|
|
$this->fuckhtml
|
|
->parseJsString(
|
|
$image_arr[2][$i]
|
|
)
|
|
),
|
|
"width" => (int)$image_arr[4][$i],
|
|
"height" => (int)$image_arr[3][$i]
|
|
]
|
|
];
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
private function getdimg($dimg){
|
|
|
|
return isset($this->dimg[$dimg]) ? $this->dimg[$dimg] : null;
|
|
}
|
|
|
|
|
|
private function unshit_thumb($url, $get_bigger_res = false){
|
|
// https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQINE2vbnNLHXqoZr3RVsaEJFyOsj1_BiBnJch-e1nyz3oia7Aj5xVj
|
|
// https://i.ytimg.com/vi/PZVIyA5ER3Y/mqdefault.jpg?sqp=-oaymwEFCJQBEFM&rs=AMzJL3nXeaCpdIar-ltNwl82Y82cIJfphA
|
|
|
|
$parts = parse_url($url);
|
|
|
|
if(
|
|
isset($parts["host"]) &&
|
|
preg_match(
|
|
'/(?:encrypted-)?tbn.*\.gstatic\.com/',
|
|
$parts["host"]
|
|
)
|
|
){
|
|
|
|
parse_str($parts["query"], $params);
|
|
|
|
if(isset($params["q"])){
|
|
|
|
if($get_bigger_res){
|
|
|
|
// this method doesnt always work, but does work for wiki thumbnails
|
|
return
|
|
"https://" . $parts["host"] . "/images?q=tbn:" .
|
|
$this->base64url_encode(
|
|
substr(
|
|
$this->base64url_decode(
|
|
explode(
|
|
":",
|
|
$params["q"])[1]
|
|
),
|
|
0,
|
|
29
|
|
)
|
|
);
|
|
}else{
|
|
|
|
return "https://" . $parts["host"] . "/images?q=" . $params["q"];
|
|
}
|
|
}
|
|
}
|
|
|
|
return $url;
|
|
}
|
|
|
|
|
|
private function parsestyles(){
|
|
|
|
$styles = [];
|
|
|
|
$style_div =
|
|
$this->fuckhtml
|
|
->getElementsByTagName(
|
|
"style"
|
|
);
|
|
|
|
$raw_styles = "";
|
|
|
|
foreach($style_div as $style){
|
|
|
|
$raw_styles .= $style["innerHTML"];
|
|
}
|
|
|
|
// filter out media/keyframe queries
|
|
$raw_styles =
|
|
preg_replace(
|
|
'/@\s*(?!font-face)[^{]+\s*{[\S\s]+?}\s*}/',
|
|
"",
|
|
$raw_styles
|
|
);
|
|
|
|
// get styles
|
|
preg_match_all(
|
|
'/(.+?){([\S\s]*?)}/',
|
|
$raw_styles,
|
|
$matches
|
|
);
|
|
|
|
for($i=0; $i<count($matches[1]); $i++){
|
|
|
|
// get style values
|
|
preg_match_all(
|
|
'/([^:;]+):([^;]*?(?:\([^)]+\)[^;]*?)?)(?:;|$)/',
|
|
$matches[2][$i],
|
|
$values_regex
|
|
);
|
|
|
|
$values = [];
|
|
for($k=0; $k<count($values_regex[1]); $k++){
|
|
|
|
$values[trim($values_regex[1][$k])] =
|
|
strtolower(trim($values_regex[2][$k]));
|
|
}
|
|
|
|
$names = explode(",", $matches[1][$i]);
|
|
|
|
// h1,h2,h3 will each get their own array index
|
|
foreach($names as $name){
|
|
|
|
$name = trim($name, "}\t\n\r\0\x0B");
|
|
|
|
foreach($values as $key => $value){
|
|
|
|
$styles[$name][$key] = $value;
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach($styles as $key => $values){
|
|
|
|
$styles[$key]["_c"] = count($values);
|
|
}
|
|
|
|
$this->styles = $styles;
|
|
|
|
// get CSS colors
|
|
$this->css_colors = [];
|
|
|
|
if(isset($this->styles[":root"])){
|
|
|
|
foreach($this->styles[":root"] as $key => $value){
|
|
|
|
$this->css_colors[$value] = strtolower($key);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
private function getstyle($styles){
|
|
|
|
$styles["_c"] = count($styles);
|
|
|
|
foreach($this->styles as $style_key => $style_values){
|
|
|
|
if(count(array_intersect_assoc($style_values, $styles)) === $styles["_c"] + 1){
|
|
|
|
$style_key =
|
|
explode(" ", $style_key);
|
|
|
|
$style_key = $style_key[count($style_key) - 1];
|
|
|
|
return
|
|
ltrim(
|
|
str_replace(
|
|
[".", "#"],
|
|
" ",
|
|
$style_key
|
|
)
|
|
);
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
|
|
|
|
private function getcolorvar($color){
|
|
|
|
if(isset($this->css_colors[$color])){
|
|
|
|
return $this->css_colors[$color];
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
|
|
private function unshiturl($url, $return_size = false){
|
|
|
|
// decode
|
|
$url =
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$url
|
|
);
|
|
|
|
$url_parts = parse_url($url);
|
|
|
|
if(isset($url_parts["query"])){
|
|
|
|
parse_str($url_parts["query"], $query);
|
|
}else{
|
|
|
|
$query = [];
|
|
}
|
|
|
|
if(
|
|
!isset(
|
|
$url_parts["host"]
|
|
) ||
|
|
stripos($url_parts["host"], "google.") !== false
|
|
){
|
|
|
|
// no host, we have a tracking url
|
|
if(isset($query["imgurl"])){
|
|
|
|
$url = $query["imgurl"];
|
|
}
|
|
elseif(isset($query["q"])){
|
|
|
|
$url = $query["q"];
|
|
}
|
|
}
|
|
|
|
// rewrite URLs to remove extra tracking parameters
|
|
$domain = parse_url($url, PHP_URL_HOST);
|
|
|
|
if(
|
|
preg_match(
|
|
'/wikipedia.org$/',
|
|
$domain
|
|
)
|
|
){
|
|
|
|
// rewrite wikipedia mobile URLs to desktop
|
|
$url =
|
|
$this->replacedomain(
|
|
$url,
|
|
preg_replace(
|
|
'/([a-z0-9]+)(\.m\.)/',
|
|
'$1.',
|
|
$domain
|
|
)
|
|
);
|
|
}
|
|
|
|
elseif(
|
|
preg_match(
|
|
'/imdb\.com$|youtube\.[^.]+$/',
|
|
$domain
|
|
)
|
|
){
|
|
|
|
// rewrite imdb and youtube mobile URLs too
|
|
$url =
|
|
$this->replacedomain(
|
|
$url,
|
|
preg_replace(
|
|
'/^m\./',
|
|
"",
|
|
$domain
|
|
)
|
|
);
|
|
|
|
}
|
|
|
|
elseif(
|
|
preg_match(
|
|
'/play\.google\.[^.]+$/',
|
|
$domain
|
|
)
|
|
){
|
|
|
|
// remove referrers from play.google.com
|
|
$u_query = parse_url($url, PHP_URL_QUERY);
|
|
if($u_query !== null){
|
|
|
|
parse_str($u_query, $u_query);
|
|
if(isset($u_query["referrer"])){ unset($u_query["referrer"]); }
|
|
if(isset($u_query["hl"])){ unset($u_query["hl"]); }
|
|
if(isset($u_query["gl"])){ unset($u_query["gl"]); }
|
|
|
|
$query = http_build_query($query);
|
|
|
|
$url =
|
|
str_replace(
|
|
$u_query,
|
|
$u_query,
|
|
$url
|
|
);
|
|
}
|
|
}
|
|
|
|
elseif(
|
|
preg_match(
|
|
'/twitter\.com$/',
|
|
$domain
|
|
)
|
|
){
|
|
// remove more referrers from twitter.com
|
|
$u_query = parse_url($url, PHP_URL_QUERY);
|
|
if($u_query !== null){
|
|
|
|
parse_str($u_query, $u_query);
|
|
if(isset($u_query["ref_src"])){ unset($u_query["ref_src"]); }
|
|
|
|
$u_query = http_build_query($u_query);
|
|
|
|
$url =
|
|
str_replace(
|
|
$oldquery,
|
|
$u_query,
|
|
$url
|
|
);
|
|
}
|
|
}
|
|
|
|
elseif(
|
|
preg_match(
|
|
'/maps\.google\.[^.]+/',
|
|
$domain
|
|
)
|
|
){
|
|
|
|
if(stripos($url, "maps?") !== false){
|
|
|
|
$u_query = parse_url($url, PHP_URL_QUERY);
|
|
|
|
if($u_query !== null){
|
|
|
|
parse_str($u_query, $u_query);
|
|
|
|
if(isset($u_query["daddr"])){
|
|
|
|
$url =
|
|
"https://maps.google.com/maps?daddr=" .
|
|
urlencode($u_query["daddr"]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if($return_size){
|
|
|
|
return [
|
|
"url" => $url,
|
|
"ref" => isset($query["imgrefurl"]) ? $query["imgrefurl"] : null,
|
|
"thumb_width" => isset($query["tbnw"]) ? (int)$query["tbnw"] : null,
|
|
"thumb_height" => isset($query["tbnh"]) ? (int)$query["tbnh"] : null,
|
|
"image_width" => isset($query["w"]) ? (int)$query["w"] : null,
|
|
"image_height" => isset($query["h"]) ? (int)$query["h"] : null
|
|
];
|
|
}
|
|
|
|
return $url;
|
|
}
|
|
|
|
private function replacedomain($url, $domain){
|
|
|
|
return
|
|
preg_replace(
|
|
'/(https?:\/\/)([^\/]+)/',
|
|
'$1' . $domain,
|
|
$url
|
|
);
|
|
}
|
|
|
|
private function titledots($title){
|
|
|
|
return trim($title, " .\t\n\r\0\x0B");
|
|
}
|
|
|
|
private function hms2int($time){
|
|
|
|
$parts = explode(":", $time, 3);
|
|
$time = 0;
|
|
|
|
if(count($parts) === 3){
|
|
|
|
// hours
|
|
$time = $time + ((int)$parts[0] * 3600);
|
|
array_shift($parts);
|
|
}
|
|
|
|
if(count($parts) === 2){
|
|
|
|
// minutes
|
|
$time = $time + ((int)$parts[0] * 60);
|
|
array_shift($parts);
|
|
}
|
|
|
|
// seconds
|
|
$time = $time + (int)$parts[0];
|
|
|
|
return $time;
|
|
}
|
|
|
|
function base64url_decode($data){
|
|
|
|
$b64 = strtr($data, "-_", "+/");
|
|
$pad = strlen($b64) % 4;
|
|
if ($pad) $b64 .= str_repeat("=", 4 - $pad);
|
|
|
|
return base64_decode($b64);
|
|
}
|
|
|
|
function base64url_encode($data){
|
|
|
|
return rtrim(strtr(base64_encode($data), "+/", "-_"), "=");
|
|
}
|
|
|
|
private function detect_sorry(){
|
|
|
|
// detect recaptcha
|
|
$captcha_form =
|
|
$this->fuckhtml
|
|
->getElementById(
|
|
"captcha-form",
|
|
"form"
|
|
);
|
|
|
|
if($captcha_form !== false){
|
|
|
|
throw new Exception("Google returned a captcha");
|
|
}
|
|
|
|
// detect JS challenge
|
|
$title =
|
|
$this->fuckhtml
|
|
->getElementsByTagName(
|
|
"title"
|
|
);
|
|
|
|
if(
|
|
count($title) !== 0 &&
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$title[0]
|
|
) == "Google Search"
|
|
){
|
|
|
|
throw new Exception("Google returned a JS challenge");
|
|
}
|
|
|
|
// bypass soft captcha
|
|
// "Verifying your request
|
|
// You'll be able to continue in a few seconds. Don't refresh this page."
|
|
$scripts =
|
|
$this->fuckhtml
|
|
->getElementsByTagName(
|
|
"script"
|
|
);
|
|
|
|
foreach($scripts as $sc){
|
|
|
|
if(
|
|
preg_match(
|
|
'/var eid ?= ?\'(.*)\'[\S\s]*, ?([0-9]+)\);/U',
|
|
$sc["innerHTML"],
|
|
$matches
|
|
)
|
|
){
|
|
|
|
sleep(10); // gotta sleep to not trigger captcha. blame google, not me
|
|
|
|
return [
|
|
"sei" => $matches[1],
|
|
"timeout" => (int)$matches[2] // set to 10000 usually
|
|
];
|
|
}
|
|
}
|
|
|
|
return false; // no need to bypass
|
|
}
|
|
}
|