Synced

new google message (b1f5974e40)

added google api image scraper
(e63a17d6db)

lib/frontend.php

@@ -2,6 +2,52 @@
 
 class frontend{
 	
+	public function validateurl($url, $net_validate = false){
+		
+		$url_parts = parse_url($url);
+		
+		// check if required parts are there
+		if(
+			!isset($url_parts["scheme"]) ||
+			!(
+				$url_parts["scheme"] == "http" ||
+				$url_parts["scheme"] == "https"
+			) ||
+			!isset($url_parts["host"])
+		){
+			return false;
+		}
+		
+		if($net_validate){
+			$ip = 
+				str_replace(
+					["[", "]"], // handle ipv6
+					"",
+					$url_parts["host"]
+				);
+			
+			// if its not an IP
+			if(!filter_var($ip, FILTER_VALIDATE_IP)){
+				
+				// resolve domain's IP
+				$ip = gethostbyname($url_parts["host"] . ".");
+			}
+			
+			// check if its localhost
+			if(
+				filter_var(
+					$ip,
+					FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE
+				) === false
+			){
+				
+				return false;
+			}
+		}
+		
+		return true;
+	}
+	
 	public function load($template, $replacements = []){
 		
 		$replacements["server_name"] = htmlspecialchars(config::SERVER_NAME);
@@ -622,6 +668,7 @@ class frontend{
 						"yandex" => "Yandex",
 						"brave" => "Brave",
 						"google" => "Google",
+						"google_api" => "Google API",
 						"google_cse" => "Google CSE",
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
@@ -638,8 +685,7 @@ class frontend{
 						"fivehpx" => "500px",
 						"vsco" => "VSCO",
 						"imgur" => "Imgur",
-						"ftm" => "FindThatMeme",
+						"ftm" => "FindThatMeme"
-						//"sankakucomplex" => "SankakuComplex"
 					]
 				];
 				break;
@@ -695,6 +741,22 @@ class frontend{
 					]
 				];
 				break;
+			
+			case "booru":
+				$filters["scraper"] = [
+					"display" => "Scraper",
+					"option" => [
+						"safebooru" => "Safebooru",
+						"konachan" => "Konachan",
+						"tbib" => "The Big Imageboard",
+						"gelbooru" => "Gelbooru",
+						"yandere" => "Yande.re",
+						"tbib" => "The Big Imageboard",
+						"sankakucomplex" => "SankakuComplex",
+						"soybooru" => "SoyBooru"
+					]
+				];
+				break;
 		}
 		
 		// get scraper name from user input, or default out to preferred scraper
@@ -871,6 +933,7 @@ class frontend{
 		
 		$html = null;
 		
+		//foreach(["web", "images", "videos", "news", "music", "booru"] as $type){
 		foreach(["web", "images", "videos", "news", "music"] as $type){
 			
 			$html .= '<a href="/' . $type . '?s=' . urlencode($query);

scraper/google.php

@@ -12,6 +12,8 @@ class google{
 		
 		include "lib/backend.php";
 		$this->backend = new backend("google");
+		
+		$this->message = "Still working on a Google scraper that uses a headful browser. It will require Firefox + a webExtension running on a dedicated server. Waiting for my EDID adapter and we can get the show going. In the meantime, use the Google CSE/API or Yahoo JP/Startpage scrapers. They're all crippled in their own special ways but they're serviceable I guess.";
 	}
 	
 	public function getfilters($page){
@@ -505,7 +507,7 @@ class google{
 		}
 	}
 	
-	private function get($proxy, $url, $get = [], $alt_ua = false){
+	private function get($proxy, $url, $get = []){
 		
 		$curlproc = curl_init();
 		
@@ -518,35 +520,22 @@ class google{
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
 		
-		if($alt_ua === true){
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-			
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
+			"User-Agent: " . config::USER_AGENT,
-				"User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 26_0_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/406.0.862495628 Mobile/15E148 Safari/604.1",
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-				"Accept: text/html, application/xml;q=0.9, */*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
-				"Accept-Language: en-US,en;q=0.8",
+			"Accept-Encoding: gzip",
-				"Accept-Encoding: gzip, deflate",
+			"DNT: 1",
-				"Connection: Keep-Alive",
+			"Connection: keep-alive",
-				"Cache-Control: no-cache"
+			"Upgrade-Insecure-Requests: 1",
-			]);
+			"Sec-Fetch-Dest: document",
-		}else{
+			"Sec-Fetch-Mode: navigate",
-			
+			"Sec-Fetch-Site: none",
-			curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+			"Sec-Fetch-User: ?1",
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
+			"Priority: u=1",
-				"User-Agent: " . config::USER_AGENT,
+			"TE: trailers"
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+		]);
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
-				"DNT: 1",
-				"Connection: keep-alive",
-				"Upgrade-Insecure-Requests: 1",
-				"Sec-Fetch-Dest: document",
-				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: none",
-				"Sec-Fetch-User: ?1",
-				"Priority: u=1",
-				"TE: trailers"
-			]);
-		}
 		
 		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
 		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
@@ -574,228 +563,22 @@ class google{
 	
 	public function web($get){
 		
-		throw new Exception("There are no known ways to scrape Google's /search endpoint without JS at this time. I'm working on a method that extracts cookies from browsers. Use Google API/CSE/Yahoo JP/Startpage for google results for now.");
+		throw new Exception($this->message);
-	}
-	
-	
-	public function video($get){
-		throw new Exception("There are no known ways to scrape Google's /search endpoint without JS at this time. I'm working on a method that extracts cookies from browsers. Use Google API/CSE/Yahoo JP/Startpage for google results for now.");
-	}
-	
-	
-	public function news($get){
-		throw new Exception("There are no known ways to scrape Google's /search endpoint without JS at this time. I'm working on a method that extracts cookies from browsers. Use Google API/CSE/Yahoo JP/Startpage for google results for now.");
 	}
 	
 	
 	public function image($get){
-		
+		throw new Exception($this->message);
-		// generate parameters
+	}
-		if($get["npt"]){
+	
-			
+	
-			[$params, $proxy] =
+	public function video($get){
-				$this->backend->get(
+		throw new Exception($this->message);
-					$get["npt"],
+	}
-					"images"
+	
-				);
+	
-			
+	public function news($get){
-			$params = json_decode($params, true);
+		throw new Exception($this->message);
-			
-			$page = $params["page"] + 1;
-			$params = $params["params"];
-			$params["async"] = "_fmt:json,p:1,ijn:{$page}";
-			
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-			
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			$country = $get["country"];
-			$nsfw = $get["nsfw"];
-			$time = $get["time"];
-			$size = $get["size"];
-			$ratio = $get["ratio"];
-			$color = $get["color"];
-			$type = $get["type"];
-			$format = $get["format"];
-			$rights = $get["rights"];
-			
-			$page = 0;
-			
-			$params = [
-				"q" => $search,
-				"tbm" => "isch",
-				"asearch" => "isch",
-				"async" => "_fmt:json,p:0,ijn:{$page}", // ijn:0 = page 1
-			];
-			
-			// country (image search uses cr instead of gl)
-			if($country != "any"){
-				
-				$params["cr"] = "country" . strtoupper($country);
-			}
-			
-			// nsfw
-			$params["safe"] = $nsfw == "yes" ? "off" : "active";
-			
-			// generate tbs
-			$tbs = [];
-			
-			// time
-			if($time != "any"){
-				
-				$tbs["qdr"] = $time;
-			}
-			
-			// size
-			if($size != "any"){
-				
-				$params["imgsz"] = $size;
-			}
-			
-			// ratio
-			if($ratio != "any"){
-				
-				$params["imgar"] = $ratio;
-			}
-			
-			// color
-			if($color != "any"){
-				
-				if(
-					$color == "color" ||
-					$color == "trans"
-				){
-					
-					$params["imgc"] = $color;
-				}elseif($color == "bnw"){
-					
-					$params["imgc"] = "gray";
-				}else{
-					
-					$tbs["ic"] = "specific";
-					$tbs["isc"] = $color;
-				}
-			}
-			
-			// type
-			if($type != "any"){
-				
-				$tbs["itp"] = $type;
-			}
-			
-			// format
-			if($format != "any"){
-				
-				$params["as_filetype"] = $format;
-			}
-			
-			// rights (tbs)
-			if($rights != "any"){
-				
-				$tbs["sur"] = $rights;
-			}
-			
-			// append tbs
-			if(count($tbs) !== 0){
-				
-				$params["tbs"] = "";
-				
-				foreach($tbs as $key => $value){
-					
-					$params["tbs"] .= $key . ":" . $value . ",";
-				}
-				
-				$params["tbs"] = rtrim($params["tbs"], ",");
-			}
-		}
-		
-		try{
-			$json = 
-				$this->get(
-					$proxy,
-					"https://www.google.com/search",
-					$params
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to get search page");
-		}
-		
-		unset($params["async"]);
-		
-		//$json = file_get_contents("scraper/google.json");
-		
-		// detect captcha
-		$this->fuckhtml->load($json);
-		$this->detect_sorry();
-		
-		// remove xssi
-		$json =
-			preg_replace(
-				'/^[^{]*/',
-				"",
-				$json
-			);
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-		
-		if(!isset($json["ischj"]["metadata"])){
-			
-			throw new Exception("Google did not return an image array");
-		}
-		
-		foreach($json["ischj"]["metadata"] as $image){
-			
-			$out["image"][] = [
-				"title" => $this->titledots($image["result"]["page_title"]),
-				"source" => [
-					[
-						"url" => $image["original_image"]["url"],
-						"width" => (int)$image["original_image"]["width"],
-						"height" => (int)$image["original_image"]["height"]
-					],
-					[
-						"url" => $image["thumbnail"]["url"],
-						"width" => (int)$image["thumbnail"]["width"],
-						"height" => (int)$image["thumbnail"]["height"]
-					]
-				],
-				"url" => $image["result"]["referrer_url"]
-			];
-		}
-		
-		$page++;
-		
-		if(count($out["image"]) === 10){
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode([
-						"params" => $params,
-						"page" => $page
-					]),
-					"images",
-					$proxy
-				);
-		}
-		
-		return $out;
 	}
 	
 	

scraper/google_api.php

@@ -264,6 +264,25 @@ class google_api{
 					"yes" => "Yes", // safe=active
 					"no" => "No" // safe=off
 				]
+			],
+			"sort" => [ // sort
+				"display" => "Sort by",
+				"option" => [
+					"any" => "Any order",
+					"date:d" => "Oldest",
+					"date:a" => "Newest"
+				]
+			],
+			"newer" => [
+				"display" => "Newer than",
+				"option" => "_DATE"
+			],			
+			"rm_dupes" => [ // filter
+				"display" => "Remove duplicates",
+				"option" => [
+					"yes" => "Yes", // 1
+					"no" => "No" // 0
+				]
 			]
 		];
 		
@@ -313,109 +332,29 @@ class google_api{
 								"zh-CN" => "Chinese (Simplified)",
 								"zh-TW" => "Chinese (Traditional)"
 							]
-						],
-						"sort" => [
-							"display" => "Sort by",
-							"option" => [
-								"any" => "Any order",
-								"date:d" => "Oldest",
-								"date:a" => "Newest"
-							]
-						],
-						"newer" => [
-							"display" => "Newer than",
-							"option" => "_DATE"
-						],
-						"rm_dupes" => [
-							"display" => "Remove duplicates",
-							"option" => [
-								"yes" => "Yes",
-								"no" => "No"
-							]
 						]
 					]
 				);
 				break;
-			/*
+			
 			case "images":
 				return array_merge(
 					$base,
 					[
-						"time" => [ // tbs=qdr:<time>
+						"size" => [ // imgSize
-							"display" => "Time posted",
-							"option" => [
-								"any" => "Any time",
-								"d" => "Past 24 hours",
-								"w" => "Past week",
-								"m" => "Past month",
-								"y" => "Past year"
-							]
-						],
-						"size" => [ // imgsz
 							"display" => "Size",
 							"option" => [
 								"any" => "Any size",
-								"l" => "Large",
+								"icon" => "Icon",
-								"m" => "Medium",
+								"small" => "Small",
-								"i" => "Icon",
+								"medium" => "Medium",
-								"qsvga" => "Larger than 400x300",
+								"large" => "Large",
-								"vga" => "Larger than 640x480",
+								"xlarge" => "X-Large",
-								"svga" => "Larger than 800x600",
+								"xxlarge" => "XX-Large",
-								"xga" => "Larger than 1024x768",
+								"huge" => "Huge"
-								"2mp" => "Larger than 2MP",
-								"4mp" => "Larger than 4MP",
-								"6mp" => "Larger than 6MP",
-								"8mp" => "Larger than 8MP",
-								"10mp" => "Larger than 10MP",
-								"12mp" => "Larger than 12MP",
-								"15mp" => "Larger than 15MP",
-								"20mp" => "Larger than 20MP",
-								"40mp" => "Larger than 40MP",
-								"70mp" => "Larger than 70MP"
 							]
 						],
-						"ratio" => [ // imgar
+						"format" => [ // fileType
-							"display" => "Aspect ratio",
-							"option" => [
-								"any" => "Any ratio",
-								"t|xt" => "Tall",
-								"s" => "Square",
-								"w" => "Wide",
-								"xw" => "Panoramic"
-							]
-						],
-						"color" => [ // imgc
-							"display" => "Color",
-							"option" => [
-								"any" => "Any color",
-								"color" => "Full color",
-								"bnw" => "Black & white",
-								"trans" => "Transparent",
-								// from here, imgcolor
-								"red" => "Red",
-								"orange" => "Orange",
-								"yellow" => "Yellow",
-								"green" => "Green",
-								"teal" => "Teal",
-								"blue" => "Blue",
-								"purple" => "Purple",
-								"pink" => "Pink",
-								"white" => "White",
-								"gray" => "Gray",
-								"black" => "Black",
-								"brown" => "Brown"
-							]
-						],
-						"type" => [ // tbs=itp:<type>
-							"display" => "Type",
-							"option" => [
-								"any" => "Any type",
-								"clipart" => "Clip Art",
-								"lineart" => "Line Drawing",
-								"animated" => "Animated"
-							]
-						],
-						"format" => [ // as_filetype
 							"display" => "Format",
 							"option" => [
 								"any" => "Any format",
@@ -429,17 +368,55 @@ class google_api{
 								"craw" => "RAW"
 							]
 						],
-						"rights" => [ // tbs=sur:<rights>
+						"color" => [
+							"display" => "Color",
+							"option" => [
+								"any" => "Any color",
+								
+								"color" => "Full color", // imgColorType
+								"mono" => "Black & White",
+								"trans" => "Transparent background",
+								
+								"red" => "Red", // imgDominantColor
+								"orange" => "Orange",
+								"yellow" => "Yellow",
+								"green" => "Green",
+								"teal" => "Teal",
+								"blue" => "Blue",
+								"purple" => "Purple",
+								"pink" => "Pink",
+								"white" => "White",
+								"gray" => "Gray",
+								"black" => "Black",
+								"brown" => "Brown"
+							]
+						],
+						"type" => [ // imgType
+							"display" => "Type",
+							"option" => [
+								"any" => "Any type",
+								"clipart" => "Clip Art",
+								"face" => "Faces",
+								"lineart" => "Line Drawing",
+								"stock" => "Stock photos",
+								"photo" => "Photos",
+								"animated" => "Animated",
+							]
+						],
+						"rights" => [ // rights
 							"display" => "Usage rights",
 							"option" => [
 								"any" => "Any license",
-								"cl" => "Creative Commons licenses",
+								"cc_publicdomain" => "Public domain",
-								"ol" => "Commercial & other licenses"
+								"cc_attribute" => "Attribution required",
+								"cc_sharealike" => "Sharealike",
+								"cc_noncommercial" => "Non-commercial use only",
+								"cc_nonderived" => "Original works"
 							]
 						]
 					]
 				);
-				break;*/
+				break;
 		}
 	}
 	
@@ -485,6 +462,7 @@ class google_api{
 		return $data;
 	}
 	
+	
 	public function web($get){
 		
 		// rotate proxy + key on EVERY request
@@ -731,8 +709,162 @@ class google_api{
 		return $out;
 	}
 	
+	
+	public function image($get){
+		
+		// rotate proxy + key on EVERY request
+		$keydata = $this->backend->get_key();
+		$proxy = $this->backend->get_ip($keydata["increment"]);
+		
+		if($get["npt"]){
+			
+			// $p is never used
+			[$params, $p] = $this->backend->get(
+				$get["npt"],
+				"web"
+			);
+			
+			$params = json_decode($params, true);
+			
+			$params["key"] = $keydata["key"];
+			
+		}else{
+			
+			//$json = file_get_contents("scraper/google.json");
+			$params = [
+				"q" => $get["s"],
+				"cx" => config::GOOGLE_CX_ENDPOINT,
+				"num" => 10,
+				"start" => 1,
+				"searchType" => "image",
+				"key" => $keydata["key"]
+			];
+			
+			//
+			// parse filters
+			//
+			if($get["newer"] !== false){
+				
+				$params["dateRestrict"] = "d" . (round((time() - $get["newer"]) / 100000));
+			}
+			
+			if($get["rm_dupes"] == "no"){ $params["filter"] = "0"; }
+			if($get["country"] != "any"){ $params["gl"] = $get["country"]; }
+			
+			if($get["nsfw"] == "yes"){
+				
+				$params["safe"] = "off";
+			}else{
+				
+				$params["safe"] = "active";
+			}
+			
+			if($get["sort"] != "any"){ $params["sort"] = $get["sort"]; }
+			
+			// image filters
+			if($get["size"] != "any"){ $params["imgSize"] = $get["size"]; }
+			if($get["format"] != "any"){ $params["fileType"] = $get["format"]; }
+			
+			switch($get["color"]){
+				
+				case "any":
+					break;
+				
+				case "color":
+				case "mono":
+				case "trans":
+					$params["imgColorType"] = $get["color"];
+					break;
+				
+				default:
+					$params["imgDominantColor"] = $get["color"];
+					break;
+			}
+			
+			if($get["type"] != "any"){ $params["imgType"] = $get["type"]; }
+			if($get["rights"] != "any"){ $params["rights"] = $get["rights"]; }
+		}
+		
+		try{
+			$json =
+				$this->get(
+					$proxy,
+					"https://www.googleapis.com/customsearch/v1",
+					$params
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		if(isset($json["error"]["message"])){
+			
+			throw new Exception(
+				"API returned an error: " .
+				$json["error"]["message"] .
+				" (key #" . $keydata["increment"] . ")"
+			);
+		}
+		
+		if(!isset($json["items"])){
+			
+			// google just doesnt return items when theres no results
+			return $out;
+		}
+		
+		foreach($json["items"] as $image){
+			
+			$out["image"][] = [
+				"title" => $this->titledots($image["title"]),
+				"source" => [
+					[
+						"url" => $image["link"],
+						"width" => (int)$image["image"]["width"],
+						"height" => (int)$image["image"]["height"]
+					],
+					[
+						"url" => $image["image"]["thumbnailLink"],
+						"width" => (int)$image["image"]["thumbnailWidth"],
+						"height" => (int)$image["image"]["thumbnailHeight"]
+					]
+				],
+				"url" => $image["image"]["contextLink"]
+			];
+		}
+		
+		// get npt
+		if(isset($json["queries"]["nextPage"][0]["startIndex"])){
+			
+			unset($params["key"]);
+			$params["start"] = (int)$json["queries"]["nextPage"][0]["startIndex"];
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($params),
+					"web",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	
 	private function titledots($title){
 		
 		return trim($title, " .\t\n\r\0\x0B…");
 	}
-}
+}

settings.php

@@ -223,6 +223,10 @@ $settings = [
 						"value" => "google",
 						"text" => "Google"
 					],
+					[
+						"value" => "google_api",
+						"text" => "Google API"
+					],
 					[
 						"value" => "google_cse",
 						"text" => "Google CSE"