Synced

fix dmca links not showing up on google (d598a560ae)
dumbass moment (28ba8422b2)
deny access to resolve (06ee6878ca)
scrape google sublinks, which are encrypted pieces of shit (90232152e9)

scraper/google.php

@@ -733,7 +733,8 @@ class google{
 			$container = &$data["container"];
 			
 			//file_put_contents("scraper/google-jp.html", $html);
-			//$html = file_get_contents("scraper/google-softcaptcha.html");
+			//$html = file_get_contents("scraper/sublink.html");
+			//$container = "";
 		}
 		
 		$out = [
@@ -771,9 +772,10 @@ class google{
 			$html = &$data2["data"];
 						
 			$this->fuckhtml->load($html);
-			$this->parsestyles();
 		}
 		
+		$this->parsestyles();
+		
 		$this->scrape_dimg($html);
 		$this->scrape_imagearr($html);
 		
@@ -914,14 +916,34 @@ class google{
 						$a["attributes"]["href"]
 					);
 				
+				$link_text =
+					strtolower(
+						$this->fuckhtml
+						->getTextContent($a)
+					);
+				
 				if(
-					preg_match(
-						'/^https?:\/\/lumendatabase\.org/',
-						$link
+					str_contains(
+						$link_text,
+						"complaint"
 					)
 				){
 					
-					$notices[] = $link;
+					// sometimes the link is an encrypted redirect, handle redirection securely
+					if(
+						preg_match(
+							'/^\/goto\?url=(.*)/',
+							$link,
+							$payload
+						)
+					){
+						
+						$notices[] = "/resolver?scraper=google&target=" . $payload[1];
+					}else{
+						
+						// normal link
+						$notices[] = $link;
+					}
 				}
 			}
 			
@@ -1248,6 +1270,47 @@ class google{
 			
 			$sublinks = [];
 			
+			// get sublinks (rare)
+			$as =
+				$this->fuckhtml
+				->getElementsByClassName(
+					$this->getstyle([
+						"padding-right" => "6px",
+						"margin-top" => "4px",
+						"display" => "inline-block"
+					]),
+					"a"
+				);
+			
+			foreach($as as $sublink){
+				
+				$sublink_url =
+					$this->fuckhtml
+					->getTextContent(
+						$sublink["attributes"]["href"]
+					);
+				
+				if(
+					preg_match(
+						'/^\/goto\?url=(.*)/',
+						$sublink_url,
+						$piece_of_shit
+					)
+				){
+					
+					// encrypted piece of shit url
+					$sublink_url = "/resolver?scraper=google&target=" . $piece_of_shit[1];
+				}
+				
+				$sublinks[] = [
+					"title" =>
+						$this->fuckhtml
+						->getTextContent($sublink),
+					"date" => null,
+					"url" => $sublink_url
+				];
+			}
+			
 			$out["web"][] = [
 				"title" => $title,
 				"description" => $description,