Synced

fix dmca links not showing up on google (d598a560ae)
dumbass moment (28ba8422b2)
deny access to resolve (06ee6878ca)
scrape google sublinks, which are encrypted pieces of shit (90232152e9)

docs/apache2.md

@@ -102,7 +102,7 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r
 	</Directory>
 
 	# deny access to private resources
-	<DirectoryMatch "^/var/www/fossq/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+	<DirectoryMatch "^/var/www/fossq/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
 		Require all denied
 	</DirectoryMatch>
 	

resolve/google.php

@@ -0,0 +1,107 @@
+<?php
+
+class google{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("google");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	private function get(/*$proxy, */$url, $get = []){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		// http2 bypass
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		
+		$headers =
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1",
+			"Priority: u=0, i"];
+		
+		// i dont think proxies are necessary here...
+		//$this->backend->assign_proxy($curlproc, $proxy);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	function resolve($id){
+		
+		$html =
+			$this->get(
+				//$this->backend->get_ip(),
+				"https://www.google.com/goto",
+				[
+					"url" => $id
+				]
+			);
+		
+		$this->fuckhtml->load($html);
+		
+		$a =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"a"
+			);
+		
+		if(count($a) === 0){
+			
+			throw new Exception("Failed to find link");
+		}
+		
+		$link =
+			$this->fuckhtml
+			->getTextContent(
+				$a[0]["attributes"]["href"]
+			);
+		
+		if(
+			preg_match(
+				'/^https?/',
+				$link
+			)
+		){
+			
+			return $link;
+		}
+		
+		throw new Exception("Got malformed link: " . $link);
+	}
+}

resolver.php

@@ -5,7 +5,8 @@ $resolver = new resolver();
 class resolver{
 	
 	public const resolvers = [
-		"sc"
+		"sc",
+		"google"
 	];
 	
 	public function __construct(){
@@ -40,9 +41,13 @@ class resolver{
 			$resolver = new $scraper();
 			$link = $resolver->resolve($target);
 			
-			if(is_string($link)){
+			if(
+				is_string($link) &&
+				!empty($link)
+			){
 				
 				header("Location: {$link}");
+				echo 'If your ass is not getting redirected, <a href="' . htmlspecialchars($link) . '">click here</a>.';
 			}
 		}catch(Exception $error){
 			
@@ -54,13 +59,13 @@ class resolver{
 		
 		header("Content-Type: text/plain");
 		http_response_code(400);
-		echo $message;
+		echo htmlspecialchars($message);
 	}
 	
 	public function do404($message){
 		
 		header("Content-Type: text/plain");
 		http_response_code(404);
-		echo $message;
+		echo htmlspecialchars($message);
 	}
 }

scraper/google.php

@@ -733,7 +733,8 @@ class google{
 			$container = &$data["container"];
 			
 			//file_put_contents("scraper/google-jp.html", $html);
-			//$html = file_get_contents("scraper/google-softcaptcha.html");
+			//$html = file_get_contents("scraper/sublink.html");
+			//$container = "";
 		}
 		
 		$out = [
@@ -771,9 +772,10 @@ class google{
 			$html = &$data2["data"];
 						
 			$this->fuckhtml->load($html);
-			$this->parsestyles();
 		}
 		
+		$this->parsestyles();
+		
 		$this->scrape_dimg($html);
 		$this->scrape_imagearr($html);
 		
@@ -914,14 +916,34 @@ class google{
 						$a["attributes"]["href"]
 					);
 				
+				$link_text =
+					strtolower(
+						$this->fuckhtml
+						->getTextContent($a)
+					);
+				
 				if(
-					preg_match(
-						'/^https?:\/\/lumendatabase\.org/',
-						$link
+					str_contains(
+						$link_text,
+						"complaint"
 					)
 				){
 					
-					$notices[] = $link;
+					// sometimes the link is an encrypted redirect, handle redirection securely
+					if(
+						preg_match(
+							'/^\/goto\?url=(.*)/',
+							$link,
+							$payload
+						)
+					){
+						
+						$notices[] = "/resolver?scraper=google&target=" . $payload[1];
+					}else{
+						
+						// normal link
+						$notices[] = $link;
+					}
 				}
 			}
 			
@@ -1248,6 +1270,47 @@ class google{
 			
 			$sublinks = [];
 			
+			// get sublinks (rare)
+			$as =
+				$this->fuckhtml
+				->getElementsByClassName(
+					$this->getstyle([
+						"padding-right" => "6px",
+						"margin-top" => "4px",
+						"display" => "inline-block"
+					]),
+					"a"
+				);
+			
+			foreach($as as $sublink){
+				
+				$sublink_url =
+					$this->fuckhtml
+					->getTextContent(
+						$sublink["attributes"]["href"]
+					);
+				
+				if(
+					preg_match(
+						'/^\/goto\?url=(.*)/',
+						$sublink_url,
+						$piece_of_shit
+					)
+				){
+					
+					// encrypted piece of shit url
+					$sublink_url = "/resolver?scraper=google&target=" . $piece_of_shit[1];
+				}
+				
+				$sublinks[] = [
+					"title" =>
+						$this->fuckhtml
+						->getTextContent($sublink),
+					"date" => null,
+					"url" => $sublink_url
+				];
+			}
+			
 			$out["web"][] = [
 				"title" => $title,
 				"description" => $description,