diff --git a/docs/apache2.md b/docs/apache2.md index 62030c0..da7c835 100644 --- a/docs/apache2.md +++ b/docs/apache2.md @@ -102,7 +102,7 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r # deny access to private resources - + Require all denied diff --git a/resolve/google.php b/resolve/google.php new file mode 100644 index 0000000..f497dbf --- /dev/null +++ b/resolve/google.php @@ -0,0 +1,107 @@ +backend = new backend("google"); + + include "lib/fuckhtml.php"; + $this->fuckhtml = new fuckhtml(); + } + + private function get(/*$proxy, */$url, $get = []){ + + $curlproc = curl_init(); + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + curl_setopt($curlproc, CURLOPT_URL, $url); + + // http2 bypass + curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); + + $headers = + ["User-Agent: " . config::USER_AGENT, + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br, zstd", + "DNT: 1", + "Sec-GPC: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1", + "Priority: u=0, i"]; + + // i dont think proxies are necessary here... + //$this->backend->assign_proxy($curlproc, $proxy); + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + function resolve($id){ + + $html = + $this->get( + //$this->backend->get_ip(), + "https://www.google.com/goto", + [ + "url" => $id + ] + ); + + $this->fuckhtml->load($html); + + $a = + $this->fuckhtml + ->getElementsByTagName( + "a" + ); + + if(count($a) === 0){ + + throw new Exception("Failed to find link"); + } + + $link = + $this->fuckhtml + ->getTextContent( + $a[0]["attributes"]["href"] + ); + + if( + preg_match( + '/^https?/', + $link + ) + ){ + + return $link; + } + + throw new Exception("Got malformed link: " . $link); + } +} diff --git a/resolver.php b/resolver.php index 7b8b5ff..ea6ccf4 100644 --- a/resolver.php +++ b/resolver.php @@ -5,7 +5,8 @@ $resolver = new resolver(); class resolver{ public const resolvers = [ - "sc" + "sc", + "google" ]; public function __construct(){ @@ -40,9 +41,13 @@ class resolver{ $resolver = new $scraper(); $link = $resolver->resolve($target); - if(is_string($link)){ + if( + is_string($link) && + !empty($link) + ){ header("Location: {$link}"); + echo 'If your ass is not getting redirected, click here.'; } }catch(Exception $error){ @@ -54,13 +59,13 @@ class resolver{ header("Content-Type: text/plain"); http_response_code(400); - echo $message; + echo htmlspecialchars($message); } public function do404($message){ header("Content-Type: text/plain"); http_response_code(404); - echo $message; + echo htmlspecialchars($message); } } diff --git a/scraper/google.php b/scraper/google.php index b96ada8..fa57cd6 100644 --- a/scraper/google.php +++ b/scraper/google.php @@ -733,7 +733,8 @@ class google{ $container = &$data["container"]; //file_put_contents("scraper/google-jp.html", $html); - //$html = file_get_contents("scraper/google-softcaptcha.html"); + //$html = file_get_contents("scraper/sublink.html"); + //$container = ""; } $out = [ @@ -771,9 +772,10 @@ class google{ $html = &$data2["data"]; $this->fuckhtml->load($html); - $this->parsestyles(); } + $this->parsestyles(); + $this->scrape_dimg($html); $this->scrape_imagearr($html); @@ -914,14 +916,34 @@ class google{ $a["attributes"]["href"] ); + $link_text = + strtolower( + $this->fuckhtml + ->getTextContent($a) + ); + if( - preg_match( - '/^https?:\/\/lumendatabase\.org/', - $link + str_contains( + $link_text, + "complaint" ) ){ - $notices[] = $link; + // sometimes the link is an encrypted redirect, handle redirection securely + if( + preg_match( + '/^\/goto\?url=(.*)/', + $link, + $payload + ) + ){ + + $notices[] = "/resolver?scraper=google&target=" . $payload[1]; + }else{ + + // normal link + $notices[] = $link; + } } } @@ -1248,6 +1270,47 @@ class google{ $sublinks = []; + // get sublinks (rare) + $as = + $this->fuckhtml + ->getElementsByClassName( + $this->getstyle([ + "padding-right" => "6px", + "margin-top" => "4px", + "display" => "inline-block" + ]), + "a" + ); + + foreach($as as $sublink){ + + $sublink_url = + $this->fuckhtml + ->getTextContent( + $sublink["attributes"]["href"] + ); + + if( + preg_match( + '/^\/goto\?url=(.*)/', + $sublink_url, + $piece_of_shit + ) + ){ + + // encrypted piece of shit url + $sublink_url = "/resolver?scraper=google&target=" . $piece_of_shit[1]; + } + + $sublinks[] = [ + "title" => + $this->fuckhtml + ->getTextContent($sublink), + "date" => null, + "url" => $sublink_url + ]; + } + $out["web"][] = [ "title" => $title, "description" => $description,