Synced

0dabcea0aa
88012f6ae2
0861450b8a
c713d52b5f
d2b0a414ad
6086c63148

README.md

@@ -6,11 +6,11 @@ https://fossq.com/about
 
 ## Totally unbiased comparison between alternatives
 
-|                            | FOSS-Q                    | searx(ng) | FbIN-LibreY | araa      | hearch.co         |
-|----------------------------|-------------------------|-----------|-------------|-----------|-------------------|
-| RAM usage                  | 200-400mb~              | 2GB~      | 200-400mb~  | 2GB~      | idk               |
-| Does it work               | no (debunked by snopes) | yes       | yes         | a little  | better than searx |
-| Does it work               | ye                      | sometimes | sometimes   | sometimes | yes               |
+|                            | FOSSQ                   | SearX(ng) | Whoogle    | Degoog                               |
+|----------------------------|-------------------------|-----------|------------|--------------------------------------|
+| RAM usage                  | 100-400mb~              | 400mb-1GB | 100mb      | 200mb-1GB                            |
+| Does it suck               | no                      | yes       | kind of?   | its kinda cool but no search filters |
+| Does it work               | yes                     | err       | is dead    | works right now...                   |
 
 ## Features
 1. Rotating proxies on a per-scraper basis
@@ -24,25 +24,29 @@ tl;dr FOSS-Q is the best way to browse for stuff.
 
 # Supported websites
 
-| Web        | Images       | Videos       | News       | Music      | Autocompleter |
-|------------|--------------|--------------|------------|------------|---------------|
-| DuckDuckGo | DuckDuckGo   | YouTube      | DuckDuckGo | Soundcloud | Brave         |
-| Brave      | Brave        | Sepia Search | Brave      |            | DuckDuckGo    |
-| Yandex     | Yandex       | DuckDuckGo   | Google     |            | Yandex        |
-| Google     | Google       | Brave        | Startpage  |            | Google        |
-| Startpage  | Startpage    | Yandex       | Qwant      |            | Startpage     |
-| Qwant      | Qwant        | Google       | Mojeek     |            | Kagi          |
-| Ghostery   | Yep          | Startpage    | Baidu      |            | Qwant         |
-| Yep        | Baidu        | Qwant        |            |            | Ghostery      |
-| Greppr     | Pinterest    | Baidu        |            |            | Yep           |
-| Crowdview  | 500px        | Coc Coc      |            |            | Marginalia    |
-| Mwmbl      | VSCO         |              |            |            | YouTube       |
-| Mojeek     | Imgur        |              |            |            | Soundcloud    |
-| Baidu      | FindThatMeme |              |            |            |               |
-| Coc Coc    |              |              |            |            |               |
-| Marginalia |              |              |            |            |               |
-| wiby       |              |              |            |            |               |
-| Curlie     |              |              |            |            |               |
+| web          | images       | videos       | news         | music      | autocomplete |
+|--------------|--------------|--------------|--------------|------------|--------------|
+| DuckDuckGo   | DuckDuckGo   | YouTube      | DuckDuckGo   | SoundCloud | Brave        |
+| Brave        | Yandex       | Vimeo        | Brave        | Swisscows  | DuckDuckGo   |
+| Yandex       | Brave        | Sepia Search | Google       |            | Yandex       |
+| Google       | Google       | DuckDuckGo   | Yahoo! JAPAN |            | Google       |
+| Google API   | Google API   | Brave        | Startpage    |            | Startpage    |
+| Google CSE   | Google CSE   | Yandex       | Qwant        |            | Kagi         |
+| Yahoo! JAPAN | Yahoo! JAPAN | Google       | Mojeek       |            | Qwant        |
+| Startpage    | Startpage    | Yahoo! JAPAN | Baidu        |            | Ghostery     |
+| Qwant        | Qwant        | Startpage    |              |            | Yep          |
+| Ghostery     | Baidu        | Qwant        |              |            | Marginalia   |
+| Yep          | Solofield    | Baidu        |              |            | YouTube      |
+| Mwmbl        | Pinterest    | Coc Coc      |              |            | SoundCloud   |
+| Mojeek       | Cara         | Solofield    |              |            |              |
+| Baidu        | Flickr       |              |              |            |              |
+| Coc Coc      | Pexels       |              |              |            |              |
+| Solofield    | Pixabay      |              |              |            |              |
+| Marginalia   | Unsplash     |              |              |            |              |
+| wiby         | 500px        |              |              |            |              |
+|              | VSCO         |              |              |            |              |
+|              | Imgur        |              |              |            |              |
+|              | FindThatMeme |              |              |            |              |
 
 # Installation
 Refer to the <a href="https://git.flossboxin.org.in/FbIN/fossq/src/branch/main/docs/">documentation index</a>. I recommend following the <a href="https://git.flossboxin.org.in/FbIN/fossq/src/branch/main/docs/apache2.md">apache2 guide</a>.

lib/fuckhtml.php

@@ -553,28 +553,21 @@ class fuckhtml{
 				
 				case "\"":
 				case "'":
-					if(
-						$i !== 0 && // only check if a quote could be there
-						(
-							(
-								$json[$i - 1] === "\\" &&
-								(
-									$i === 2 ||
-									$json[$i - 2] === "\\"
-								)
-							) ||
-							$json[$i - 1] !== "\\"
-						)
-					){
-						// found a non-escaped quote
-						
+					// count preceding backslashes
+					$bsCount = 0;
+					$j = $i - 1;
+
+					while($j >= 0 && $json[$j] === "\\"){
+						$bsCount++;
+						$j--;
+					}
+
+					// quote is NOT escaped if even number of backslashes
+					if($bsCount % 2 === 0){
 						if($in_quote === null){
-							
 							// open quote
 							$in_quote = $json[$i];
-							
 						}elseif($in_quote === $json[$i]){
-							
 							// close quote
 							$in_quote = null;
 						}
@@ -632,4 +625,4 @@ class fuckhtml{
 		// fallback
 		return "[]";
 	}
-}
+}

scraper/brave.php

@@ -347,11 +347,8 @@ class brave{
 				$q["spellcheck"] = "0";
 			}
 		}
-		/*
-		$handle = fopen("scraper/brave.html", "r");
-		$html = fread($handle, filesize("scraper/brave.html"));
-		fclose($handle);*/
 		
+		//$html = file_get_contents("scraper/brave.html");
 		try{
 			$html =
 				$this->get(

scraper/pinterest.php

@@ -297,6 +297,14 @@ class pinterest{
 			throw new Exception("Failed to decode JSON");
 		}
 		
+		if(
+			isset($json["client_context"]["is_bad_bot"]) &&
+			(int)$json["client_context"]["is_bad_bot"] === 1
+		){
+			
+			throw new Exception("Pinterest blocked this instance or request proxy.");
+		}
+		
 		$out = [
 			"status" => "ok",
 			"npt" => null,
@@ -426,7 +434,7 @@ class pinterest{
 							]
 						],
 						"url" =>
-							$item["link"] === null ?
+							!isset($item["link"]) ?
 							"https://ca.pinterest.com/pin/" . $item["id"] :
 							$item["link"]
 					];

scraper/yandex.php

@@ -390,7 +390,7 @@ class yandex{
 			$title[0]["innerHTML"] == "403"
 		){
 			
-			throw new Exception("Yandex blocked this proxy or FOSS-Q instance.");
+			throw new Exception("Yandex blocked this proxy or 4get instance.");
 		}
 		
 		// get nextpage
@@ -733,7 +733,7 @@ class yandex{
 			$json["type"] == "captcha"
 		){
 			
-			throw new Exception("Yandex blocked this FOSS-Q instance. Please try again in ~7 minutes.");
+			throw new Exception("Yandex blocked this 4get instance. Please try again in ~7 minutes.");
 		}
 		
 		$out = [
@@ -868,123 +868,71 @@ class yandex{
 		
 		if($get["npt"]){
 			
-			[$params, $proxy] =
+			[$get, $proxy] =
 				$this->backend->get(
 					$get["npt"],
 					"video"
 				);
 			
-			$params = json_decode($params, true);
-			
-			$nsfw = $params["nsfw"];
-			unset($params["nsfw"]);
+			$get = json_decode($get, true);
 		}else{
 			
-			$search = $get["s"];
-			if(strlen($search) === 0){
+			if(strlen($get["s"]) === 0){
 				
 				throw new Exception("Search term is empty!");
 			}
 			
 			$proxy = $this->backend->get_ip();
-			$nsfw = $get["nsfw"];
-			$time = $get["time"];
-			$duration = $get["duration"];
-			
-			// https://yandex.com/video/search
-			// ?tmpl_version=releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63
-			// &format=json
-			// &request=
-			// {
-			//		"blocks":[
-			//			{"block":"extra-content","params":{},"version":2},
-			//			{"block":"i-global__params:ajax","params":{},"version":2},
-			//			{"block":"search2:ajax","params":{},"version":2},
-			//			{"block":"vital-incut","params":{},"version":2},
-			//			{"block":"content_type_search","params":{},"version":2},
-			//			{"block":"serp-controller","params":{},"version":2},
-			//			{"block":"cookies_ajax","params":{},"version":2}
-			//		],
-			//		"metadata":{
-			//			"bundles":{"lb":"^G]!q<X120"},
-			//			"assets":{"las":"react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"},
-			//			"extraContent":{"names":["i-react-ajax-adapter"]}
-			//		}
-			// }
-			// &yu=4861394161661655015
-			// &from=tabbar
-			// &reqid=1693106278500184-6825210746979814879-balancer-l7leveler-kubr-yp-sas-7-BAL-4237
-			// &suggest_reqid=486139416166165501562797413447032
-			// &text=minecraft
-			
-			$params = [
-				"tmpl_version" => "releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63",
-				"format" => "json",
-				"request" => json_encode([
-					"blocks" => [
-						(object)[
-							"block" => "extra-content",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "i-global__params:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "search2:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "vital-incut",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "content_type_search",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "serp-controller",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "cookies_ajax",
-							"params" => (object)[],
-							"version" => 2
-						]
-					],
-					"metadata" => (object)[
-						"bundles" => (object)[
-							"lb" => "^G]!q<X120"
-						],
-						"assets" => (object)[
-							"las" => "react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"
-						],
-						"extraContent" => (object)[
-							"names" => [
-								"i-react-ajax-adapter"
-							]
-						]
-					]
-				]),
-				"text" => $search
-			];
-			
-			if($duration != "any"){
-				
-				$params["duration"] = $duration;
-			}
-			
-			if($time != "any"){
-				
-				$params["within"] = $time;
-			}
 		}
+		
+		// https://yandex.com/video/search?text=skycamefalling&from=tabbar&format=json&ncrnd=7271&p=0&parent-reqid=&request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]}&serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL&yu=3091577281773194415&tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
+		// https://yandex.com/video/search
+		// ?text=skycamefalling
+		// &from=tabbar
+		// &format=json
+		// &ncrnd=7271
+		// &p=0
+		// &parent-reqid=
+		// &request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]} {"blocks":[{"block":"video-app","params":{}}]}
+		// &serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL
+		// &yu=3091577281773194415
+		// &tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
+		
+		$params = [
+			"text" => $get["s"],
+			"from" => "tabbar",
+			"format" => "json",
+			"ncrnd" => 7271,
+			"p" => 0,
+			"parent-reqid" => "",
+			"request" => json_encode((object)[
+				"blocks" => [
+					(object)[
+						"block" => "video-app",
+						"params" => (object)[]
+					]
+				]
+			]),
+			"serpid" => "1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL",
+			"yu" => 3091577281773194415,
+			"tmpl_version" => "releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a"
+		];
+		
+		if(isset($get["p"])){
+			
+			$params["p"] = $get["p"];
+		}
+		
+		if($get["duration"] != "any"){
+			
+			$params["duration"] = $get["duration"];
+		}
+		
+		if($get["time"] != "any"){
+			
+			$params["within"] = $get["time"];
+		}
+		
 		/*
 		$handle = fopen("scraper/yandex-video.json", "r");
 		$json = fread($handle, filesize("scraper/yandex-video.json"));
@@ -996,7 +944,7 @@ class yandex{
 					$proxy,
 					"https://yandex.com/video/search",
 					$params,
-					$nsfw,
+					$get["nsfw"],
 					"yandex_v"
 				);
 		}catch(Exception $error){
@@ -1011,9 +959,9 @@ class yandex{
 			throw new Exception("Could not parse JSON");
 		}
 		
-		if(!isset($json["blocks"])){
+		if(!isset($json["results"]["clips"]["items"])){
 			
-			throw new Exception("Yandex blocked this FOSS-Q instance. Please try again in 7~ minutes.");
+			throw new Exception("Yandex blocked this 4get instance. Please try again in 7~ minutes.");
 		}
 		
 		$out = [
@@ -1026,209 +974,120 @@ class yandex{
 			"reel" => []
 		];
 		
-		$html = null;
-		foreach($json["blocks"] as $block){
+		foreach($json["results"]["clips"]["items"] as $k => $data){
 			
-			if(isset($block["html"])){
+			if(isset($data["preview"]["posterSrc"])){
 				
-				$html .= $block["html"];
+				$poster = $data["preview"]["posterSrc"];
+				
+				if(
+					preg_match(
+						'/^\/\//',
+						$data["preview"]["posterSrc"]
+					)
+				){
+					
+					$poster = "https:" . $poster;
+				}
+				
+				$thumb = [
+					"ratio" => "16:9",
+					"url" => $poster
+				];
+			}else{
+				
+				$thumb = [
+					"ratio" => null,
+					"url" => null
+				];
 			}
+			
+			$out["video"][] = [
+				"title" => $data["relatedParams"]["text"],
+				"description" => $this->titledots($data["description"]),
+				"author" => [
+					"name" =>
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"] : null,
+					"url" => 
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"] : null,
+					"avatar" => null
+				],
+				"date" =>
+					isset($json["results"]["clips"]["dups"][$k]["date"]) ?
+					strtotime($json["results"]["clips"]["dups"][$k]["date"]) : null,
+				"duration" =>
+					isset($json["results"]["clips"]["dups"][$k]["duration"]["value"]) ?
+					(int)$json["results"]["clips"]["dups"][$k]["duration"]["value"] : null,
+				"views" =>
+					isset($json["results"]["clips"]["dups"][$k]["views"]["text"]) ?
+					$this->parseviews($json["results"]["clips"]["dups"][$k]["views"]["text"]) : null,
+				"thumb" => $thumb,
+				"url" =>
+					preg_replace(
+						'/^http:\/\//',
+						"https://",
+						$data["relatedParams"]["related_url"]
+					)
+			];
 		}
 		
-		$this->fuckhtml->load($html);
-		
-		$div =
-			$this->fuckhtml
-			->getElementsByTagName("div");
-		
-		/*
-			Get nextpage
-		*/
-		$npt =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"more more_direction_next i-bem",
-				$div
-			);
-		
-		if(count($npt) !== 0){
+		// get npt
+		if($json["results"]["search"]["hasNextPage"]){
+			
+			$get["p"] = (int)$json["results"]["search"]["currentPage"] + 1;
 			
-			$params["p"] = "1";
-			$params["nsfw"] = $nsfw;
 			$out["npt"] =
 				$this->backend->store(
-					json_encode($params),
+					json_encode($get),
 					"video",
 					$proxy
 				);
 		}
 		
-		$items =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"serp-item",
-				$div
-			);
-		
-		foreach($items as $item){
-			
-			$data =
-				json_decode(
-					$this->fuckhtml
-					->getTextContent(
-						$item["attributes"]["data-video"]
-					),
-					true
-				);
-			
-			$this->fuckhtml->load($item);
-			
-			$thumb =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"thumb-image__image",
-					"img"
-				);
-			
-			$c = 1;
-			if(count($thumb) === 0){
-				
-				$thumb = [
-					"url" => null,
-					"ratio" => null
-				];
-			}else{
-				
-				$thumb = [
-					"url" =>
-						str_replace(
-							"//",
-							"https://",
-							$this->fuckhtml
-							->getTextContent(
-								$thumb
-								[0]
-								["attributes"]
-								["src"]
-							),
-							$c
-						),
-					"ratio" => "16:9"
-				];
-			}
-			
-			$smallinfos =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__sitelinks-item",
-					"div"
-				);
-			
-			$date = null;
-			$views = null;
-			$first = true;
-			
-			foreach($smallinfos as $info){
-				
-				if($first){
-					
-					$first = false;
-					continue;
-				}
-				
-				$info =
-					$this->fuckhtml
-					->getTextContent(
-						$info
-					);
-				
-				if($temp_date = strtotime($info)){
-					
-					$date = $temp_date;
-				}else{
-					
-					$views = $this->parseviews($info);
-				}
-			}
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__text serp-item__text_visibleText_always",
-					"div"
-				);
-			
-			if(count($description) === 0){
-				
-				$description = null;
-			}else{
-				
-				$description =
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$description[0]
-						)
-					);
-			}
-			
-			$out["video"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$this->titledots(
-							$data["title"]
-						)
-					),
-				"description" => $description,
-				"author" => [
-					"name" => null,
-					"url" => null,
-					"avatar" => null
-				],
-				"date" => $date,
-				"duration" =>
-					(int)$data
-					["counters"]
-					["toHostingLoaded"]
-					["stredParams"]
-					["duration"],
-				"views" => $views,
-				"thumb" => $thumb,
-				"url" =>
-					str_replace(
-						"http://",
-						"https://",
-						$this->fuckhtml
-						->getTextContent(
-							$data["counters"]
-							["toHostingLoaded"]
-							["postfix"]
-							["href"]
-						),
-						$c
-					)
-			];
-		}
-		
 		return $out;
 	}
 	
-	private function parseviews($text){
+	private function parseviews($number){
 		
-		$text = explode(" ", $text);
+		// decimal should always be 1 number long
+		$number = explode(" ", $number, 2);
+		$number = $number[0];
 		
-		$num = (float)$text[0];
-		$mod = $text[1];
+		$unit = strtolower($number[strlen($number) - 1]);
 		
-		switch($mod){
+		$tmp = explode(".", $number, 2);
+		$number = (int)$number;
+		
+		if(count($tmp) === 2){
 			
-			case "bln.": $num = $num * 1000000000; break;
-			case "mln.": $num = $num * 1000000; break;
-			case "thsd.": $num = $num * 1000; break;
+			$decimal = (int)$tmp[1];
+		}else{
+			
+			$decimal = 0;
 		}
 		
-		return $num;
+		switch($unit){
+			
+			case "k":
+				$exponant = 1000;
+				break;
+			
+			case "m":
+				$exponant = 1000000;
+				break;
+			
+			case "b";
+				$exponant = 1000000000;
+				break;
+			
+			default:
+				$exponant = 1;
+				break;
+		}
+		
+		return ($number * $exponant) + ($decimal * ($exponant / 10));
 	}
 	
 	private function titledots($title){
@@ -1245,4 +1104,4 @@ class yandex{
 		
 		return trim($title);
 	}
-}
+}