Synced

foogle scraper fix: db90a39c4f
fix proxy function: 9a4763ed3c
more fixes: 919714d472

data/config.php

@@ -24,15 +24,21 @@ class config{
 	const API_ENABLED = true;
 	
 	//
-	// 4play (session provider)
+	// 4play (settings)
 	//
-	// Enable 4play API?
-	const FPLAY_ENABLE_API = true;
+	// Enable fossq's /api/v2/provide_sesh endpoint
+	const FPLAY_ENABLE_API = false;
 	
-	// 4play password. Please set this to something secure if you enable the 4play API.
+	// 4play password. Please set this to something secure if you use this extension.
 	// This password is used to POST sessions to /api/v2/provide_sesh
+	// It's also used as the external endpoint password.
 	const FPLAY_PASSWORD = "cnc";
 	
+	// Endpoint fossq hits to render webpages
+	// Eg: null=no endpoint provided
+	// Or "https://your-endpoint.com/my-endpoint" (excluding get parameters like &pw)
+	const FPLAY_EXTERNAL_ENDPOINT = null;
+	
 	//
 	// BOT PROTECTION
 	//

docs/configure.md

@@ -30,26 +30,29 @@ sudo make firefox-install
 sudo ldconfig
 ```
 
-Now, after compiling, you should have a `libcurl-impersonate-ff.so` sitting somewhere. Mine is located at `/usr/local/lib/libcurl-impersonate-ff.so`. Patch your PHP install so that it loads the right library: 
+Now, after compiling, you should have a `libcurl-impersonate-ff.so` sitting somewhere. Mine is located at `/usr/local/lib/libcurl-impersonate-ff.so`. Patch your PHP install so that it loads the right library:
 
 ```sh
 sudo systemctl edit php8.4-fpm.service
 ```
 
 ^This will open a text editor. Add the following in there, in between those 2 comments I pasted just for reference:
+
 ```sh
 ### Editing /etc/systemd/system/php8.4-fpm.service.d/override.conf
 ### Anything between here and the comment below will become the contents of the>
+
 [Service]
 Environment="LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so"
 Environment="CURL_IMPERSONATE=firefox117"
+
 ### Edits below this comment will be discarded
 ```
 
-Restart php8.4-fpm. (`sudo service php8.4-fpm restart`). To test things out, try making a search on "Yep", they check for SSL. If you get results (or a timeout) that means it works! 
+Restart php8.4-fpm. (`sudo service php8.4-fpm restart`). To test things out, try making a search on "Yep", they check for SSL. If you get results (or a timeout), that means it works!
 
 # Robots.txt
-Make sure you configure this right to optimize your search engine presence! Head over to `/robots.txt` and change the 4get.ca domain to your own domain.
+Make sure you configure this right to optimize your search engine presence! Head over to `/robots.txt` and change the fossq.com domain to your own domain.
 
 # Server listing
 To be listed on https://4get.ca/instances , you must contact *any* of the people in the server list and ask them to add you to their list of instances in their configuration. The instance list is distributed, and I don't have control over it.
@@ -75,4 +78,15 @@ FOSS-Q supports rotating proxies for scrapers! Configuring one is really easy.
 Done! The scraper you chose should now be using the rotating proxies. When asking for the next page of results, it will use the same proxy to avoid detection!
 
 ## Important!
-If you ever test out a `socks5` proxy locally on your machine and find out it works but doesn't on your server, try supplying the `socks5_hostname` protocol instead. Hopefully this tip can save you 3 hours of your life!
+If you ever test out a `socks5` proxy locally on your machine and find out it works but doesn't on your server, try supplying the `socks5_hostname` protocol instead. Hopefully this tip can save you 3 hours of your life!
+
+# 4play
+4play is a Firefox extension. I made it, and it allows me to control a Firefox browser through code. More information about the project [here](https://git.lolcat.ca/lolcat/4play). It allows you to scrape Google (& probably more sites in the future)
+
+1. Install the [firefox extension](https://addons.mozilla.org/en-US/firefox/addon/4play/) onto a clean Firefox profile. Make sure you have a real display connected and that you use real hardware with an user-grade GPU. Websites detect this now.
+2. Navigate to `/extra/4play/`
+3. Install the 4play server: `npm install @lawlers/4play`
+4. Edit the `render-server.js` file. Change the line `var password = "cnc";` so that it uses a secure password.
+5. Edit `/data/config.php` and update the 4play password. Also update the `FPLAY_EXTERNAL_ENDPOINT` so that it points to the render HTTP server.
+
+Now, if you did everything correctly, you should be able to render search pages on scrapers that require it. If your render server sits on a different server, I highly recommend setting up a reverse proxy to add TLS support.

extra/4play/page-render.js

@@ -0,0 +1,341 @@
+const fplay = require("@lawlers/4play");
+const http = require("http");
+
+// 4play settings
+var port = 3030;
+var timeout = 30000;
+var password = "cnc";
+
+// ==================================================
+
+// webserver
+
+var global_ws = null;
+const webserver_port = 3000;
+
+var callback_q = new Map();
+
+const server = http.createServer(async function(req, res){
+	
+	const parsed = new URL(req.url, "http://localhost");
+	
+	const url = parsed.searchParams.get("url");
+	const pw = parsed.searchParams.get("pw");
+	var proxy = parsed.searchParams.get("proxy");
+	const url_match = parsed.searchParams.get("url_match");
+	const content_match = parsed.searchParams.get("content_match");
+	const fail_url_match = parsed.searchParams.get("fail_url_match");
+	const fail_content_match = parsed.searchParams.get("fail_content_match");
+	
+	var container = parsed.searchParams.get("container");
+	
+	if(fplay.browser_connected === false){
+				
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain"
+			}
+		);
+		
+		res.end("No browser available to render the motherfucking page");
+		return;
+	}
+	
+	var errors = [];
+	
+	if(
+		pw === null ||
+		pw != password
+	){
+		
+		errors.push("pw");
+	}
+	
+	if(url === null){ errors.push("url"); }
+	if(proxy === null){ errors.push("proxy"); }
+	
+	if(errors.length !== 0){
+		
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain"
+			}
+		);
+		
+		res.end("The following parameters are missing or invalid: " + errors.join(", "));
+		return;
+	}
+	
+	console.log("Rendering " + url);
+	
+	if(
+		container === null ||
+		await fplay.container_exists(global_ws, container) === false
+	){
+		
+		container = await fplay.container_create(global_ws);
+		
+		proxy = get_proxy(proxy);
+		
+		// assign proxy
+		if(proxy.config !== false){
+			
+			await fplay.container_attach_proxy(
+				global_ws,
+				container,
+				proxy.config
+			);
+		}
+	}else{
+		
+		proxy = {raw: proxy};
+	}
+	
+	// open tab
+	const newtab = await fplay.tab_open(global_ws, url, false, container);
+	
+	if(newtab === false){
+		
+		// failed to open tab
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain",
+				"X-Proxy": proxy.raw,
+				"X-Container-ID": typeof container.id == "undefined" ? container : container.id
+			}
+		);
+		
+		res.end("Failed to open tab");
+		return;
+	}
+		
+	// callback will be handled in "web_response"
+	callback_q.set(
+		newtab.id,
+		{
+			http_res: res,
+			proxy: proxy,
+			container: container,
+			url_match: url_match,
+			content_match: content_match,
+			fail_url_match: fail_url_match,
+			fail_content_match: fail_content_match
+		}
+	);
+	
+	// close tab in 30 seconds
+	await fplay.wait_random(15000, 35000);
+	await fplay.tab_close(global_ws, newtab);
+});
+
+server.listen(webserver_port, function(){
+	
+	console.log("http server running");
+});
+
+function get_proxy(proxy_line){
+	
+	parts = proxy_line.split(":");
+	// format: type:address:port:username:password
+	// PHP explode(":", $ip, 5) splits into max 5 parts
+	var type = parts[0];
+	const address = parts[1];
+	const port = parts[2];
+	const username = parts[3];
+	const password = parts[4];
+	
+	var proxy_dns = true;
+	
+	switch(type){
+		case "raw_ip":
+		case "ip":
+			// no proxy, use direct connection
+			return {
+				config: false,
+				raw: "raw_ip::::"
+			}
+			break;
+		
+		case "socks5":
+			type = "socks";
+			proxy_dns = false;
+			break;
+			
+		case "socks4a":
+			type = "socks4";
+			break;
+			
+		case "socks4":
+			proxy_dns = false;
+			break;
+			
+		case "socks5_hostname":
+		case "socks5h":
+		case "socks5a":
+			type = "socks";
+			break;
+		
+		case "http":
+		case "https":
+			// do nothing
+			break;
+		
+		default:
+			// invalid proxy provided
+			return {
+				config: false,
+				raw: "raw_ip::::"
+			}
+	}
+	
+	return {
+		config: {
+			type: type,
+			host: address,
+			port: port,
+			username: username,
+			password: password,
+			proxyDNS: proxy_dns
+		},
+		raw: proxy_line
+	};
+}
+
+fplay.event.on("server_ready", function(){
+	
+	console.log("listening on port " + port + " (timeout=" + timeout + ")");
+});
+
+fplay.event.on("browser_connect", async function(ws){
+	
+	global_ws = ws;
+	
+	const ua = await fplay.get_ua(ws);
+	console.log("New browser instance connected: " + ua);
+	
+	await fplay.close_all_tabs(ws);
+	await fplay.delete_all_containers(ws);
+	
+	fplay.web_response_whitelist(ws, ["main_frame"]);
+	
+	// clean up
+	const blanktab = await fplay.close_all_tabs(ws);
+	await fplay.delete_all_containers(ws);
+});
+
+fplay.event.on("web_response", async function(page){
+	
+	var callback_data = callback_q.get(page.id);
+	
+	if(typeof callback_data == "undefined"){ return; }
+	
+	var body = page.body.toString();
+	
+	var url_match = page.url.match(new RegExp(callback_data.url_match, "i"));
+	var content_match = body.match(new RegExp(callback_data.content_match, "i"));
+	var fail_url_match = page.url.match(new RegExp(callback_data.fail_url_match, "i"));
+	var fail_content_match = body.match(new RegExp(callback_data.fail_content_match, "i"));
+	
+	// handle positive response
+	if(
+		(
+			callback_data.url_match === null &&
+			callback_data.content_match === null
+		) ||
+		(
+			url_match &&
+			content_match
+		) ||
+		(
+			url_match &&
+			callback_data.content_match === null
+		) ||
+		(
+			callback_data.url_match === null &&
+			content_match
+		)
+	){
+		
+		callback_data.http_res.writeHead(
+			200,
+			{
+				"Content-Type": "text/plain;charset=UTF-8",
+				"X-Proxy": callback_data.proxy.raw,
+				"X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id
+			}
+		);
+		
+		callback_data.http_res.end(body);
+		
+		callback_q.delete(page.id); // cleanup callback only
+		return;
+	}
+	
+	// handle negative response
+	if(
+		(
+			callback_data.fail_url_match !== null &&
+			callback_data.fail_content_match !== null
+		) &&
+		(
+			(
+				fail_url_match &&
+				fail_content_match
+			) ||
+			(
+				fail_url_match &&
+				callback_data.fail_content_match === null
+			) ||
+			(
+				callback_data.fail_url_match === null &&
+				fail_content_match
+			)
+		)
+	){
+		
+		callback_data.http_res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain",
+				"X-Proxy": callback_data.proxy.raw,
+				"X-Container-ID": callback_data.container.id
+			}
+		);
+		
+		callback_data.http_res.end("Reached fail state (matched fail_url_match or fail_content_match)");
+		
+		// cleanup browser + callback
+		await fplay.tab_close(global_ws, page.id);
+		await fplay.container_delete(global_ws, callback_data.container);
+		callback_q.delete(page.id);
+	}
+});
+
+fplay.event.on("dom_load_fail", async function(page){
+	
+	var callback_data = callback_q.get(page.id);
+	
+	if(typeof callback_data == "undefined"){ return; }
+	
+	callback_q.delete(page.id);
+	
+	callback_data.http_res.writeHead(
+		400,
+		{
+			"Content-Type": "text/plain",
+			"X-Proxy": callback_data.proxy.raw,
+			"X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id
+		}
+	);
+	
+	callback_data.http_res.end("Network error (" + page.error + ")");
+		
+	// cleanup browser
+	await fplay.tab_close(global_ws, page.id);
+	await fplay.container_delete(global_ws, callback_data.container);
+});
+
+fplay.init(port, password, timeout);

lib/backend.php

@@ -4,6 +4,9 @@ class backend{
 	public function __construct($scraper){
 		
 		$this->scraper = $scraper;
+		$this->db = null;
+		$this->db_path_prefix = "data/sessions/";
+		$this->db_path = "{$this->db_path_prefix}{$scraper}.sqlite";
 	}
 	
 	/*
@@ -208,4 +211,127 @@ class backend{
 			$payload[1] // proxy
 		];
 	}
-}
+	
+	//
+	// Sessions
+	//
+	public function session_init_db($force_init = false){
+		
+		if(
+			$force_init ||
+			$this->db === null
+		){
+			
+			$this->db = new SQLite3($this->db_path);
+			
+			// create table if it doesnt exist	
+			$this->db->exec(
+				"CREATE TABLE IF NOT EXISTS sessions ( " .
+					"id INTEGER PRIMARY KEY AUTOINCREMENT, " .
+					"created_at INTEGER NOT NULL, " .
+					"proxy TEXT NOT NULL, " .
+					"headers_json TEXT NOT NULL" .
+				")"
+			);
+			
+			return true;
+		}
+		
+		return false;
+	}
+	
+	public function session_count(){
+		
+		$this->session_init_db();
+		
+		$stmt = $this->db->prepare(
+			"SELECT COUNT(*) as count FROM sessions"
+		);
+		
+		$result = $stmt->execute();
+		$row = $result->fetchArray(SQLITE3_ASSOC);
+		
+		return (int)($row["count"] ?? 0);
+	}
+	
+	public function session_write($proxy, $headers){
+		
+		$this->session_init_db();
+		
+		$stmt = $this->db->prepare(
+			"INSERT INTO sessions ( " .
+				"created_at, " .
+				"proxy, " .
+				"headers_json " .
+			") VALUES ( " .
+				":created_at, " .
+				":proxy, " .
+				":headers_json" .
+			")"
+		);
+		
+		$stmt->bindValue(':created_at', time(), SQLITE3_INTEGER);
+		$stmt->bindValue(':proxy', $proxy, SQLITE3_TEXT);
+		$stmt->bindValue(':headers_json', json_encode($headers), SQLITE3_TEXT);
+		
+		$stmt->execute();
+		
+		return $this->db->lastInsertRowID();
+	}
+	
+	public function session_get(){
+		
+		$count = $this->session_count();
+		
+		if($count === 0){
+			
+			return false;
+		}
+		
+		$offset = apcu_inc("d." . $this->scraper) % $count;
+		
+		$stmt = $this->db->prepare(
+			"SELECT * " .
+			"FROM sessions " .
+			"ORDER BY id ASC " .
+			"LIMIT 1 OFFSET :offset"
+		);
+		
+		$stmt->bindValue(':offset', $offset, SQLITE3_INTEGER);
+		
+		$result = $stmt->execute();
+		$row = $result->fetchArray(SQLITE3_ASSOC);
+		
+		return [
+			"id" => $row["id"],
+			"created_at" => $row["created_at"],
+			"proxy" => $row["proxy"],
+			"headers" => json_decode($row["headers_json"], true)
+		];
+	}
+	
+	public function session_destroy_list(){
+		
+		if(
+			!preg_match(
+				'/^[A-Za-z0-9-]+$/',
+				$this->scraper
+			)
+		){
+			
+			// path traversal
+			return false;
+		}
+		
+		if(
+			file_exists("{$this->db_path_prefix}{$this->scraper}.sqlite") &&
+			is_file("{$this->db_path_prefix}{$this->scraper}.sqlite")
+		){
+			
+			unlink("{$this->db_path_prefix}{$this->scraper}.sqlite");
+			return true;
+		}
+		
+		return false;
+	}
+}

static/style.css

@@ -1013,6 +1013,23 @@ table tr a:last-child{
 	font-weight:bold;
 }
 
+/* booru */
+#images .image .i-title-wrapper{
+	margin-bottom:7px;
+	white-space:nowrap;
+	overflow:hidden;
+	text-overflow:ellipsis;
+}
+#images .image .i-title-wrapper a{
+	display:inline;
+}
+
+#images .image .i-title-wrapper .title{
+	display:inline;
+	margin-bottom:0;
+}
+/* end booru */
+
 #images .image .description{
 	overflow:hidden;
 	height:45px;
@@ -1113,6 +1130,17 @@ table tr a:last-child{
 /*
 	Settings page
 */
+.web .settings select{
+	all:unset;
+	user-select:none;
+	display:block;
+	border:1px solid var(--504945);
+	border-radius:2px;
+	font-size:14px;
+	padding:0 20px 0 4px;
+	height:24px;
+}
+
 .web .settings{
 	margin-top:17px;
 	border:1px dashed var(--504945);