fix request count not resetting, added request graph (be3abbef16)
(1a4b91435a) (2305bd37f6) (bfebf441b8) (462c5e1729)
This commit is contained in:
FbIN Support 2026-09-06 18:01:22 +05:30
commit 3591fd879a
18 changed files with 549 additions and 25 deletions

View file

@ -5,8 +5,20 @@ header("Access-Control-Allow-Origin: *");
include "data/config.php";
$real_requests = apcu_fetch("real_requests");
$bot_requests = apcu_fetch("captcha_gen");
// get request count in last 24 hours
$bot_requests = 0;
$real_requests = 0;
$time = time();
for($i=0; $i<24; $i++){
$bot_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".bot_requests");
}
for($i=0; $i<24; $i++){
$real_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".real_requests");
}
echo json_encode(
[
@ -16,8 +28,8 @@ echo json_encode(
"name" => config::SERVER_NAME,
"description" => config::SERVER_LONG_DESCRIPTION,
"bot_protection" => config::BOT_PROTECTION,
"real_requests" => $real_requests === false ? 0 : $real_requests,
"bot_requests" => $bot_requests === false ? 0 : $bot_requests,
"real_requests" => $real_requests,
"bot_requests" => $bot_requests,
"api_enabled" => config::API_ENABLED,
"alt_addresses" => config::ALT_ADDRESSES,
"version" => config::VERSION

View file

@ -185,7 +185,7 @@ class autocomplete{
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0",
["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0",
"Accept: application/json, text/javascript, */*; q=0.01",
"Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip",

View file

@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "images", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->image($get),

View file

@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "music", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->music($get),

View file

@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "news", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->news($get),

View file

@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "videos", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->video($get),

View file

@ -38,6 +38,9 @@ if(
$get["extendedsearch"] = "no";
}
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo

87
api/v2/graph.php Normal file
View file

@ -0,0 +1,87 @@
<?php
date_default_timezone_set('America/Toronto');
include "../../data/config.php";
function get_requests($slug){
$time = time();
$requests = [];
for($i=0; $i<73; $i++){
$data = apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".$slug");
$requests[] = [$i, ($data === false ? 0 : $data)];
}
// rtrim() out all entries that are 0
$out = [];
$found = false;
for($i=count($requests) - 1; $i>=0; $i--){
if($requests[$i][1] !== 0){
$found = $i;
break;
}
}
if($found === false){
return [];
}
$requests = array_slice($requests, 0, $found + 1);
return $requests;
}
$bot_requests = get_requests("bot_requests");
$real_requests = get_requests("real_requests");
$real_reqs_24h = 0;
for($i=0; $i<count($real_requests); $i++){
if($real_requests[$i][1] !== false){
$real_reqs_24h += $real_requests[$i][1];
}
}
$real_reqs_72h = 0;
for($i=0; $i<count($real_requests); $i++){
if($real_requests[$i][1] !== false){
$real_reqs_72h += $real_requests[$i][1];
}
}
header("Content-Type: image/png");
include "../../lib/graph_gen.php";
$graph = new graph();
$c = count($real_requests);
if($c > 25){
$c = 25;
}elseif($c <= 2){
$c = 2;
}
echo $graph->render([
"title" => "Number of requests within the last 72 hours (non-cummulative)",
"subtitle" => date("jS M y @ g:ia", time()) . " || 24h=" . number_format($real_reqs_24h) . ", 72h=" . number_format($real_reqs_72h) . " || " . config::SERVER_NAME,
"x_label" => "<= Hours passed =>",
"y_label" => "<= Request count/hour =>",
"grad_x" => $c,
"grad_y" => 20,
"data" => [
"Bot requests" => $bot_requests,
"Real requests" => $real_requests,
]
]);

View file

@ -5,7 +5,7 @@ class config{
// any parameters.
// FOSS-Q version. Please keep this updated
const VERSION = 8;
const VERSION = 9;
// Will be shown pretty much everywhere.
const SERVER_NAME = "FOSS-Q";
@ -134,7 +134,7 @@ class config{
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
// Changing this might break things.
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0";
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0";
// User agent to use with 4get-friendly APIs
const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";

View file

@ -24,6 +24,9 @@ $payload = [
"nextpage" => ""
];
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
$results = $scraper->image($get);

View file

@ -7,7 +7,6 @@ class bot_protection{
// check if we want captcha
if(config::BOT_PROTECTION !== 1){
apcu_inc("real_requests");
if($output === true){
$frontend->loadheader(
$get,
@ -46,8 +45,6 @@ class bot_protection{
}else{
// the cookie is OK! dont die() and give results
apcu_inc("real_requests");
if($output === true){
$frontend->loadheader(
$get,
@ -132,6 +129,7 @@ class bot_protection{
$answers[] = $regex;
}
// dedup
$answers = array_unique($answers);
@ -163,8 +161,7 @@ class bot_protection{
$key = "k" . $inc . "." . $this->randomchars();
apcu_inc($key, 1, $stupid, 86400);
apcu_inc("real_requests");
apcu_dec(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
setcookie(
"pass",
@ -189,6 +186,7 @@ class bot_protection{
}
}
apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 259200);
$key = "c" . apcu_inc("captcha_gen", 1) . "." . $this->randomchars();
$payload = [
@ -251,7 +249,8 @@ class bot_protection{
$frontend->loadheader(
$get,
$filters,
$page
$page,
false
);
echo $frontend->load("search.html", $payload);

View file

@ -118,7 +118,7 @@ class frontend{
return trim($html);
}
public function loadheader(array $get, array $filters, string $page){
public function loadheader(array $get, array $filters, string $page, bool $increment_bot_counter = true){
echo
$this->load("header.html", [
@ -177,13 +177,15 @@ class frontend{
){
// bot detected !!
apcu_inc("captcha_gen");
if($increment_bot_counter){
apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
}
$this->drawerror(
"Tshh, blocked!",
'Your browser, IP or IP range has been blocked from this FOSS-Q instance. If this is an error, please <a href="/about">contact the administrator</a>.'
);
die();
}
}
@ -639,8 +641,8 @@ class frontend{
//"yahoo" => "Yahoo!",
"brave" => "Brave",
"yandex" => "Yandex",
// "google" => "Google",
// "google_api" => "Google API",
"google" => "Google",
"google_api" => "Google API",
"google_cse" => "Google CSE",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
@ -1052,6 +1054,12 @@ class frontend{
return http_build_query($out);
}
public function increment_real_reqs($scraper){
apcu_inc(intdiv(time(), 3600) . ".real_requests", 1, $s, 262800);
apcu_inc(intdiv(time(), 3600) . ".$scraper.requests", 1, $s, 262800);
}
public function htmlimage($image, $format){
if(

388
lib/graph_gen.php Normal file
View file

@ -0,0 +1,388 @@
<?php
class graph{
public function render(array $graph = [], array $size = [], array $colors = [], array $colors_line = []){
$graph = array_merge([
"title" => "Title",
"subtitle" => "description",
"grad_x" => 25,
"grad_y" => 20,
"x_label" => "<= x =>",
"y_label" => "<= y =>",
"antialias" => false,
"data" => []
], $graph);
$size = array_merge([
"image_width" => 750,
"image_height" => 500,
"x" => 90,
"y" => 50,
"width" => 650,
"height" => 300,
"square" => 11,
"column" => 6
], $size);
$colors = array_merge([
"bg" => new ImagickPixel("#1d2021"),
"lines_bg" => new ImagickPixel("#3c3836"),
"outline" => new ImagickPixel("#a89984"),
"text" => new ImagickPixel("#ebdbb2"),
"transparent" => new ImagickPixel("transparent")
], $colors);
$colors_line = array_merge([
new ImagickPixel("#83a598"),
new ImagickPixel("#fb4934"),
new ImagickPixel("#b8bb26"),
new ImagickPixel("#fabd2f"),
new ImagickPixel("#d3869b"),
new ImagickPixel("#8ec07c"),
new ImagickPixel("#fe8019"),
new ImagickPixel("#458588"),
new ImagickPixel("#cc241d"),
new ImagickPixel("#98971a"),
new ImagickPixel("#d79921"),
new ImagickPixel("#b16286"),
new ImagickPixel("#689d6a"),
new ImagickPixel("#d65d0e"),
new ImagickPixel("#a89984") // other
], $colors_line);
$not_enough_data = false;
// compute datapoint max ranges
$max_range_x = 0;
$max_range_y = 0;
if(count($graph["data"]) === 0){
$not_enough_data = true;
}else{
foreach($graph["data"] as $datapoint => $value){
if(count($value) < 2){
$not_enough_data = true;
}else{
$nv1 = [];
$nv2 = [];
foreach($value as $k => $v){
$nv1[] = $v[0];
$nv2[] = $v[1];
}
$max1 = max($nv1);
$max2 = max($nv2);
if($max_range_x < $max1){ $max_range_x = $max1; }
if($max_range_y < $max2){ $max_range_y = $max2; }
}
}
}
$im = new Imagick();
$im->newImage($size["image_width"], $size["image_height"], $colors["bg"]);
$im->setImageFormat("png");
$draw = new ImagickDraw();
//
// Draw background grid
//
$draw->setStrokeColor($colors["lines_bg"]);
$draw->setStrokeAntialias($graph["antialias"]);
$draw->setStrokeWidth(1);
$draw->setFillColor($colors["transparent"]);
$text = new ImagickDraw();
$text->setFontSize(11);
$text->setTextAntialias($graph["antialias"]);
$text->setFillColor($colors["text"]);
// vertical lines
$offset = $size["width"] / ($graph["grad_x"] - 1);
for($i=0; $i<$graph["grad_x"]; $i++){
$x = round($size["x"] + ($offset * $i));
if(
$i === 0 ||
$i === $graph["grad_x"] - 1
){
$draw->setStrokeColor($colors["outline"]);
}else{
$draw->setStrokeColor($colors["lines_bg"]);
}
$draw->line(
$x,
$size["y"],
$x,
$size["y"] + $size["height"],
);
if($not_enough_data === false){
// add numbas
$number = number_format(round(($graph["grad_x"] - 1 - $i) * ($max_range_x / ($graph["grad_x"] - 1))));
$number_width = $im->queryFontMetrics($text, $number)["textWidth"];
$im->annotateImage(
$text,
round($x - ($number_width / 2)),
$size["y"] + $size["height"] + 14,
0,
$number
);
}
}
// horizontal lines
$text->setTextAlignment(Imagick::ALIGN_RIGHT);
$offset = $size["height"] / ($graph["grad_y"] - 1);
$vertical_text_size = 0;
for($i=0; $i<$graph["grad_y"]; $i++){
$y = round($size["y"] + ($offset * $i));
if(
$i === 0 ||
$i === $graph["grad_y"] - 1
){
$draw->setStrokeColor($colors["outline"]);
}else{
$draw->setStrokeColor($colors["lines_bg"]);
}
$draw->line(
$size["x"] + 1,
$y,
$size["x"] + $size["width"] - 1,
$y,
);
if($not_enough_data === false){
$val = number_format(round(($graph["grad_y"] - 1 - $i) * ($max_range_y / ($graph["grad_y"] - 1))));
$query = $im->queryFontMetrics($text, $val);
if($query["textWidth"] > $vertical_text_size){
$vertical_text_size = $query["textWidth"];
}
// add numbas
$im->annotateImage(
$text,
$size["x"] - 4,
$y + 4,
0,
$val
);
}
}
//
// Draw legend
//
$text->setTextAlignment(Imagick::ALIGN_LEFT);
$top_padding = 44;
$c = count($colors_line);
$padding = $size["width"] / $size["column"];
$line = new ImagickDraw();
//$line->setStrokeWidth(1);
$line->setStrokeAntialias($graph["antialias"]);
if($not_enough_data === false){
$i = 0;
foreach($graph["data"] as $datafield => $data){
$used_color = $colors_line[$i % $c];
$draw->setStrokeColor($used_color);
$row = floor($i / $size["column"]);
$x = round($size["x"] + (($i % $size["column"]) * $padding));
$y = $size["y"] + $size["height"] + $top_padding + ($row * 17);
// draw legend
$draw->rectangle(
$x,
$y,
$x + $size["square"],
$y + $size["square"]
);
$im->annotateImage(
$text,
$x + $size["square"] + 4,
$y + 10,
0,
$datafield
);
//
// Draw lines
//
$line->setStrokeColor($used_color);
$old_x = null;
$old_y = null;
$continue = true;
foreach($data as $field){
// field 0 = hour (sorted)
// field 1 = datapoint
$x = $size["x"] + (($max_range_x - $field[0]) * ($size["width"] / $max_range_x));
$y = $size["y"] + (($max_range_y - $field[1]) * ($size["height"] / $max_range_y));
if($continue){
$continue = false;
$old_x = $x;
$old_y = $y;
continue;
}
$line->line($old_x, $old_y, $x, $y);
$old_x = $x;
$old_y = $y;
}
$i++;
}
}
//
// add background labels
//
$text->setFontSize(13);
$text->setFillColor($colors["bg"]);
$draw->setFillColor($colors["text"]);
$draw->setStrokeColor($colors["transparent"]);
// horizontal
$label_width_x = $im->queryFontMetrics($text, $graph["x_label"]);
$x_x = $size["x"] + ($size["width"] / 2) - ($label_width_x["textWidth"] / 2);
$x_y = $size["y"] + $size["height"] + ($not_enough_data ? 21 : 30);
$draw->rectangle(
$x_x - 2,
$x_y - $label_width_x["textHeight"] + 4,
$x_x + $label_width_x["textWidth"] + 2,
$x_y + 3,
);
// vertical
$label_width_y = $im->queryFontMetrics($text, $graph["y_label"]);
$y_x = $size["x"] - $label_width_y["textHeight"] - 13 - $vertical_text_size;
$y_y = $size["y"] + ($size["height"] / 2) - ($label_width_y["textWidth"] / 2);
$draw->rectangle(
$y_x + 4,
$y_y - 2,
$y_x + $label_width_y["textHeight"] + 3,
$y_y + $label_width_y["textWidth"] + 2,
);
$im->drawImage($draw);
$im->drawImage($line);
// horizontal text
$im->annotateImage(
$text,
$x_x,
$x_y,
0,
$graph["x_label"]
);
// vertical text
$im->annotateImage(
$text,
$y_x + $label_width_y["textHeight"],
$y_y + $label_width_y["textWidth"],
270,
$graph["y_label"]
);
//
// draw title + subtext
//
$text->setFontSize(20);
$text->setFontWeight(900);
$text->setFillColor($colors["text"]);
$im->annotateImage(
$text,
$size["x"],
$size["y"] - 25,
0,
$graph["title"]
);
$text->setFontSize(13);
$text->setFontWeight(100);
$im->annotateImage(
$text,
$size["x"],
$size["y"] - 10,
0,
$graph["subtitle"]
);
if($not_enough_data){
// add not enough data notice
$rect = new ImagickDraw();
$rect->setFillColor($colors["bg"]);
$rect->setStrokeColor($colors_line[1]);
$rect_w = 200;
$rect_h = 24;
$rect_x = round($size["x"] + ($size["width"] / 2) - ($rect_w / 2));
$rect_y = round($size["y"] + ($size["height"] / 2) - ($rect_h / 2));
$rect->rectangle(
$rect_x,
$rect_y,
$rect_x + $rect_w,
$rect_y + $rect_h
);
$im->drawImage($rect);
$label = "Not enough data";
$text->setFontWeight(900);
$query = $im->queryFontMetrics($text, $label);
$im->annotateImage(
$text,
round($size["x"] + ($size["width"] / 2) - ($query["textWidth"] / 2)),
round($size["y"] + ($size["height"] / 2) + ($query["textHeight"] / 2)) - 3,
0,
$label
);
}
return $im->getImageBlob();
}
}

View file

@ -26,6 +26,9 @@ $payload = [
"left" => ""
];
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
$results = $scraper->music($get);

View file

@ -26,6 +26,9 @@ $payload = [
"left" => ""
];
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
$results = $scraper->news($get);

View file

@ -22,7 +22,7 @@ var list = [];
var pinged_list = [];
var reqs = 0;
var errors = 0;
var sort = 6; // highest version first
var sort = 12;
// check for instance redirect stuff
var redir = [];
@ -68,8 +68,8 @@ table.innerHTML =
'<th class="extend">Server</th>' +
'<th>Address</th>' +
'<th>Bot protection</th>' +
'<th title="Amount of legit requests processed since the last APCU cache clear (usually happens at midnight)">Real reqs (?)</th>' +
'<th title="Amount of filtered requests processed since the last APCU cache clear (usually happens at midnight)">Bot reqs (?)</th>' +
'<th title="Amount of legit requests processed within the last 24 hours">Real reqs (?)</th>' +
'<th title="Amount of filtered requests processed within the last 24 hours">Bot reqs (?)</th>' +
'<th>API</th>' +
'<th><div class="arrow up"></div>Version</th>' +
'</tr>' +

View file

@ -26,6 +26,9 @@ $payload = [
"left" => ""
];
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
$results = $scraper->video($get);

View file

@ -1,7 +1,7 @@
<?php
/*
Initialize random shit
Initialize random stuff
*/
include "data/config.php";
@ -26,6 +26,9 @@ $payload = [
"left" => ""
];
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
$results = $scraper->web($get);