(5a351e0dfc)

(429e53d41f)

added pixabay (8954d1aea5)

added pexels and unsplash (70f00aa55f)

add evergarden theme (7fc6244738)
This commit is contained in:
FbIN Support 2026-04-08 22:33:54 +05:30
commit f576d2c44c
7 changed files with 1052 additions and 404 deletions

View file

@ -118,7 +118,10 @@ class config{
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
// Changing this might break things.
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:142.0) Gecko/20100101 Firefox/145.0";
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0";
// User agent to use with 4get-friendly APIs
const USER_AGENT_FRIENDLY = "4get-scrapist";
// Proxy pool assignments for each scraper
// false = Use server's raw IP
@ -130,9 +133,8 @@ class config{
const PROXY_BRAVE = false;
const PROXY_FB = false; // facebook
const PROXY_GOOGLE = false;
const PROXY_GOOGLE_API = false;
const PROXY_GOOGLE_CSE = false;
const PROXY_MULLVAD_GOOGLE = false;
const PROXY_MULLVAD_BRAVE = false;
const PROXY_STARTPAGE = false;
const PROXY_QWANT = false;
const PROXY_BAIDU = false;
@ -153,6 +155,11 @@ class config{
const PROXY_VIMEO = false;
const PROXY_YEP = false;
const PROXY_PINTEREST = false;
const PROXY_SANKAKUCOMPLEX = false;
const PROXY_FLICKR = false;
const PROXY_PIXABAY = false;
const PROXY_UNSPLASH = false;
const PROXY_PEXELS = false;
const PROXY_FIVEHPX = false;
const PROXY_VSCO = false;
const PROXY_SEZNAM = false;
@ -178,4 +185,4 @@ class config{
// Use "null" to default out to HTML scraping OR specify a string to
// use the API (Eg: "public"). API has less filters.
const MARGINALIA_API_KEY = null;
}
}

View file

@ -446,407 +446,57 @@ class frontend{
Add favicon
*/
$host = parse_url($link);
$esc =
explode(
".",
$host["host"],
2
);
if(
count($esc) === 2 &&
$esc[0] == "www"
){
// special case for when we're not drawing a full url
if(!isset($host["host"])){
$esc = $esc[1];
$payload =
'<div class="url">' .
'<button class="favicon" tabindex="-1">' .
'<img src="/favicon?s=404" alt="xx">' .
'</button>';
}else{
$esc = $esc[0];
}
$esc = substr($esc, 0, 2);
$urlencode = urlencode($link);
$payload =
'<div class="url">' .
'<button class="favicon" tabindex="-1">' .
'<img src="/favicon?s=' . htmlspecialchars($host["scheme"] . "://" . $host["host"]) . '" alt="' . htmlspecialchars($esc) . '">' .
//'<img src="/404.php" alt="' . htmlspecialchars($esc) . '">' .
'</button>' .
'<div class="favicon-dropdown">';
/*
Add archive links
*/
if(
$host["host"] == "boards.4chan.org" ||
$host["host"] == "boards.4channel.org"
){
$esc =
explode(
".",
$host["host"],
2
);
$archives = [];
$path = explode("/", $host["path"]);
$count = count($path);
// /pol/thread/417568063/post-shitty-memes-if-you-want-to
if($count !== 0){
if(
count($esc) === 2 &&
$esc[0] == "www"
){
$isboard = true;
$esc = $esc[1];
}else{
switch($path[1]){
case "con":
break;
case "q":
$archives[] = "desuarchive.org";
break;
case "qa":
$archives[] = "desuarchive.org";
break;
case "qb":
$archives[] = "arch.b4k.co";
break;
case "trash":
$archives[] = "desuarchive.org";
break;
case "a":
$archives[] = "desuarchive.org";
break;
case "c":
$archives[] = "desuarchive.org";
break;
case "w":
break;
case "m":
$archives[] = "desuarchive.org";
break;
case "cgl":
$archives[] = "desuarchive.org";
$archives[] = "warosu.org";
break;
case "f":
$archives[] = "archive.4plebs.org";
break;
case "n":
break;
case "jp":
$archives[] = "warosu.org";
break;
case "vt":
$archives[] = "warosu.org";
break;
case "v":
$archives[] = "arch.b4k.co";
break;
case "vg":
$archives[] = "arch.b4k.co";
break;
case "vm":
$archives[] = "arch.b4k.co";
break;
case "vmg":
$archives[] = "arch.b4k.co";
break;
case "vp":
$archives[] = "arch.b4k.co";
break;
case "vr":
$archives[] = "desuarchive.org";
$archives[] = "warosu.org";
break;
case "vrpg":
$archives[] = "arch.b4k.co";
break;
case "vst":
$archives[] = "arch.b4k.co";
break;
case "co":
$archives[] = "desuarchive.org";
break;
case "g":
$archives[] = "desuarchive.org";
$archives[] = "arch.b4k.co";
break;
case "tv":
$archives[] = "archive.4plebs.org";
break;
case "k":
$archives[] = "desuarchive.org";
break;
case "o":
$archives[] = "archive.4plebs.org";
break;
case "an":
$archives[] = "desuarchive.org";
break;
case "tg":
$archives[] = "desuarchive.org";
$archives[] = "archive.4plebs.org";
break;
case "sp":
$archives[] = "archive.4plebs.org";
break;
case "xs":
$archives[] = "eientei.xyz";
break;
case "pw":
break;
case "sci":
$archives[] = "warosu.org";
$archives[] = "eientei.xyz";
break;
case "his":
$archives[] = "desuarchive.org";
break;
case "int":
$archives[] = "desuarchive.org";
break;
case "out":
break;
case "toy":
break;
case "i":
$archives[] = "archiveofsins.com";
$archives[] = "eientei.xyz";
break;
case "po":
break;
case "p":
break;
case "ck":
$archives[] = "warosu.org";
break;
case "ic":
$archives[] = "warosu.org";
break;
case "wg":
break;
case "lit":
$archives[] = "warosu.org";
break;
case "mu":
$archives[] = "desuarchive.org";
break;
case "fa":
$archives[] = "warosu.org";
break;
case "3":
$archives[] = "warosu.org";
$archives[] = "eientei.xyz";
break;
case "gd":
break;
case "diy":
$archives[] = "warosu.org";
break;
case "wsg":
$archives[] = "desuarchive.org";
break;
case "qst":
break;
case "biz":
$archives[] = "warosu.org";
break;
case "trv":
$archives[] = "archive.4plebs.org";
break;
case "fit":
$archives[] = "desuarchive.org";
break;
case "x":
$archives[] = "archive.4plebs.org";
break;
case "adv":
$archives[] = "archive.4plebs.org";
break;
case "lgbt":
$archives[] = "archiveofsins.com";
break;
case "mlp":
$archives[] = "desuarchive.org";
$archives[] = "arch.b4k.co";
break;
case "news":
break;
case "wsr":
break;
case "vip":
break;
case "b":
$archives[] = "thebarchive.com";
break;
case "r9k":
$archives[] = "desuarchive.org";
break;
case "pol":
$archives[] = "archive.4plebs.org";
break;
case "bant":
$archives[] = "thebarchive.com";
break;
case "soc":
$archives[] = "archiveofsins.com";
break;
case "s4s":
$archives[] = "archive.4plebs.org";
break;
case "s":
$archives[] = "archiveofsins.com";
break;
case "hc":
$archives[] = "archiveofsins.com";
break;
case "hm":
$archives[] = "archiveofsins.com";
break;
case "h":
$archives[] = "archiveofsins.com";
break;
case "e":
break;
case "u":
$archives[] = "archiveofsins.com";
break;
case "d":
$archives[] = "desuarchive.org";
break;
case "t":
$archives[] = "archiveofsins.com";
break;
case "hr":
$archives[] = "archive.4plebs.org";
break;
case "gif":
break;
case "aco":
$archives[] = "desuarchive.org";
break;
case "r":
$archives[] = "archiveofsins.com";
break;
default:
$isboard = false;
break;
}
if($isboard === true){
$archives[] = "archived.moe";
}
$trail = "";
if(
isset($path[2]) &&
isset($path[3]) &&
$path[2] == "thread"
){
$trail .= "/" . $path[1] . "/thread/" . $path[3];
}elseif($isboard){
$trail = "/" . $path[1] . "/";
}
for($i=0; $i<count($archives); $i++){
$payload .=
'<a href="https://' . $archives[$i] . $trail . '" class="list" target="_BLANK">' .
'<img src="/favicon?s=https://' . $archives[$i] . '" alt="' . $archives[$i][0] . $archives[$i][1] . '">' .
$archives[$i] .
'</a>';
}
$esc = $esc[0];
}
$esc = substr($esc, 0, 2);
$urlencode = urlencode($link);
$payload =
'<div class="url">' .
'<button class="favicon" tabindex="-1">' .
'<img src="/favicon?s=' . htmlspecialchars($host["scheme"] . "://" . $host["host"]) . '" alt="' . htmlspecialchars($esc) . '">' .
//'<img src="/404.php" alt="' . htmlspecialchars($esc) . '">' .
'</button>' .
'<div class="favicon-dropdown">';
$payload .=
'<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' .
'<a href="https://archive.ph/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' .
'<a href="https://ghostarchive.org/search?term=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://ghostarchive.org" alt="gh">Ghostarchive</a>' .
'<a href="https://arquivo.pt/wayback/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://arquivo.pt" alt="ar">Arquivo.pt</a>' .
'<a href="https://www.bing.com/search?q=url%3A' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://bing.com" alt="bi">Bing cache</a>' .
'<a href="https://megalodon.jp/?url=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://megalodon.jp" alt="me">Megalodon</a>' .
'</div>';
}
$payload .=
'<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' .
'<a href="https://archive.ph/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' .
'<a href="https://ghostarchive.org/search?term=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://ghostarchive.org" alt="gh">Ghostarchive</a>' .
'<a href="https://arquivo.pt/wayback/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://arquivo.pt" alt="ar">Arquivo.pt</a>' .
'<a href="https://www.bing.com/search?q=url%3A' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://bing.com" alt="bi">Bing cache</a>' .
'<a href="https://megalodon.jp/?url=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://megalodon.jp" alt="me">Megalodon</a>' .
'</div>';
/*
Draw link
*/
@ -862,10 +512,12 @@ class frontend{
}
// merge https://site together
$parts = [
$parts[0] . $parts[1] . '//' . $parts[2],
...array_slice($parts, 3, count($parts) - 1)
];
if(isset($host["host"])){
$parts = [
$parts[0] . $parts[1] . '//' . $parts[2],
...array_slice($parts, 3, count($parts) - 1)
];
}
$c = count($parts);
for($i=0; $i<$c; $i++){
@ -937,11 +589,11 @@ class frontend{
"display" => "Scraper",
"option" => [
"ddg" => "DuckDuckGo",
// "yahoo" => "Yahoo!",
//"yahoo" => "Yahoo!",
"brave" => "Brave",
"yandex" => "Yandex",
// "google" => "Google",
// "google_api" => "Google API",
"google" => "Google",
"google_api" => "Google API",
"google_cse" => "Google CSE",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
@ -980,6 +632,9 @@ class frontend{
"pinterest" => "Pinterest",
"cara" => "Cara",
"flickr" => "Flickr",
"pexels" => "Pexels",
"pixabay" => "Pixabay",
"unsplash" => "Unsplash",
"fivehpx" => "500px",
"vsco" => "VSCO",
"imgur" => "Imgur",
@ -1034,7 +689,8 @@ class frontend{
$filters["scraper"] = [
"display" => "Scraper",
"option" => [
"sc" => "SoundCloud"
"sc" => "SoundCloud",
"swisscows" => "Swisscows (SoundCloud)"
//"spotify" => "Spotify"
]
];
@ -1360,4 +1016,4 @@ class frontend{
return $page . "?" . $query . "&npt=" . $npt;
}
}
}

285
scraper/pexels.php Normal file
View file

@ -0,0 +1,285 @@
<?php
class pexels{
public function __construct(){
include "lib/backend.php";
$this->backend = new backend("pexels");
}
public function getfilters($page){
return [
"time" => [ // date_from
"display" => "Time posted",
"option" => [
"any" => "Any time",
"last_24_hours" => "Last 24 hours",
"last_week" => "Last week",
"last_month" => "Last month",
"last_year" => "Last year"
]
],
"orientation" => [
"display" => "Orientation",
"option" => [
"any" => "Any orientation",
"landscape" => "Horizontal",
"portrait" => "Vertical",
"square" => "Square"
]
],
"color" => [
"display" => "Color",
"option" => [
"any" => "Any color",
"795548" => "Brown",
"F44336" => "Red",
"E91E63" => "Hot pink",
"9C27B0" => "Magenta",
"673AB7" => "Purple",
"3F51B5" => "Indigo",
"2196F3" => "Blue",
"03A9F4" => "Light blue",
"00BCD4" => "Cyan",
"009688" => "Forest green",
"4CAF50" => "Green",
"8BC34A" => "Lime",
"CDDC39" => "Pear",
"FFEB3B" => "Yellow",
"FFC107" => "Gold",
"FF9800" => "Orange",
"FF5722" => "Tomato",
"9E9E9E" => "Gray",
"607D8B" => "Teal",
"000000" => "Black",
"FFFFFF" => "White"
]
],
"people_count" => [
"display" => "Head count",
"option" => [
"any" => "Any number",
"0" => "0",
"1" => "1",
"2" => "2",
"3_plus" => "3+",
]
],
"people_age" => [
"display" => "People's age",
"option" => [
"any" => "Any age",
"baby" => "Baby",
"child" => "Child",
"teenager" => "Teenager",
"adult" => "Adult",
"senior_adult" => "Senior adult"
]
]
];
}
private function get($proxy, $url, $get = []){
$curlproc = curl_init();
$search = $get["query"];
if($get !== []){
$get = http_build_query($get);
$url .= "?" . $get;
}
curl_setopt($curlproc, CURLOPT_URL, $url);
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT,
"Accept: */*",
"Accept-Language: en-US,en;q=0.9",
"Accept-Encoding: gzip, deflate, br, zstd",
"Referer: https://www.pexels.com/search/" . rawurlencode($search) . "/",
"Content-Type: application/json",
"secret-key: H2jk9uKnhRmL6WPwh89zBezWvr", // hardcoded but like, people on github have been using this shit since 23'
"X-Client-Type: react",
"X-Next-Forwarded-CF-Connecting-IP: ",
"X-Next-Forwarded-CF-IPCountry: ",
"X-Next-Forwarded-CF-IPRegionCode: ",
"X-Active-Experiment: ",
"DNT: 1",
"Sec-GPC: 1",
"Alt-Used: www.pexels.com",
"Connection: keep-alive",
"Sec-Fetch-Dest: empty",
"Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-origin",
"Priority: u=4",
"TE: trailers"]
);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
$this->backend->assign_proxy($curlproc, $proxy);
$data = curl_exec($curlproc);
if(curl_errno($curlproc)){
throw new Exception(curl_error($curlproc));
}
curl_close($curlproc);
return $data;
}
public function image($get){
if($get["npt"]){
[$filter, $proxy] =
$this->backend->get(
$get["npt"],
"images"
);
$filter = json_decode($filter, true);
}else{
$search = $get["s"];
if(strlen($search) === 0){
throw new Exception("Search term is empty!");
}
$proxy = $this->backend->get_ip();
// ?query=blue%20footed%20booby&page=1&per_page=24&seo_tags=true
$filter = [
"query" => $search,
"page" => 1,
"per_page" => 24,
"seo_tags" => "true"
];
// add filters
if($get["time"] != "any"){
$filter["date_from"] = $get["time"];
}
if($get["orientation"] != "any"){
$filter["orientation"] = $get["orientation"];
}
if($get["color"] != "any"){
$filter["color"] = $get["color"];
}
if($get["people_count"] != "any"){
$filter["people_count"] = $get["people_count"];
}
if($get["people_age"] != "any"){
$filter["people_age"] = $get["people_age"];
}
}
$out = [
"status" => "ok",
"npt" => null,
"image" => []
];
try{
$html =
$this->get(
$proxy,
"https://www.pexels.com/en-us/api/v3/search/photos",
$filter
);
}catch(Exception $error){
throw new Exception("Failed to fetch JSON");
}
$json = json_decode($html, true);
if(!isset($json["data"])){
throw new Exception("Pexels did not return a data object");
}
foreach($json["data"] as $image){
$thumb_size =
$this->imgratio(
(int)$image["attributes"]["width"],
(int)$image["attributes"]["height"],
350
);
$out["image"][] = [
"title" => $image["attributes"]["title"] . ": " . $image["attributes"]["description"],
"source" => [
[
"url" => $image["attributes"]["image"]["download_link"],
"width" => (int)$image["attributes"]["width"],
"height" => (int)$image["attributes"]["height"]
],
[
"url" =>
preg_replace(
'/(?:w|h)=[0-9]+$/',
"w=350",
$image["attributes"]["image"]["small"]
),
"width" => $thumb_size[0],
"height" => $thumb_size[1]
]
],
"url" =>
"https://pexels.com/photo/" .
$image["attributes"]["slug"] . "-" .
$image["attributes"]["id"] . "/"
];
}
// get next page
if((int)$json["pagination"]["current_page"] < (int)$json["pagination"]["total_pages"]){
$filter["page"]++;
$out["npt"] =
$this->backend->store(
json_encode($filter),
"images",
$proxy
);
}
return $out;
}
private function imgratio($width, $height, $max_width){
$ratio = $max_width / $width;
$new_height = floor($height * $ratio);
return [
$max_width,
$new_height
];
}
}

396
scraper/pixabay.php Normal file
View file

@ -0,0 +1,396 @@
<?php
class pixabay{
public function __construct(){
include "lib/backend.php";
$this->backend = new backend("pixabay");
}
public function getfilters($page){
return [
"nsfw" => [
"display" => "NSFW",
"option" => [
"yes" => "Yes",
"no" => "No"
]
],
"time" => [
"display" => "Time posted",
"option" => [
"any" => "Any time",
"1d" => "< 24 hours",
"3d" => "< 72 hours",
"1w" => "< 7 days",
"6m" => "< 6 months",
"1y" => "< 12 months"
]
],
"category" => [
"display" => "Category",
"option" => [
"images" => "All images",
"photos" => "Photos",
"illustrations" => "Illustrations",
"vectors" => "Vectors",
"gifs" => "GIFs"
]
],
"content_type" => [
"display" => "Authenticity",
"option" => [
"authentic" => "Authentic only",
"show_all" => "Authentic + AI slop",
"ai" => "AI slop only"
]
],
"order" => [
"display" => "Sort by",
"option" => [
"relevance" => "Most relevant",
"latest" => "Latest",
"ec" => "Editor's choice",
"trending" => "Trending"
]
],
"orientation" => [
"display" => "Orientation",
"option" => [
"any" => "Any orientation",
"vertical" => "Vertical",
"horizontal" => "Horizontal"
]
],
"color" => [
"display" => "Color",
"option" => [
"any" => "Any color",
"transparent" => "Transparent",
"grayscale" => "Grayscale",
"red" => "Red",
"orange" => "Orange",
"yellow" => "Yellow",
"green" => "Green",
"turquoise" => "Turquoise",
"blue" => "Blue",
"brown" => "Brown",
"black" => "Black",
"gray" => "Gray",
"white" => "White",
"pink" => "Pink",
"lilac" => "Lilac"
]
]
];
}
private function get($proxy, $url, $get = [], $nsfw = true){
$curlproc = curl_init();
if($get !== []){
$get = http_build_query($get);
$url .= "?" . $get;
}
curl_setopt($curlproc, CURLOPT_URL, $url);
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT . " Pixabay",
"Accept: application/json",
"Accept-Language: en-US,en;q=0.9",
"Accept-Encoding: gzip, deflate, br, zstd",
"Referer: {$url}",
"Cookie: " . ($nsfw ? "g_rated=off" : "g_rated=1"),
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Sec-Fetch-Dest: empty",
"Sec-Fetch-Mode: no-cors",
"Sec-Fetch-Site: same-origin",
"x-fetch-bootstrap: 1",
"Alt-Used: pixabay.com",
"Priority: u=0",
"Pragma: no-cache",
"Cache-Control: no-cache"]
);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
$this->backend->assign_proxy($curlproc, $proxy);
$data = curl_exec($curlproc);
if(curl_errno($curlproc)){
throw new Exception(curl_error($curlproc));
}
curl_close($curlproc);
return $data;
}
public function image($get){
if($get["npt"]){
[$npt, $proxy] =
$this->backend->get(
$get["npt"],
"images"
);
$npt = json_decode($npt, true);
}else{
$search = $get["s"];
if(strlen($search) === 0){
throw new Exception("Search term is empty!");
}
$proxy = $this->backend->get_ip();
$npt = [
"s" => $search,
"pagi" => 1,
"category" => $get["category"]
];
// filters
if($get["content_type"] != "show_all"){
$npt["content_type"] = $get["content_type"];
}
if($get["order"] != "relevance"){
$npt["order"] = $get["order"];
}
$npt["nsfw"] = $get["nsfw"] == "yes" ? true : false;
if($get["orientation"] != "any"){
$npt["orientation"] = $get["orientation"];
}
if($get["color"] != "any"){
$npt["colors"] = $get["color"];
}
if($get["time"] != "any"){
$npt["date"] = $get["time"];
}
}
$out = [
"status" => "ok",
"npt" => null,
"image" => []
];
// https://pixabay.com/images/search/japan/?pagi=1
$npt_pass = $npt;
unset($npt_pass["s"]);
unset($npt_pass["category"]);
unset($npt_pass["nsfw"]);
try{
$json =
$this->get(
$proxy,
"https://pixabay.com/{$npt["category"]}/search/" . rawurlencode($npt["s"]) . "/",
$npt_pass,
$npt["nsfw"]
);
}catch(Exception $error){
throw new Exception("Failed to fetch JSON");
}
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
if(!isset($json["page"]["results"])){
throw new Exception("Pixabay API did not return a results object");
}
//print_r($json);
foreach($json["page"]["results"] as $image){
if(
!(
$image["mediaType"] == "photo" ||
$image["mediaType"] == "animation"
)
){
continue;
}
// handle images that supply canvaRetouchUrl
if(
isset($image["canvaRetouchUrl"]) &&
$image["canvaRetouchUrl"] !== null
){
parse_str(
parse_url(
$image["canvaRetouchUrl"],
PHP_URL_QUERY
),
$base
);
if(!isset($base["image-url"])){
// should not happen
continue;
}
// get extension
$base = $base["image-url"];
$parsed_base = parse_url($base);
$ext =
explode(
".",
$parsed_base["path"]
);
$ext = $ext[count($ext) - 1];
// restructure url
$base =
explode(
"_",
$base
);
unset($base[count($base) - 1]);
$base = implode("_", $base);
$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], 180);
$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], 1920);
$source = [
[
"url" => "{$base}_1920.{$ext}",
"width" => $ratio_2x[0],
"height" => $ratio_2x[1]
],
[
"url" => "{$base}_180.{$ext}",
"width" => $ratio_1x[0],
"height" => $ratio_1x[1]
]
];
}else{
// get intended sizes if canva is not set
// get 1x
$source_1x = null;
if(isset($image["sources"]["image_1x"])){ $source_1x = $image["sources"]["image_1x"]; }
elseif(isset($image["sources"]["1x"])){ $source_1x = $image["sources"]["1x"]; }
if($source_1x === null){ continue; } // should not happen
preg_match(
'/_([0-9]+)\./',
$image["sources"]["1x"],
$size_1x
);
$size_1x = (int)$size_1x[1];
// get 2x
$source_2x = null;
if(isset($image["sources"]["gif_2x"])){ $source_2x = $image["sources"]["gif_2x"]; }
elseif(isset($image["sources"]["2x"])){ $source_2x = $image["sources"]["2x"]; }
if($source_2x === null){ continue; } // should not happen
preg_match(
'/_([0-9]+)\./',
$source_2x,
$size_2x
);
$size_2x = (int)$size_2x[1];
// compute ratios
$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], $size_1x);
$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], $size_2x);
// handle images that only give normal thumbnails
$source = [
[
"url" => $source_2x,
"width" => $ratio_2x[0],
"height" => $ratio_2x[1]
],
[
"url" => $source_1x,
"width" => $ratio_1x[0],
"height" => $ratio_1x[1]
]
];
}
$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], 180);
$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], 1920);
$out["image"][] = [
"title" => $image["name"],
"source" => $source,
"url" => "https://pixabay.com" . $image["href"]
];
}
// add next page
if((int)$json["page"]["page"] < (int)$json["page"]["pages"]){
$npt["pagi"]++;
$out["npt"] =
$this->backend->store(
json_encode($npt),
"images",
$proxy
);
}
return $out;
}
private function imgratio($width, $height, $max_width){
$ratio = $max_width / $width;
$new_height = floor($height * $ratio);
return [
$max_width,
$new_height
];
}
}

267
scraper/unsplash.php Normal file
View file

@ -0,0 +1,267 @@
<?php
class unsplash{
public function __construct(){
include "lib/fuckhtml.php";
$this->fuckhtml = new fuckhtml();
include "lib/backend.php";
$this->backend = new backend("unsplash");
}
public function getfilters($page){
return [
"order_by" => [
"display" => "Order by",
"option" => [
"relevance" => "Relevance",
"latest" => "Newest",
"editorial" => "Curated"
]
],
"orientation" => [
"display" => "Order by",
"option" => [
"any" => "Any orientation",
"landscape" => "Landscape",
"portrait" => "Portrait",
"squarish" => "Square"
]
],
"license" => [
"display" => "License",
"option" => [
"any" => "Any license",
"only" => "Unsplash+",
"none" => "Free"
]
]
];
}
private function get($proxy, $url, $get = [], $referer){
$curlproc = curl_init();
if($get !== []){
$get = http_build_query($get);
$url .= "?" . $get;
}
curl_setopt($curlproc, CURLOPT_URL, $url);
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT,
"Accept: */*",
"Accept-Language: en-US",
"Accept-Encoding: gzip, deflate, br, zstd",
"Referer: {$referer}",
"client-geo-region: global",
"x-client-version: 8999df28be3f138bf2c646df5d656e4dc6970ba0",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Sec-Fetch-Dest: empty",
"Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-origin",
"Priority: u=0",
"TE: trailers"]
);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
$this->backend->assign_proxy($curlproc, $proxy);
$data = curl_exec($curlproc);
if(curl_errno($curlproc)){
throw new Exception(curl_error($curlproc));
}
curl_close($curlproc);
return $data;
}
public function image($get){
if($get["npt"]){
[$filter, $proxy] =
$this->backend->get(
$get["npt"],
"images"
);
$filter = json_decode($filter, true);
}else{
$search = $get["s"];
if(strlen($search) === 0){
throw new Exception("Search term is empty!");
}
$proxy = $this->backend->get_ip();
$filter = [
"page" => 1,
"per_page" => 20,
"query" => $search
];
// add filters
if($get["order_by"] != "relevance"){
$filter["order_by"] = $get["order_by"];
}
if($get["orientation"] != "any"){
$filter["orientation"] = $get["orientation"];
}
if($get["license"] != "any"){
$filter["plus"] = $get["license"];
}
}
$out = [
"status" => "ok",
"npt" => null,
"image" => []
];
// https://unsplash.com/s/photos/shibuya-night?order_by=latest&orientation=landscape&license=free
// https://unsplash.com/s/photos/%3Ctest-%3F!-haha == <test ?! haha
// ?page=1&per_page=20&query=japan
try{
$json =
$this->get(
$proxy,
"https://unsplash.com/napi/search/photos",
$filter,
"https://unsplash.com/s/photos/" . rawurlencode(str_replace(" ", "-", $filter["query"])),
);
}catch(Exception $error){
throw new Exception("Failed to fetch JSON");
}
$json = json_decode($json, true);
if($json === null){
throw new Exception("Could not decode the JSON payload");
}
foreach($json["results"] as $image){
$base = explode("?", $image["urls"]["raw"])[0];
if(
(bool)$image["premium"] ||
(bool)$image["plus"]
){
// when we get "premium" images, give
// 1. highest resolution with watermarks
// 2. highest resolution without watermarks
// (if width of image is above 900px, it has watermarks)
// 3. thumbnail without watermark
$x900 = $this->imgratio((int)$image["width"], (int)$image["height"], 900);
$x500 = $this->imgratio((int)$image["width"], (int)$image["height"], 500);
$source = [
[
"url" => $base,
"width" => (int)$image["width"],
"height" => (int)$image["height"]
],
[
"url" => $base . "?w=900",
"width" => $x900[0],
"height" => $x900[1]
],
[
"url" => $base . "?w=500",
"width" => $x500[0],
"height" => $x500[1]
]
];
}else{
$x500 = $this->imgratio((int)$image["width"], (int)$image["height"], 500);
// image is free as in freedom(tm)
$source = [
[
"url" => $base,
"width" => (int)$image["width"],
"height" => (int)$image["height"]
],
[
"url" => $base . "?w=500",
"width" => $x500[0],
"height" => $x500[1]
]
];
}
$title = [];
$image["description"] = trim($image["description"]);
$image["alt_description"] = trim($image["alt_description"]);
if(!empty($image["description"])){ $title[] = $image["description"]; }
if(!empty($image["alt_description"])){ $title[] = $image["alt_description"]; }
$title = implode(": ", $title);
$out["image"][] = [
"title" => $title,
"source" => $source,
"url" => "https://unsplash.com/photos/" . $image["slug"]
];
}
// next page stuff
if($filter["page"] < (int)$json["total_pages"]){
$filter["page"]++;
$out["npt"] =
$this->backend->store(
json_encode($filter),
"images",
$proxy
);
}
return $out;
}
private function imgratio($width, $height, $max_width){
$ratio = $max_width / $width;
$new_height = floor($height * $ratio);
return [
$max_width,
$new_height
];
}
}

View file

@ -263,6 +263,18 @@ $settings = [
"value" => "flickr",
"text" => "Flickr"
],
[
"value" => "pexels",
"text" => "Pexels"
],
[
"value" => "pixabay",
"text" => "Pixabay"
],
[
"value" => "unsplash",
"text" => "Unsplash"
],
[
"value" => "fivehpx",
"text" => "500px"

View file

@ -0,0 +1,25 @@
:root {
/* background */
--1d2021: #232a2e;
--282828: #2b3337;
--3c3836: #374145;
--504945: #4a585c;
/* font */
--928374: #839e9a;
--a89984: #adc9bc;
--bdae93: #f8f9e8;
--8ec07c: #cbe3b3;
--ebdbb2: #adc9bc;
/* code highlighter */
--comment: #839e9a;
--default: #f8f9e8;
--keyword: #6f8788;
--string: #cbe3b3;
/* color codes for instance list */
--green: #cbe3b3;
--yellow: #f5d098;
--red: #f57f82;
}