mirror of
https://git.bakhai.co.in/FbIN/4Get.git
synced 2025-11-04 12:01:31 +05:30
Synced
greppr fix (319640cd77) fix wikipedia crash (cdf958d293) fix MDN answers not rendering properly (2d63475b07) these comments were too long (ae31274db9) re-added stackoverflow instant answers (20ef5b3e3a) doc config changes number twoo (362cf61508) remove backtick (2ca8fb0006) add more hacks (da1ea1d6e8) this was so much pain to figure out (dea8b0a362) i always forget the fucking config (4215f2678d) fix syntax highlighter (2c2bd28a9f) fix #2 for real this time (7c970031d0) added cara.app (acd02d83d4)
This commit is contained in:
parent
145c1e1388
commit
29e1532be0
7 changed files with 1557 additions and 430 deletions
|
|
@ -143,6 +143,7 @@ class config{
|
|||
const PROXY_CURLIE = false;
|
||||
const PROXY_YT = false; // youtube
|
||||
const PROXY_SEPIASEARCH = false;
|
||||
const PROXY_ODYSEE = false;
|
||||
const PROXY_VIMEO = false;
|
||||
const PROXY_YEP = false;
|
||||
const PROXY_PINTEREST = false;
|
||||
|
|
@ -155,6 +156,7 @@ class config{
|
|||
const PROXY_MWMBL = false;
|
||||
const PROXY_FTM = false; // findthatmeme
|
||||
const PROXY_IMGUR = false;
|
||||
const PROXY_CARA = false;
|
||||
const PROXY_YANDEX_W = false; // yandex web
|
||||
const PROXY_YANDEX_I = false; // yandex images
|
||||
const PROXY_YANDEX_V = false; // yandex videos
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# 4Get configuation options
|
||||
# 4get configuation options
|
||||
|
||||
Welcome! This guide assumes that you have a working 4get instance. This will help you configure your instance to the best it can be!
|
||||
|
||||
|
|
@ -9,37 +9,67 @@ Welcome! This guide assumes that you have a working 4get instance. This will hel
|
|||
4. The captcha font is located in `data/fonts/captcha.ttf`
|
||||
|
||||
# Cloudflare bypass (TLS check)
|
||||
**Note: this only allows you to bypass the browser integrity checks. Captchas & javascript challenges will not be bypassed.**
|
||||
>These instructions have been updated to work with Debian 13 Trixie.
|
||||
|
||||
Configuring this lets you fetch images sitting behind Cloudflare and allows you to scrape the **Yep** & the **Mwmbl** search engines. Please be aware that APT will fight against you and will re-install the openSSL-version of curl constantly when updating.
|
||||
**Note: this only allows you to bypass the browser integrity checks. Captchas & javascript challenges will not be bypassed by this program!**
|
||||
|
||||
First, follow these instructions. Only install the Firefox modules:
|
||||
Configuring this lets you fetch images sitting behind Cloudflare and allows you to scrape the **Yep** search engine.
|
||||
|
||||
https://github.com/lwthiker/curl-impersonate/blob/main/INSTALL.md#native-build
|
||||
|
||||
Once you did this, you should be able to run the following inside your terminal:
|
||||
To come up with this set of instructions, I used [this guide](https://github.com/lwthiker/curl-impersonate/blob/main/INSTALL.md#native-build) as a reference, but trust me you probably want to stick to what's written on this page.
|
||||
|
||||
First, compile curl-impersonate (the firefox flavor).
|
||||
```sh
|
||||
$ curl_ff117 --version
|
||||
curl 8.1.1 (x86_64-pc-linux-gnu) libcurl/8.1.1 NSS/3.92 zlib/1.2.13 brotli/1.0.9 zstd/1.5.4 libidn2/2.3.3 nghttp2/1.56.0
|
||||
Release-Date: 2023-05-23
|
||||
Protocols: dict file ftp ftps gopher gophers http https imap imaps mqtt pop3 pop3s rtsp smb smbs smtp smtps telnet tftp ws wss
|
||||
Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTPS-proxy IDN IPv6 Largefile libz NTLM NTLM_WB SSL threadsafe UnixSockets zstd
|
||||
```
|
||||
Now, after compiling, you should have a `libcurl-impersonate-ff.so` sitting somewhere. Mine (on my debian install) is located at `/usr/local/lib/libcurl-impersonate-ff.so`.
|
||||
|
||||
Find the `libcurl.so.4` file used by your current installation of curl. For me, this file is located at `/usr/lib/x86_64-linux-gnu/libcurl.so.4`
|
||||
|
||||
Now comes the sketchy part: replace `libcurl.so.4` with `libcurl-impersonate-ff.so`. You can do this in the following way:
|
||||
```sh
|
||||
sudo rm /usr/lib/x86_64-linux-gnu/libcurl.so.4
|
||||
sudo cp /usr/local/lib/libcurl-impersonate-ff.so /usr/lib/x86_64-linux-gnu/libcurl.so.4
|
||||
git clone https://github.com/lwthiker/curl-impersonate/
|
||||
cd curl-impersonate
|
||||
sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool python3-pip libnss3 libnss3-dev
|
||||
mkdir build
|
||||
cd build
|
||||
../configure
|
||||
make firefox-build
|
||||
sudo make firefox-install
|
||||
sudo ldconfig
|
||||
```
|
||||
|
||||
Make sure to restart your webserver and/or PHP daemon, otherwise it will keep using the old library. You should now be able to bypass Cloudflare's shitty checks!!
|
||||
Now, after compiling, you should have a `libcurl-impersonate-ff.so` sitting somewhere. Mine is located at `/usr/local/lib/libcurl-impersonate-ff.so`. Do some patch fuckery:
|
||||
|
||||
```sh
|
||||
sudo su
|
||||
LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so
|
||||
CURL_IMPERSONATE=firefox117
|
||||
patchelf --set-soname libcurl.so.4 /usr/local/lib/libcurl-impersonate-ff.so
|
||||
ldconfig
|
||||
```
|
||||
|
||||
From here, you will have a broken curl:
|
||||
```sh
|
||||
root@fuckedmachine:/# curl --version
|
||||
curl: /usr/local/lib/libcurl.so.4: no version information available (required by curl)
|
||||
curl: symbol lookup error: curl: undefined symbol: curl_global_trace, version CURL_OPENSSL_4
|
||||
```
|
||||
|
||||
Or not... During testing, I've seen that sometimes curl still works for some reason. What really matters is the output of this command:
|
||||
```
|
||||
root@fuckedmachine:/# php -r 'print_r(curl_version());' | grep ssl_version
|
||||
[ssl_version_number] => 0
|
||||
[ssl_version] => NSS/3.92
|
||||
```
|
||||
|
||||
It **MUST** say NSS, otherwise it didn't work. There's also the option of using the [forked project](https://github.com/lexiforest/curl-impersonate), but that garbage doesn't support NSS. I'm kind of against impersonating chrome cause you never know when Google is gonna add more fingerprinting bullshit.
|
||||
|
||||
Appendix: If you want a functioning `curl` command line utility again in case it doesn't work anymore, you can do the following hack:
|
||||
|
||||
```
|
||||
sudo apt remove curl
|
||||
sudo ln -s /usr/local/bin/curl-impersonate-ff /usr/bin/curl
|
||||
```
|
||||
|
||||
# Robots.txt
|
||||
Make sure you configure this right to optimize your search engine presence! Head over to `/robots.txt` and change the `4g.flossboxin.org.in` domain to your own domain.
|
||||
Make sure you configure this right to optimize your search engine presence! Head over to `/robots.txt` and change the 4get.ca domain to your own domain.
|
||||
|
||||
# Server listing
|
||||
To be listed on https://4get.ca/instances , you must contact *any* of the people in the server list and ask them to add you to their list of instances in their configuration. The instance list is distributed, and I don't have control over it.
|
||||
|
||||
If you see spammy entries in your instances list, simply remove the instance from your list that pushes the offending entries.
|
||||
|
||||
# Proxies
|
||||
4get supports rotating proxies for scrapers! Configuring one is really easy.
|
||||
|
|
@ -60,4 +90,4 @@ Make sure you configure this right to optimize your search engine presence! Head
|
|||
Done! The scraper you chose should now be using the rotating proxies. When asking for the next page of results, it will use the same proxy to avoid detection!
|
||||
|
||||
## Important!
|
||||
If you ever test out a `socks5` proxy locally on your machine and find out it works but doesn't on your server, try supplying the `socks5_hostname` protocol instead. Hopefully this tip can save you 3 hours of your life!
|
||||
If you ever test out a `socks5` proxy locally on your machine and find out it works but doesn't on your server, try supplying the `socks5_hostname` protocol instead. Hopefully this tip can save you 3 hours of your life!
|
||||
|
|
@ -403,27 +403,28 @@ class frontend{
|
|||
$text =
|
||||
trim(
|
||||
preg_replace(
|
||||
'/<\/span>$/',
|
||||
"", // remove stray ending span because of the <?php stuff
|
||||
'/<code [^>]+>/',
|
||||
"",
|
||||
str_replace(
|
||||
[
|
||||
'<br />',
|
||||
' '
|
||||
"<br />",
|
||||
" ",
|
||||
"<pre>",
|
||||
"</pre>",
|
||||
"</code>"
|
||||
],
|
||||
[
|
||||
"\n", // replace <br> with newlines
|
||||
" " // replace html entity to space
|
||||
],
|
||||
str_replace(
|
||||
[
|
||||
// leading <?php garbage
|
||||
"<span style=\"color: c-default\">\n<?php ",
|
||||
"<code>",
|
||||
"</code>"
|
||||
],
|
||||
"\n",
|
||||
" ",
|
||||
"",
|
||||
highlight_string("<?php " . $text, true)
|
||||
)
|
||||
"",
|
||||
""
|
||||
],
|
||||
explode(
|
||||
"<?php",
|
||||
highlight_string("<?php " . $text, true),
|
||||
2
|
||||
)[1]
|
||||
)
|
||||
)
|
||||
);
|
||||
|
|
@ -951,7 +952,6 @@ class frontend{
|
|||
"mojeek" => "Mojeek",
|
||||
"baidu" => "Baidu",
|
||||
"coccoc" => "Cốc Cốc",
|
||||
//"sepiasearch" => "Sepia Search",
|
||||
//"solofield" => "Solofield",
|
||||
"marginalia" => "Marginalia",
|
||||
"wiby" => "wiby",
|
||||
|
|
@ -975,6 +975,7 @@ class frontend{
|
|||
"baidu" => "Baidu",
|
||||
//"solofield" => "Solofield",
|
||||
"pinterest" => "Pinterest",
|
||||
"cara" => "Cara",
|
||||
"flickr" => "Flickr",
|
||||
"fivehpx" => "500px",
|
||||
"vsco" => "VSCO",
|
||||
|
|
@ -991,6 +992,7 @@ class frontend{
|
|||
"option" => [
|
||||
"yt" => "YouTube",
|
||||
"vimeo" => "Vimeo",
|
||||
//"odysee" => "Odysee",
|
||||
"sepiasearch" => "Sepia Search",
|
||||
//"fb" => "Facebook videos",
|
||||
"ddg" => "DuckDuckGo",
|
||||
|
|
|
|||
847
scraper/cara.php
Normal file
847
scraper/cara.php
Normal file
|
|
@ -0,0 +1,847 @@
|
|||
<?php
|
||||
|
||||
class cara{
|
||||
|
||||
public function __construct(){
|
||||
|
||||
include "lib/backend.php";
|
||||
$this->backend = new backend("cara");
|
||||
}
|
||||
|
||||
public function getfilters($page){
|
||||
|
||||
return [
|
||||
"sort" => [
|
||||
"display" => "Sort by",
|
||||
"option" => [
|
||||
"Top" => "Top",
|
||||
"MostRecent" => "Most Recent"
|
||||
]
|
||||
],
|
||||
"type" => [
|
||||
"display" => "Post type",
|
||||
"option" => [
|
||||
"any" => "Any type",
|
||||
"portfolio" => "Portfolio", // {"posts":["portfolio"]}
|
||||
"timeline" => "Timeline" // {"posts":["timeline"]}
|
||||
]
|
||||
],
|
||||
"fields" => [
|
||||
"display" => "Field/Medium",
|
||||
"option" => [
|
||||
"any" => "Any field",
|
||||
"2D" => "2D Work",
|
||||
"3D" => "3D Work",
|
||||
"3DPrinting" => "3D Printing",
|
||||
"Acrylic" => "Acrylic",
|
||||
"AlcoholMarkers" => "Alcohol Markers",
|
||||
"Animation" => "Animation",
|
||||
"Chalk" => "Chalk",
|
||||
"Charcoal" => "Charcoal",
|
||||
"Colored pencil" => "Colored pencil",
|
||||
"Conte" => "Conte",
|
||||
"Crayon" => "Crayon",
|
||||
"Digital" => "Digital",
|
||||
"Gouache" => "Gouache",
|
||||
"Ink" => "Ink",
|
||||
"MixedMedia" => "Mixed-Media",
|
||||
"Oil" => "Oil",
|
||||
"Oil-based Markers" => "Oil-based Markers",
|
||||
"Other" => "Other",
|
||||
"Pastels" => "Pastels",
|
||||
"Photography" => "Photography",
|
||||
"Sculpture" => "Sculpture",
|
||||
"Sketches" => "Sketches",
|
||||
"Tattoos" => "Tattoos",
|
||||
"Traditional" => "Traditional",
|
||||
"VFX" => "VFX",
|
||||
"Watercolor" => "Watercolor"
|
||||
]
|
||||
],
|
||||
"category" => [
|
||||
"display" => "Category",
|
||||
"option" => [
|
||||
"any" => "Any category",
|
||||
"3DScanning" => "3D Scanning",
|
||||
"Abstract" => "Abstract",
|
||||
"Adoptable" => "Adoptable",
|
||||
"Anatomy" => "Anatomy",
|
||||
"Animals" => "Animals",
|
||||
"Anime" => "Anime",
|
||||
"App" => "App",
|
||||
"ArchitecturalConcepts" => "Architectural Concepts",
|
||||
"ArchitecturalVisualization" => "Architectural Visualization",
|
||||
"AugmentedReality" => "Augmented Reality",
|
||||
"Automotive" => "Automotive",
|
||||
"BoardGameArt" => "Board Game Art",
|
||||
"BookIllustration" => "Book Illustration",
|
||||
"CardGameArt" => "Card Game Art",
|
||||
"CeramicsPottery" => "Ceramics/Pottery",
|
||||
"CharacterAnimation" => "Character Animation",
|
||||
"CharacterDesign" => "Character Design",
|
||||
"CharacterModeling" => "Character Modeling",
|
||||
"ChildrensArt" => "Children's Illustration",
|
||||
"Collectibles" => "Collectibles",
|
||||
"ColoringPage" => "Coloring Page",
|
||||
"ComicArt" => "Comic Art",
|
||||
"ConceptArt" => "Concept Art",
|
||||
"Cosplay" => "Cosplay",
|
||||
"CostumeDesign" => "Costume Design",
|
||||
"CoverArt" => "Cover Art",
|
||||
"Creatures" => "Creatures",
|
||||
"Diorama" => "Diorama",
|
||||
"EditorialIllustration" => "Editorial Illustration",
|
||||
"EmbroiderySewing" => "Embroidery/Sewing",
|
||||
"EnvironmentalConceptArt" => "Environmental Concept Art",
|
||||
"EnvironmentalConceptDesign" => "Environmental Concept Design",
|
||||
"FanArt" => "Fan Art",
|
||||
"Fantasy" => "Fantasy",
|
||||
"Fashion" => "Fashion",
|
||||
"FashionStyling" => "Fashion Styling",
|
||||
"FiberArts" => "Fiber Arts",
|
||||
"Furry" => "Furry",
|
||||
"GameArt" => "Game Art",
|
||||
"GameplayDesign" => "Gameplay Design",
|
||||
"GamesEnvironmentArt" => "Games Environment Art",
|
||||
"Gem" => "Gem",
|
||||
"GraphicDesign" => "Graphic Design",
|
||||
"Handicraft" => "Handicraft",
|
||||
"HairStyling" => "Hair Styling",
|
||||
"HardSurface" => "Hard Surface",
|
||||
"Horror" => "Horror",
|
||||
"Illustration" => "Illustration",
|
||||
"IllustrationVisualization" => "Illustration Visualization",
|
||||
"IndustrialDesign" => "Industrial Design",
|
||||
"Jewelry" => "Jewelry",
|
||||
"KnittingCrochet" => "Knitting/Crochet",
|
||||
"Landscape" => "Landscape",
|
||||
"LevelDesign" => "Level Design",
|
||||
"Lighting" => "Lighting",
|
||||
"Makeup" => "Makeup",
|
||||
"Manga" => "Manga",
|
||||
"MapsCartography" => "Maps/Cartography",
|
||||
"MattePainting" => "Matte Painting",
|
||||
"Materials" => "Materials",
|
||||
"MechanicalDesign" => "Mechanical Design",
|
||||
"Medical" => "Medical",
|
||||
"Mecha" => "Mecha",
|
||||
"MiniatureArt" => "Miniature Art",
|
||||
"MotionGraphics" => "Motion Graphics",
|
||||
"FrescoMurals" => "Fresco/Murals",
|
||||
"Natural" => "Natural",
|
||||
"Original Character" => "Original Character",
|
||||
"Overlay" => "Overlay",
|
||||
"PleinAir" => "Plein Air",
|
||||
"Photogrammetry" => "Photogrammetry",
|
||||
"PixelArt" => "Pixel Art",
|
||||
"Portraits" => "Portraits",
|
||||
"Props" => "Props",
|
||||
"ProductDesign" => "Product Design",
|
||||
"PublicDomain" => "Public Domain or Royalty Free",
|
||||
"Real-Time3DEnvironmentArt" => "Real-Time 3D Environment Art",
|
||||
"Realism" => "Realism",
|
||||
"ScienceFiction" => "Science Fiction",
|
||||
"ScientificVisualization" => "Scientific Visualization",
|
||||
"Scripts" => "Scripts",
|
||||
"StillLife" => "Still Life",
|
||||
"Storyboards" => "Storyboards",
|
||||
"Stylized" => "Stylized",
|
||||
"Surreal" => "Surreal",
|
||||
"TechnicalArt" => "Technical Art",
|
||||
"Textures" => "Textures",
|
||||
"Tools" => "Tools",
|
||||
"Toys" => "Toys",
|
||||
"ToyPackaging" => "Toy Packaging",
|
||||
"Tutorials" => "Tutorials",
|
||||
"UIArt" => "User Interface (UI) Art",
|
||||
"UrbanSketch" => "Urban Sketch",
|
||||
"VFXforAnimation" => "VFX for Animation",
|
||||
"VFXforFilm" => "VFX for Film",
|
||||
"VFXforGames" => "VFX for Games",
|
||||
"VFXforRealTime" => "VFX for Real-Time",
|
||||
"VFXforTV" => "VFX for TV",
|
||||
"Vehicles" => "Vehicles",
|
||||
"VirtualReality" => "Virtual Reality",
|
||||
"VisualDevelopment" => "Visual Development",
|
||||
"VoxelArt" => "Voxel Art",
|
||||
"Vtubers" => "Vtubers",
|
||||
"WIP" => "WIP (Work in Progress)",
|
||||
"Web" => "Web",
|
||||
"Weapons" => "Weapons",
|
||||
"Wildlife" => "Wildlife",
|
||||
"Woodcutting" => "Woodcutting"
|
||||
]
|
||||
],
|
||||
"software" => [
|
||||
"display" => "Software",
|
||||
"option" => [
|
||||
"any" => "Any software",
|
||||
"123D" => "123D",
|
||||
"123DCatch" => "123D Catch",
|
||||
"3DBee" => "3DBee",
|
||||
"3DCoat" => "3DCoat",
|
||||
"3DCoatPrint" => "3DCoatPrint",
|
||||
"3DCoatTextura" => "3DCoatTextura",
|
||||
"3DEqualizer" => "3DEqualizer",
|
||||
"3DFZephyr" => "3DF Zephyr",
|
||||
"3Delight" => "3Delight",
|
||||
"3dpeople" => "3dpeople",
|
||||
"3dsMax" => "3ds Max",
|
||||
"3DSPaint" => "3DS Paint",
|
||||
"ACDSeeCanvas" => "ACDSee Canvas",
|
||||
"AbletonLive" => "Ableton Live",
|
||||
"Acrobat" => "Acrobat",
|
||||
"AdobeDraw" => "Adobe Draw",
|
||||
"AdobeFlash" => "Adobe Flash",
|
||||
"AdobeFresco" => "Adobe Fresco",
|
||||
"AdobeSubstance3Dassets" => "Adobe Substance 3D assets",
|
||||
"AdobeXD" => "Adobe XD",
|
||||
"AffinityDesigner" => "Affinity Designer",
|
||||
"AffinityPhoto" => "Affinity Photo",
|
||||
"AfterEffects" => "After Effects",
|
||||
"Akeytsu" => "Akeytsu",
|
||||
"Alchemy" => "Alchemy",
|
||||
"AliasDesign" => "Alias Design",
|
||||
"AlightMotion" => "Alight Motion",
|
||||
"Amadine" => "Amadine",
|
||||
"Amberlight" => "Amberlight",
|
||||
"Animate" => "Animate",
|
||||
"AnimationMaster" => "Animation:Master",
|
||||
"AnimeStudio" => "Anime Studio",
|
||||
"Apophysis" => "Apophysis",
|
||||
"ArchiCAD" => "ArchiCAD",
|
||||
"Arion" => "Arion",
|
||||
"ArionFX" => "ArionFX",
|
||||
"Arnold" => "Arnold",
|
||||
"ArtEngine" => "ArtEngine",
|
||||
"ArtFlow" => "ArtFlow",
|
||||
"ArtRage" => "ArtRage",
|
||||
"ArtstudioPro" => "Artstudio Pro",
|
||||
"Artweaver" => "Artweaver",
|
||||
"Aseprite" => "Aseprite",
|
||||
"Audition" => "Audition",
|
||||
"AutoCAD" => "AutoCAD",
|
||||
"AutodeskSketchBook" => "Autodesk SketchBook",
|
||||
"AvidMediaComposer" => "Avid Media Composer",
|
||||
"AzPainter" => "AzPainter",
|
||||
"babylonjs" => "babylon.js",
|
||||
"BalsamiqMockup" => "Balsamiq Mockup",
|
||||
"Bforartists" => "Bforartists",
|
||||
"BlackInk" => "Black Ink",
|
||||
"BlackmagicDesignFusion" => "Blackmagic Design Fusion",
|
||||
"Blender" => "Blender",
|
||||
"Blender DeepPaint" => "Blender DeepPaint",
|
||||
"BlenderGreasePencil" => "Blender Grease Pencil",
|
||||
"Blockbench" => "Blockbench",
|
||||
"BodyPaint" => "BodyPaint",
|
||||
"Boxcutter" => "Boxcutter",
|
||||
"BraidMaker" => "Braid Maker",
|
||||
"BrickLinkStudio" => "BrickLink Studio",
|
||||
"Bridge" => "Bridge",
|
||||
"Brushifyio" => "Brushify.io",
|
||||
"C" => "C",
|
||||
"C#" => "C#",
|
||||
"C++" => "C++",
|
||||
"CACANi" => "CACANi",
|
||||
"CLIPSTUDIOPAINT" => "CLIP STUDIO PAINT",
|
||||
"CLO" => "CLO",
|
||||
"CRYENGINE" => "CRYENGINE",
|
||||
"Callipeg" => "Callipeg",
|
||||
"Canva" => "Canva",
|
||||
"CaptureOne" => "Capture One",
|
||||
"CartoonAnimator" => "Cartoon Animator",
|
||||
"Carveco" => "Carveco",
|
||||
"Cavalry" => "Cavalry",
|
||||
"Chaotica" => "Chaotica",
|
||||
"CharacterAnimator" => "Character Animator",
|
||||
"CharacterCreator" => "Character Creator",
|
||||
"Cinema4D" => "Cinema 4D",
|
||||
"ClarisseiFX" => "Clarisse iFX",
|
||||
"Coiffure" => "Coiffure",
|
||||
"ColorsLive" => "Colors Live",
|
||||
"Combustion" => "Combustion",
|
||||
"Construct2" => "Construct 2",
|
||||
"Core" => "Core",
|
||||
"CorelPainter" => "Corel Painter",
|
||||
"CorelDRAWGraphicsSuite" => "CorelDRAW Graphics Suite",
|
||||
"CoronaRenderer" => "Corona Renderer",
|
||||
"ProMotionNG" => "Cosmigo Pro Motion NG",
|
||||
"CrazyBump" => "CrazyBump",
|
||||
"Crocotile3D" => "Crocotile 3D",
|
||||
"Curvy3D" => "Curvy 3D",
|
||||
"Cycles4D" => "Cycles 4D",
|
||||
"Darkroom" => "Darkroom",
|
||||
"DAZStudio" => "DAZ Studio",
|
||||
"DDO" => "DDO",
|
||||
"DECIMA" => "DECIMA",
|
||||
"Darktable" => "Darktable",
|
||||
"DaVinciResolve" => "DaVinci Resolve",
|
||||
"Dimension" => "Dimension",
|
||||
"DragonBones" => "DragonBones",
|
||||
"Dragonframe" => "Dragonframe",
|
||||
"Drawpile" => "Drawpile",
|
||||
"Dreams" => "Dreams",
|
||||
"Dreamweaver" => "Dreamweaver",
|
||||
"DxOPhotoLab" => "DxO PhotoLab",
|
||||
"ECycles" => "E-Cycles",
|
||||
"EmberGen" => "EmberGen",
|
||||
"Encore" => "Encore",
|
||||
"Expresii" => "Expresii",
|
||||
"FStorm" => "FStorm",
|
||||
"FadeIn" => "FadeIn",
|
||||
"Feather3D" => "Feather 3D",
|
||||
"FiberShop" => "FiberShop",
|
||||
"Figma" => "Figma",
|
||||
"FilmoraWondershare" => "Filmora Wondershare",
|
||||
"FilterForge" => "Filter Forge",
|
||||
"FinalCutPro" => "Final Cut Pro",
|
||||
"FinalDraft" => "Final Draft",
|
||||
"finalRender" => "finalRender",
|
||||
"FireAlpaca" => "FireAlpaca",
|
||||
"Fireworks" => "Fireworks",
|
||||
"FlamePainter" => "Flame Painter",
|
||||
"Flash" => "Flash",
|
||||
"FlipaClip" => "FlipaClip",
|
||||
"FlipnoteStudio" => "Flipnote Studio",
|
||||
"Fluent" => "Fluent",
|
||||
"ForestPack" => "Forest Pack",
|
||||
"FormZ" => "Form-Z",
|
||||
"Fractorium" => "Fractorium",
|
||||
"FreeCAD" => "FreeCAD",
|
||||
"FreeHand" => "FreeHand",
|
||||
"Forger" => "Forger",
|
||||
"FrostbiteEngine" => "Frostbite Engine",
|
||||
"fSpy" => "fSpy",
|
||||
"FumeFX" => "FumeFX",
|
||||
"Fusion360" => "Fusion 360",
|
||||
"GIMP" => "GIMP",
|
||||
"GSCurveTools" => "GS CurveTools",
|
||||
"GSToolbox" => "GS Toolbox",
|
||||
"Gaea" => "Gaea",
|
||||
"GameTextures" => "Game Textures",
|
||||
"GameMakerStudio" => "GameMaker: Studio",
|
||||
"GarageFarmNET" => "GarageFarm.NET",
|
||||
"GeoGlyph" => "GeoGlyph",
|
||||
"GigapixelAl" => "Gigapixel Al",
|
||||
"Glaxnimate" => "Glaxnimate",
|
||||
"GnomePaint" => "Gnome Paint",
|
||||
"Godot" => "Godot",
|
||||
"Goxel" => "Goxel",
|
||||
"Graphite" => "Graphite",
|
||||
"Graswald" => "Graswald",
|
||||
"GravitySketch" => "Gravity Sketch",
|
||||
"GuerillaRender" => "GuerillaRender",
|
||||
"HDRLightStudio" => "HDR Light Studio",
|
||||
"HairStrandDesigner" => "Hair Strand Designer",
|
||||
"HairTGHairFur" => "HairTG - Hair & Fur",
|
||||
"HairTGSurfaceFeatherEdition" => "HairTG - Surface, Feather Edition",
|
||||
"HairTGSurfaceHairEdition" => "HairTG - Surface, Hair Edition",
|
||||
"Handplane" => "Handplane",
|
||||
"Hansoft" => "Hansoft",
|
||||
"HardOps" => "Hard Ops",
|
||||
"HardMesh" => "HardMesh",
|
||||
"Harmony" => "Harmony",
|
||||
"HeavypaintWebbypaint" => "Heavypaint/Webbypaint",
|
||||
"HelloPaint" => "HelloPaint",
|
||||
"HeliconFocus" => "Helicon Focus",
|
||||
"Hexels" => "Hexels",
|
||||
"HiPaint" => "HiPaint",
|
||||
"Houdini" => "Houdini",
|
||||
"HydraRenderer" => "Hydra Renderer",
|
||||
"iArtbook" => "iArtbook",
|
||||
"IbisPaint" => "ibisPaint",
|
||||
"Ideas" => "Ideas",
|
||||
"IllustStudio" => "Illust Studio",
|
||||
"Illustrator" => "Illustrator",
|
||||
"IllustratorDraw" => "Illustrator Draw",
|
||||
"InDesign" => "InDesign",
|
||||
"Inochi2D" => "Inochi2D",
|
||||
"InVision" => "InVision",
|
||||
"InVisionCraft" => "InVision Craft",
|
||||
"InfinitePainter" => "Infinite Painter",
|
||||
"Inkscape" => "Inkscape",
|
||||
"Inspirit" => "Inspirit",
|
||||
"InstaLOD" => "InstaLOD",
|
||||
"InstaMAT" => "InstaMAT",
|
||||
"InstantLightRealtimePBR" => "Instant Light Realtime PBR",
|
||||
"InstantMeshes" => "Instant Meshes",
|
||||
"InstantTerra" => "Instant Terra",
|
||||
"Inventor" => "Inventor",
|
||||
"Iray" => "Iray",
|
||||
"JWildfire" => "JWildfire",
|
||||
"Java" => "Java",
|
||||
"Jira" => "Jira",
|
||||
"JumpPaint" => "Jump Paint by MediBang",
|
||||
"JSPaint" => "JS Paint",
|
||||
"Katana" => "Katana",
|
||||
"Keyshot" => "Keyshot",
|
||||
"KidPix" => "Kid Pix",
|
||||
"KitBash3D" => "KitBash3D",
|
||||
"Knald" => "Knald",
|
||||
"Kodon" => "Kodon",
|
||||
"KolourPaint" => "KolourPaint",
|
||||
"Krakatoa" => "Krakatoa",
|
||||
"KRESKA" => "KRESKA",
|
||||
"Krita" => "Krita",
|
||||
"LensStudio" => "Lens Studio",
|
||||
"LibreSprite" => "LibreSprite",
|
||||
"LightWave3D" => "LightWave 3D",
|
||||
"Lightroom" => "Lightroom",
|
||||
"Linearity" => "Linearity",
|
||||
"LiquiGen" => "LiquiGen",
|
||||
"Live2DCubism" => "Live2D Cubism",
|
||||
"LookatmyHair" => "Look at my Hair",
|
||||
"Lotpixel" => "Lotpixel",
|
||||
"Lumion" => "Lumion",
|
||||
"LuxRender" => "LuxRender",
|
||||
"MacPaint" => "MacPaint",
|
||||
"MagicaCSG" => "MagicaCSG",
|
||||
"MagicaVoxel" => "MagicaVoxel",
|
||||
"Magma" => "Magma",
|
||||
"MakeHuman" => "MakeHuman",
|
||||
"Malmal" => "Malmal",
|
||||
"Mandelbulb3D" => "Mandelbulb 3D",
|
||||
"Mandelbulber" => "Mandelbulber",
|
||||
"MangaStudio" => "Manga Studio",
|
||||
"Mari" => "Mari",
|
||||
"MarmosetToolbag" => "Marmoset Toolbag",
|
||||
"MarvelousDesigner" => "Marvelous Designer",
|
||||
"MasterpieceStudioPro" => "Masterpiece Studio Pro",
|
||||
"MasterpieceVR" => "MasterpieceVR",
|
||||
"Maverick" => "Maverick",
|
||||
"MaxwellRender" => "Maxwell Render",
|
||||
"Maya" => "Maya",
|
||||
"MediBangPaint" => "MediBang Paint",
|
||||
"MediumbyAdobe" => "Medium by Adobe",
|
||||
"Megascans" => "Megascans",
|
||||
"mentalray" => "mental ray",
|
||||
"MeshLab" => "MeshLab",
|
||||
"Meshroom" => "Meshroom",
|
||||
"MetaHumanCreator" => "MetaHuman Creator",
|
||||
"Metashape" => "Metashape",
|
||||
"MightyBake" => "MightyBake",
|
||||
"MikuMikuDance" => "MikuMikuDance",
|
||||
"Minecraft" => "Minecraft",
|
||||
"Mischief" => "Mischief",
|
||||
"Mixamo" => "Mixamo",
|
||||
"Mixer" => "Mixer",
|
||||
"MoI3D" => "MoI3D",
|
||||
"Mocha" => "Mocha",
|
||||
"Modo" => "Modo",
|
||||
"Moho" => "Moho",
|
||||
"MotionBuilder" => "MotionBuilder",
|
||||
"Mudbox" => "Mudbox",
|
||||
"Muse" => "Muse",
|
||||
"MSPaint" => "MS Paint",
|
||||
"MyPaint" => "MyPaint",
|
||||
"NDO" => "NDO",
|
||||
"NX" => "NX",
|
||||
"NdotCAD" => "NdotCAD",
|
||||
"NintendoNotes" => "Nintendo Notes",
|
||||
"NomadSculpt" => "Nomad Sculpt",
|
||||
"Notability" => "Notability",
|
||||
"Nuke" => "Nuke",
|
||||
"Nvil" => "Nvil",
|
||||
"OctaneRender" => "Octane Render",
|
||||
"Omniverse" => "Omniverse",
|
||||
"OmniverseCreate" => "Omniverse Create",
|
||||
"ON1PhotoRAW" => "ON1 Photo RAW",
|
||||
"Open3DEngine" => "Open 3D Engine",
|
||||
"OpenCanvas" => "OpenCanvas",
|
||||
"OpenGL" => "OpenGL",
|
||||
"OpenToonz" => "OpenToonz",
|
||||
"Ornatrix" => "Ornatrix",
|
||||
"OsciRender" => "Osci-Render",
|
||||
"OurPaint" => "Our Paint",
|
||||
"PBRMAX" => "PBRMAX",
|
||||
"PFTrack" => "PFTrack",
|
||||
"PTGui" => "PTGui",
|
||||
"Paintbrush" => "Paintbrush",
|
||||
"PaintNET" => "Paint.NET",
|
||||
"PaintShopPro" => "PaintShop Pro",
|
||||
"PaintToolSAI" => "Paint Tool SAI",
|
||||
"PaintstormStudio" => "Paintstorm Studio",
|
||||
"Paper" => "Paper",
|
||||
"Pencil2D" => "Pencil2D",
|
||||
"Penpot" => "Penpot",
|
||||
"PhoenixFD" => "Phoenix FD",
|
||||
"Phonto" => "Phonto",
|
||||
"PhotoLab2" => "PhotoLab 2",
|
||||
"Photopea" => "Photopea",
|
||||
"Photoscan" => "Photoscan",
|
||||
"Photoshop" => "Photoshop",
|
||||
"PhotoshopElements" => "Photoshop Elements",
|
||||
"PicoCAD" => "picoCAD",
|
||||
"PicoCAD2" => "picoCAD 2",
|
||||
"Pinta" => "Pinta",
|
||||
"Piskel" => "Piskel",
|
||||
"Pixilart" => "Pixilart",
|
||||
"Pixelitor" => "Pixelitor",
|
||||
"Pixelmator" => "Pixelmator",
|
||||
"Pixelorama" => "Pixelorama",
|
||||
"PixivSketch" => "pixiv Sketch",
|
||||
"Pixquare" => "Pixquare",
|
||||
"PlantCatalog" => "PlantCatalog",
|
||||
"PlantFactory" => "PlantFactory",
|
||||
"Plasticity" => "Plasticity",
|
||||
"PNGtuberPlus" => "PNGtuber Plus",
|
||||
"Poliigon" => "Poliigon",
|
||||
"Polybrush" => "Polybrush",
|
||||
"PopcornFx" => "PopcornFx",
|
||||
"Poser" => "Poser",
|
||||
"Premiere" => "Premiere",
|
||||
"PremiereElements" => "Premiere Elements",
|
||||
"PresagisCreator" => "Presagis Creator",
|
||||
"ProTools" => "Pro Tools",
|
||||
"Procreate" => "Procreate",
|
||||
"ProcreateDreams" => "Procreate Dreams",
|
||||
"Producer" => "Producer",
|
||||
"PrometheanAI" => "Promethean AI",
|
||||
"PureRef" => "PureRef",
|
||||
"Python" => "Python",
|
||||
"PyxelEdit" => "PyxelEdit",
|
||||
"QuadRemesher" => "Quad Remesher",
|
||||
"QuarkXPress" => "QuarkXPress",
|
||||
"Qubicle" => "Qubicle",
|
||||
"Quill" => "Quill",
|
||||
"QuixelBridge" => "Quixel Bridge",
|
||||
"QuixelMegascans" => "Quixel Megascans",
|
||||
"QuixelMixer" => "Quixel Mixer",
|
||||
"QuixelSuite" => "Quixel Suite",
|
||||
"R3DSWrap" => "R3DS Wrap",
|
||||
"R3DSZWRAP" => "R3DS ZWRAP",
|
||||
"RDTextures" => "RD-Textures",
|
||||
"RailClone" => "RailClone",
|
||||
"RealFlow" => "RealFlow",
|
||||
"RealisticPaintStudio" => "Realistic Paint Studio",
|
||||
"RealityCapture" => "RealityCapture",
|
||||
"RealityScan" => "RealityScan",
|
||||
"RealtimeBoard" => "Realtime Board",
|
||||
"Rebelle" => "Rebelle",
|
||||
"Redshift" => "Redshift",
|
||||
"RenderMan" => "RenderMan",
|
||||
"RenderNetwork" => "Render Network",
|
||||
"Revit" => "Revit",
|
||||
"Rhino" => "Rhino",
|
||||
"Rhinoceros" => "Rhinoceros",
|
||||
"RizomUV" => "RizomUV",
|
||||
"RoughAnimator" => "Rough Animator",
|
||||
"SamsungNotes" => "Samsung Notes",
|
||||
"SamsungPENUP" => "Samsung PENUP",
|
||||
"ScansLibrary" => "ScansLibrary",
|
||||
"Scrivener" => "Scrivener",
|
||||
"Sculpt+" => "Sculpt+",
|
||||
"Sculptris" => "Sculptris",
|
||||
"ShaveandaHaircut" => "Shave and a Haircut",
|
||||
"ShiVa3D" => "ShiVa3D",
|
||||
"Shotgun" => "Shotgun",
|
||||
"Silo" => "Silo",
|
||||
"Silugen" => "Silugen",
|
||||
"Sketch" => "Sketch",
|
||||
"SketchApp" => "Sketch App",
|
||||
"SketchBookPro" => "SketchBook Pro",
|
||||
"SketchClub" => "SketchClub",
|
||||
"SketchUp" => "SketchUp",
|
||||
"Sketchable" => "Sketchable",
|
||||
"Sketchfab" => "Sketchfab",
|
||||
"Skyshop" => "Skyshop",
|
||||
"Snapseed" => "Snapseed",
|
||||
"Snowdrop" => "Snowdrop",
|
||||
"Softimage" => "Softimage",
|
||||
"SolidWorks" => "SolidWorks",
|
||||
"SonySketch" => "Sony Sketch",
|
||||
"Soundbooth" => "Soundbooth",
|
||||
"Source2" => "Source 2",
|
||||
"SourceControl" => "Source Control",
|
||||
"SourceFilmmaker" => "Source Filmmaker",
|
||||
"SpeedTree" => "SpeedTree",
|
||||
"Speedgrade" => "Speedgrade",
|
||||
"SpeedyPainter" => "SpeedyPainter",
|
||||
"Spine2D" => "Spine 2D",
|
||||
"Spriter" => "Spriter",
|
||||
"Stingray" => "Stingray",
|
||||
"Storyboarder" => "Storyboarder",
|
||||
"StoryboardPro" => "Storyboard Pro",
|
||||
"SublimeText" => "Sublime Text",
|
||||
"Substance3DDesigner" => "Substance 3D Designer",
|
||||
"Substance3DModeler" => "Substance 3D Modeler",
|
||||
"Substance3DPainter" => "Substance 3D Painter",
|
||||
"Substance3DSampler" => "Substance 3D Sampler",
|
||||
"Substance3DStager" => "Substance 3D Stager",
|
||||
"SubstanceB2M" => "Substance B2M",
|
||||
"SweetHome3D" => "Sweet Home 3D",
|
||||
"SynthEyes" => "SynthEyes",
|
||||
"TTools" => "TTools",
|
||||
"TVPaint" => "TVPaint",
|
||||
"TVPaintAnimation" => "TVPaint Animation",
|
||||
"TayasuiSketches" => "Tayasui Sketches",
|
||||
"TayasuiSketchesMobileApp" => "Tayasui Sketches Mobile App",
|
||||
"TayasuiSketchesPro" => "Tayasui Sketches Pro",
|
||||
"Terragen" => "Terragen",
|
||||
"Texturescom" => "Textures.com",
|
||||
"Texturingxyz" => "Texturingxyz",
|
||||
"TeyaConceptor" => "Teya Conceptor",
|
||||
"TheGrove3D" => "The Grove 3D",
|
||||
"TheaRender" => "Thea Render",
|
||||
"Threejs" => "Three.js",
|
||||
"Tiled" => "Tiled",
|
||||
"TiltBrush" => "Tilt Brush",
|
||||
"Tooll3" => "Tooll3",
|
||||
"ToonBoomHarmony" => "Toon Boom Harmony",
|
||||
"ToonBoomStudio" => "Toon Boom Studio",
|
||||
"ToonSquid" => "ToonSquid",
|
||||
"TopoGun" => "TopoGun",
|
||||
"TuxPaint" => "Tux Paint",
|
||||
"Tvori" => "Tvori",
|
||||
"Twinmotion" => "Twinmotion",
|
||||
"UNIGINEEngine" => "UNIGINE Engine",
|
||||
"UVLayout" => "UVLayout",
|
||||
"UltraFractal" => "Ultra Fractal",
|
||||
"uMake" => "uMake",
|
||||
"Unfold3D" => "Unfold 3D",
|
||||
"Unity" => "Unity",
|
||||
"UnrealEngine" => "Unreal Engine",
|
||||
"Vengi" => "vengi",
|
||||
"VRay" => "V-Ray",
|
||||
"VRED" => "VRED",
|
||||
"VTubeStudio" => "VTube Studio",
|
||||
"Vectary" => "Vectary",
|
||||
"VectorayGen" => "VectorayGen",
|
||||
"Vectorworks" => "Vectorworks",
|
||||
"VegasPro" => "Vegas Pro",
|
||||
"VisualDesigner3D" => "Visual Designer 3D",
|
||||
"VisualStudio" => "Visual Studio",
|
||||
"VRoidStudio" => "VRoid Studio",
|
||||
"Vue" => "Vue",
|
||||
"Vuforia" => "Vuforia",
|
||||
"WebGL" => "WebGL",
|
||||
"WhiteboardFox" => "Whiteboard Fox",
|
||||
"WickEditor" => "Wick Editor",
|
||||
"Wings3D" => "Wings 3D",
|
||||
"Word" => "Word",
|
||||
"WorldCreator" => "World Creator",
|
||||
"WorldMachine" => "World Machine",
|
||||
"XParticles" => "X-Particles",
|
||||
"Xfrog" => "Xfrog",
|
||||
"Xgen" => "Xgen",
|
||||
"xNormal" => "xNormal",
|
||||
"xTex" => "xTex",
|
||||
"XoliulShader" => "Xoliul Shader",
|
||||
"Yafaray" => "Yafaray",
|
||||
"Yeti" => "Yeti",
|
||||
"ZBrush" => "ZBrush",
|
||||
"ZBrushCore" => "ZBrushCore",
|
||||
"ZenBrush" => "Zen Brush"
|
||||
]
|
||||
]
|
||||
];
|
||||
}
|
||||
|
||||
private function get($proxy, $url, $get = [], $search){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
if($get !== []){
|
||||
$get = http_build_query($get);
|
||||
$url .= "?" . $get;
|
||||
}
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, $url);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: application/json, text/plain, */*",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
//"sentry-trace: 72b0318a7141fe18cbacbd905572eddf-a60de161b66b1e6f-1
|
||||
//"baggage: sentry-environment=vercel-production,sentry-release=251ff5179b4de94974f36d9b8659a487bbb8a819,sentry-public_key=2b87af2b44c84643a011838ad097735f,sentry-trace_id=72b0318a7141fe18cbacbd905572eddf,sentry-transaction=GET%20%2Fsearch,sentry-sampled=true,sentry-sample_rand=0.09967130764937493,sentry-sample_rate=0.5",
|
||||
"DNT: 1",
|
||||
"Sec-GPC: 1",
|
||||
"Connection: keep-alive",
|
||||
//"Referer: https://cara.app/search?q=jak+and+daxter&type=&sortBy=Top&filters=%7B%7D",
|
||||
"Referer: https://cara.app/search?q=" . urlencode($search),
|
||||
//"Cookie: __Host-next-auth.csrf-token=b752c4296375bccb7b480ff010e1e916c65c35c311a4a57ac6cd871468730578%7C4d3783cfb72a98f390e534abd149806432b6cf8d50555a52d00e99216a516911; __Secure-next-auth.callback-url=https%3A%2F%2Fcara.app; crumb=BV0HDt87G5+fOWE0ZDQ5MWM0ZTQ3YTZmMzM4MGU5MGNjNDNmMzY2",
|
||||
"Sec-Fetch-Dest: empty",
|
||||
"Sec-Fetch-Mode: cors",
|
||||
"Sec-Fetch-Site: same-origin",
|
||||
"TE: trailers"]
|
||||
);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
$this->backend->assign_proxy($curlproc, $proxy);
|
||||
|
||||
$data = curl_exec($curlproc);
|
||||
|
||||
if(curl_errno($curlproc)){
|
||||
|
||||
throw new Exception(curl_error($curlproc));
|
||||
}
|
||||
|
||||
curl_close($curlproc);
|
||||
return $data;
|
||||
}
|
||||
|
||||
public function image($get){
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
[$npt, $proxy] =
|
||||
$this->backend->get(
|
||||
$get["npt"],
|
||||
"images"
|
||||
);
|
||||
|
||||
$npt = json_decode($npt, true);
|
||||
}else{
|
||||
|
||||
$search = $get["s"];
|
||||
if(strlen($search) === 0){
|
||||
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$npt = [
|
||||
"q" => $get["s"],
|
||||
"sortBy" => $get["sort"],
|
||||
"take" => 24,
|
||||
"skip" => 0,
|
||||
"filters" => []
|
||||
];
|
||||
|
||||
// parse filters
|
||||
if($get["type"] != "any"){
|
||||
|
||||
$npt["filters"]["posts"] = [$get["type"]];
|
||||
}
|
||||
|
||||
if($get["fields"] != "any"){
|
||||
|
||||
$npt["filters"]["fields"] = [$get["fields"]];
|
||||
}
|
||||
|
||||
if($get["category"] != "any"){
|
||||
|
||||
$npt["filters"]["categories"] = [$get["category"]];
|
||||
}
|
||||
|
||||
if($get["software"] != "any"){
|
||||
|
||||
$npt["filters"]["softwares"] = [$get["software"]];
|
||||
}
|
||||
|
||||
if($npt["filters"] == []){
|
||||
|
||||
$npt["filters"] = "{}";
|
||||
}else{
|
||||
|
||||
$npt["filters"] = json_encode($npt["filters"]);
|
||||
}
|
||||
}
|
||||
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"npt" => null,
|
||||
"image" => []
|
||||
];
|
||||
|
||||
// https://cara.app/api/search/portfolio-posts?q=jak+and+daxter&sortBy=Top&take=24&skip=0&filters=%7B%7D
|
||||
try{
|
||||
$json =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://cara.app/api/search/posts",
|
||||
$npt,
|
||||
$npt["q"]
|
||||
);
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch JSON");
|
||||
}
|
||||
|
||||
$json = json_decode($json, true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON");
|
||||
}
|
||||
|
||||
$imagecount = 0;
|
||||
foreach($json as $image){
|
||||
|
||||
if(count($image["images"]) === 0){
|
||||
|
||||
// sometimes the api returns no images for an object
|
||||
$imagecount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
$cover = null;
|
||||
$sources = [];
|
||||
|
||||
foreach($image["images"] as $source){
|
||||
|
||||
if($source["isCoverImg"]){
|
||||
|
||||
$cover = [
|
||||
"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
|
||||
"width" => 500,
|
||||
"height" => 500
|
||||
];
|
||||
}else{
|
||||
|
||||
$sources[] = [
|
||||
"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
|
||||
"width" => null,
|
||||
"height" => null
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
if($cover !== null){
|
||||
|
||||
$sources[] = $cover;
|
||||
}
|
||||
|
||||
$out["image"][] = [
|
||||
"title" => str_replace("\n", " ", $image["content"]),
|
||||
"source" => $sources,
|
||||
"url" => "https://cara.app/post/" . $image["id"]
|
||||
];
|
||||
|
||||
$imagecount++;
|
||||
}
|
||||
|
||||
if($imagecount === 24){
|
||||
|
||||
$npt["skip"] += 24;
|
||||
|
||||
$out["npt"] =
|
||||
$this->backend->store(
|
||||
json_encode($npt),
|
||||
"images",
|
||||
$proxy
|
||||
);
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
private function fix_url($url){
|
||||
|
||||
return
|
||||
str_replace(
|
||||
[" "],
|
||||
["%20"],
|
||||
$url
|
||||
);
|
||||
}
|
||||
}
|
||||
680
scraper/ddg.php
680
scraper/ddg.php
|
|
@ -1046,20 +1046,38 @@ class ddg{
|
|||
|
||||
if(isset($json["Abstract"])){
|
||||
|
||||
$description[] =
|
||||
[
|
||||
"type" => "text",
|
||||
"value" => $json["Abstract"]
|
||||
];
|
||||
$description = $this->parse_rich_text($json["Abstract"]);
|
||||
}
|
||||
|
||||
if(
|
||||
!isset($json["Image"]) ||
|
||||
$json["Image"] == "" ||
|
||||
$json["Image"] === null ||
|
||||
$json["Image"] == "https://duckduckgo.com/i/"
|
||||
){
|
||||
|
||||
$image = null;
|
||||
}else{
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/^https?:\/\//',
|
||||
$json["Image"]
|
||||
)
|
||||
){
|
||||
|
||||
$image = $json["Image"];
|
||||
}else{
|
||||
|
||||
$image = "https://duckduckgo.com" . $json["Image"];
|
||||
}
|
||||
}
|
||||
|
||||
$out["answer"][] = [
|
||||
"title" => $json["Heading"],
|
||||
"description" => $description,
|
||||
"url" => $json["AbstractURL"],
|
||||
"thumb" =>
|
||||
(!isset($json["Image"]) || $json["Image"] == "" || $json["Image"] === null) ?
|
||||
null : "https://duckduckgo.com" . $json["Image"],
|
||||
"thumb" => $image,
|
||||
"table" => $table,
|
||||
"sublink" => $sublinks
|
||||
];
|
||||
|
|
@ -1072,11 +1090,11 @@ class ddg{
|
|||
}
|
||||
|
||||
//
|
||||
// Get wordnik definition
|
||||
// Parse additional data endpoints
|
||||
//
|
||||
//nrj('/js/spice/dictionary/definition/create', null, null, null, null, 'dictionary_definition');
|
||||
|
||||
preg_match(
|
||||
preg_match_all(
|
||||
'/nrj\(\s*\'([^\']+)\'/',
|
||||
$js,
|
||||
$nrj
|
||||
|
|
@ -1084,234 +1102,318 @@ class ddg{
|
|||
|
||||
if(isset($nrj[1])){
|
||||
|
||||
$nrj = $nrj[1];
|
||||
|
||||
preg_match(
|
||||
'/\/js\/spice\/dictionary\/definition\/([^\/]+)/',
|
||||
$nrj,
|
||||
$word
|
||||
);
|
||||
|
||||
if(isset($word[1])){
|
||||
foreach($nrj[1] as $potential_endpoint){
|
||||
|
||||
$word = $word[1];
|
||||
//
|
||||
// Probe for wordnik definition
|
||||
//
|
||||
preg_match(
|
||||
'/\/js\/spice\/dictionary\/definition\/([^\/]+)/',
|
||||
$potential_endpoint,
|
||||
$word
|
||||
);
|
||||
|
||||
// found wordnik definition & word
|
||||
try{
|
||||
$nik =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://duckduckgo.com/js/spice/dictionary/definition/" . $word,
|
||||
[],
|
||||
ddg::req_xhr
|
||||
if(isset($word[1])){
|
||||
|
||||
$word = $word[1];
|
||||
|
||||
// found wordnik definition & word
|
||||
try{
|
||||
$nik =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://duckduckgo.com/js/spice/dictionary/definition/" . $word,
|
||||
[],
|
||||
ddg::req_xhr
|
||||
);
|
||||
|
||||
}catch(Exception $e){
|
||||
|
||||
// fail gracefully
|
||||
return $out;
|
||||
}
|
||||
|
||||
// remove javascript
|
||||
$js_tmp =
|
||||
preg_split(
|
||||
'/ddg_spice_dictionary_definition\(\s*/',
|
||||
$nik,
|
||||
2
|
||||
);
|
||||
|
||||
}catch(Exception $e){
|
||||
|
||||
// fail gracefully
|
||||
return $out;
|
||||
}
|
||||
|
||||
// remove javascript
|
||||
$js_tmp =
|
||||
preg_split(
|
||||
'/ddg_spice_dictionary_definition\(\s*/',
|
||||
$nik,
|
||||
2
|
||||
);
|
||||
|
||||
if(count($js_tmp) > 1){
|
||||
|
||||
$nik =
|
||||
json_decode(
|
||||
$this->fuckhtml
|
||||
->extract_json(
|
||||
$js_tmp[1]
|
||||
),
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
if($nik === null){
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
$answer_cat = [];
|
||||
$answer = [];
|
||||
|
||||
foreach($nik as $snippet){
|
||||
|
||||
if(!isset($snippet["partOfSpeech"])){ continue; }
|
||||
|
||||
$push = [];
|
||||
|
||||
// add text snippet
|
||||
if(isset($snippet["text"])){
|
||||
if(count($js_tmp) > 1){
|
||||
|
||||
$push[] = [
|
||||
"type" => "text",
|
||||
"value" =>
|
||||
$nik =
|
||||
json_decode(
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$snippet["text"]
|
||||
)
|
||||
];
|
||||
}
|
||||
|
||||
// add example uses
|
||||
if(isset($snippet["exampleUses"])){
|
||||
|
||||
foreach($snippet["exampleUses"] as $example){
|
||||
|
||||
$push[] = [
|
||||
"type" => "quote",
|
||||
"value" => "\"" .
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$example["text"]
|
||||
) . "\""
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// add citations
|
||||
if(isset($snippet["citations"])){
|
||||
|
||||
foreach($snippet["citations"] as $citation){
|
||||
|
||||
if(!isset($citation["cite"])){ continue; }
|
||||
|
||||
$text =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$citation["cite"]
|
||||
);
|
||||
|
||||
if(isset($citation["source"])){
|
||||
|
||||
$text .=
|
||||
" - " .
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$citation["source"]
|
||||
);
|
||||
}
|
||||
|
||||
$push[] = [
|
||||
"type" => "quote",
|
||||
"value" => $text
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// add related words
|
||||
if(isset($snippet["relatedWords"])){
|
||||
|
||||
$relations = [];
|
||||
|
||||
foreach($snippet["relatedWords"] as $related){
|
||||
|
||||
$words = [];
|
||||
foreach($related["words"] as $wrd){
|
||||
|
||||
$words[] =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$wrd
|
||||
);
|
||||
}
|
||||
|
||||
if(
|
||||
count($words) !== 0 &&
|
||||
isset($related["relationshipType"])
|
||||
){
|
||||
|
||||
$relations[ucfirst($related["relationshipType"]) . "s"] =
|
||||
implode(", ", $words);
|
||||
}
|
||||
}
|
||||
|
||||
foreach($relations as $relation_title => $relation_content){
|
||||
|
||||
$push[] = [
|
||||
"type" => "quote",
|
||||
"value" => $relation_title . ": " . $relation_content
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(count($push) !== 0){
|
||||
|
||||
// push data to answer_cat
|
||||
if(!isset($answer_cat[$snippet["partOfSpeech"]])){
|
||||
|
||||
$answer_cat[$snippet["partOfSpeech"]] = [];
|
||||
}
|
||||
|
||||
$answer_cat[$snippet["partOfSpeech"]] =
|
||||
array_merge(
|
||||
$answer_cat[$snippet["partOfSpeech"]],
|
||||
$push
|
||||
->extract_json(
|
||||
$js_tmp[1]
|
||||
),
|
||||
true
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
foreach($answer_cat as $answer_title => $answer_content){
|
||||
|
||||
$i = 0;
|
||||
$answer[] = [
|
||||
"type" => "title",
|
||||
"value" => $answer_title
|
||||
];
|
||||
|
||||
$old_type = $answer[count($answer) - 1]["type"];
|
||||
|
||||
foreach($answer_content as $ans){
|
||||
if($nik === null){
|
||||
|
||||
if(
|
||||
$ans["type"] == "text" &&
|
||||
$old_type == "text"
|
||||
){
|
||||
return $out;
|
||||
}
|
||||
|
||||
$answer_cat = [];
|
||||
$answer = [];
|
||||
|
||||
foreach($nik as $snippet){
|
||||
|
||||
if(!isset($snippet["partOfSpeech"])){ continue; }
|
||||
|
||||
$push = [];
|
||||
|
||||
// add text snippet
|
||||
if(isset($snippet["text"])){
|
||||
|
||||
$i++;
|
||||
$c = count($answer) - 1;
|
||||
|
||||
// append text to existing textfield
|
||||
$answer[$c] = [
|
||||
$push[] = [
|
||||
"type" => "text",
|
||||
"value" => $answer[$c]["value"] . "\n" . $i . ". " . $ans["value"]
|
||||
"value" =>
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$snippet["text"]
|
||||
)
|
||||
];
|
||||
|
||||
}elseif($ans["type"] == "text"){
|
||||
|
||||
$i++;
|
||||
$answer[] = [
|
||||
"type" => "text",
|
||||
"value" => $i . ". " . $ans["value"]
|
||||
];
|
||||
}else{
|
||||
|
||||
// append normally
|
||||
$answer[] = $ans;
|
||||
}
|
||||
|
||||
$old_type = $ans["type"];
|
||||
// add example uses
|
||||
if(isset($snippet["exampleUses"])){
|
||||
|
||||
foreach($snippet["exampleUses"] as $example){
|
||||
|
||||
$push[] = [
|
||||
"type" => "quote",
|
||||
"value" => "\"" .
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$example["text"]
|
||||
) . "\""
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// add citations
|
||||
if(isset($snippet["citations"])){
|
||||
|
||||
foreach($snippet["citations"] as $citation){
|
||||
|
||||
if(!isset($citation["cite"])){ continue; }
|
||||
|
||||
$text =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$citation["cite"]
|
||||
);
|
||||
|
||||
if(isset($citation["source"])){
|
||||
|
||||
$text .=
|
||||
" - " .
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$citation["source"]
|
||||
);
|
||||
}
|
||||
|
||||
$push[] = [
|
||||
"type" => "quote",
|
||||
"value" => $text
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// add related words
|
||||
if(isset($snippet["relatedWords"])){
|
||||
|
||||
$relations = [];
|
||||
|
||||
foreach($snippet["relatedWords"] as $related){
|
||||
|
||||
$words = [];
|
||||
foreach($related["words"] as $wrd){
|
||||
|
||||
$words[] =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$wrd
|
||||
);
|
||||
}
|
||||
|
||||
if(
|
||||
count($words) !== 0 &&
|
||||
isset($related["relationshipType"])
|
||||
){
|
||||
|
||||
$relations[ucfirst($related["relationshipType"]) . "s"] =
|
||||
implode(", ", $words);
|
||||
}
|
||||
}
|
||||
|
||||
foreach($relations as $relation_title => $relation_content){
|
||||
|
||||
$push[] = [
|
||||
"type" => "quote",
|
||||
"value" => $relation_title . ": " . $relation_content
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(count($push) !== 0){
|
||||
|
||||
// push data to answer_cat
|
||||
if(!isset($answer_cat[$snippet["partOfSpeech"]])){
|
||||
|
||||
$answer_cat[$snippet["partOfSpeech"]] = [];
|
||||
}
|
||||
|
||||
$answer_cat[$snippet["partOfSpeech"]] =
|
||||
array_merge(
|
||||
$answer_cat[$snippet["partOfSpeech"]],
|
||||
$push
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
foreach($answer_cat as $answer_title => $answer_content){
|
||||
|
||||
$i = 0;
|
||||
$answer[] = [
|
||||
"type" => "title",
|
||||
"value" => $answer_title
|
||||
];
|
||||
|
||||
$old_type = $answer[count($answer) - 1]["type"];
|
||||
|
||||
foreach($answer_content as $ans){
|
||||
|
||||
if(
|
||||
$ans["type"] == "text" &&
|
||||
$old_type == "text"
|
||||
){
|
||||
|
||||
$i++;
|
||||
$c = count($answer) - 1;
|
||||
|
||||
// append text to existing textfield
|
||||
$answer[$c] = [
|
||||
"type" => "text",
|
||||
"value" => $answer[$c]["value"] . "\n" . $i . ". " . $ans["value"]
|
||||
];
|
||||
|
||||
}elseif($ans["type"] == "text"){
|
||||
|
||||
$i++;
|
||||
$answer[] = [
|
||||
"type" => "text",
|
||||
"value" => $i . ". " . $ans["value"]
|
||||
];
|
||||
}else{
|
||||
|
||||
// append normally
|
||||
$answer[] = $ans;
|
||||
}
|
||||
|
||||
$old_type = $ans["type"];
|
||||
}
|
||||
}
|
||||
|
||||
// yeah.. sometimes duckduckgo doesnt give us a definition back
|
||||
if(count($answer) !== 0){
|
||||
|
||||
$out["answer"][] = [
|
||||
"title" => ucfirst($word),
|
||||
"description" => $answer,
|
||||
"url" => "https://www.wordnik.com/words/" . $word,
|
||||
"thumb" => null,
|
||||
"table" => [],
|
||||
"sublink" => []
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// yeah.. sometimes duckduckgo doesnt give us a definition back
|
||||
if(count($answer) !== 0){
|
||||
//
|
||||
// Parse stackoverflow answer
|
||||
//
|
||||
if(
|
||||
preg_match(
|
||||
'/^\/a\.js.*src_id=stack_overflow/',
|
||||
$potential_endpoint
|
||||
)
|
||||
){
|
||||
|
||||
$out["answer"][] = [
|
||||
"title" => ucfirst($word),
|
||||
"description" => $answer,
|
||||
"url" => "https://www.wordnik.com/words/" . $word,
|
||||
"thumb" => null,
|
||||
"table" => [],
|
||||
"sublink" => []
|
||||
];
|
||||
// found stackoverflow answer
|
||||
try{
|
||||
$json =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://duckduckgo.com" . $potential_endpoint,
|
||||
[],
|
||||
ddg::req_xhr
|
||||
);
|
||||
|
||||
}catch(Exception $e){
|
||||
|
||||
// fail gracefully
|
||||
return $out;
|
||||
}
|
||||
|
||||
$json = explode("DDG.duckbar.add_array(", $json, 2);
|
||||
|
||||
if(count($json) === 2){
|
||||
|
||||
$json =
|
||||
json_decode(
|
||||
$this->fuckhtml
|
||||
->extract_json(
|
||||
$json[1]
|
||||
),
|
||||
true
|
||||
);
|
||||
|
||||
if(
|
||||
$json !== null &&
|
||||
isset($json[0]["data"])
|
||||
){
|
||||
|
||||
$json = $json[0]["data"];
|
||||
|
||||
foreach($json as $answer){
|
||||
|
||||
if(isset($answer["Heading"])){
|
||||
|
||||
$title = $answer["Heading"];
|
||||
}elseif(isset($answer["title"])){
|
||||
|
||||
$title = $answer["title"];
|
||||
}else{
|
||||
|
||||
$title = null;
|
||||
}
|
||||
|
||||
if(
|
||||
$title !== null &&
|
||||
isset($answer["Abstract"])
|
||||
){
|
||||
|
||||
$description = $this->parse_rich_text($answer["Abstract"]);
|
||||
|
||||
$out["answer"][] = [
|
||||
"title" => $title,
|
||||
"description" => $description,
|
||||
"url" => $answer["AbstractURL"],
|
||||
"thumb" => null,
|
||||
"table" => [],
|
||||
"sublink" => []
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1841,6 +1943,146 @@ class ddg{
|
|||
return $out;
|
||||
}
|
||||
|
||||
private function parse_rich_text($html){
|
||||
|
||||
$description = [];
|
||||
|
||||
// pre-process the html, remove useless elements
|
||||
$html =
|
||||
strip_tags(
|
||||
$html,
|
||||
[
|
||||
"h1", "h2", "h3", "h4", "h5", "h6", "h7",
|
||||
"pre", "code"
|
||||
]
|
||||
);
|
||||
|
||||
$html =
|
||||
preg_replace(
|
||||
'/<(\/?)pre *[^>]*>\s*<\/?code *[^>]*>/i',
|
||||
'<$1pre>',
|
||||
$html
|
||||
);
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$tags =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName(
|
||||
"*"
|
||||
);
|
||||
|
||||
if(count($tags) === 0){
|
||||
|
||||
$description[] = [
|
||||
"type" => "text",
|
||||
"value" =>
|
||||
trim(
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$html,
|
||||
true,
|
||||
false
|
||||
)
|
||||
)
|
||||
];
|
||||
}else{
|
||||
|
||||
$start = 0;
|
||||
$was_code_block = true;
|
||||
foreach($tags as $tag){
|
||||
|
||||
$text =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
substr(
|
||||
$html,
|
||||
$start,
|
||||
$tag["startPos"] - $start
|
||||
),
|
||||
true,
|
||||
false
|
||||
);
|
||||
|
||||
if($was_code_block){
|
||||
|
||||
$text = ltrim($text);
|
||||
$was_code_block = false;
|
||||
}
|
||||
|
||||
$description[] = [
|
||||
"type" => "text",
|
||||
"value" => $text
|
||||
];
|
||||
|
||||
switch($tag["tagName"]){
|
||||
|
||||
case "pre":
|
||||
$append = "code";
|
||||
$was_code_block = true;
|
||||
$c = count($description) - 1;
|
||||
$description[$c]["value"] =
|
||||
rtrim($description[$c]["value"]);
|
||||
break;
|
||||
|
||||
case "code":
|
||||
$append = "inline_code";
|
||||
$c = count($description) - 1;
|
||||
$description[$c]["value"] =
|
||||
rtrim($description[$c]["value"]) . " ";
|
||||
break;
|
||||
|
||||
case "h1":
|
||||
case "h2":
|
||||
case "h3":
|
||||
case "h4":
|
||||
case "h5":
|
||||
case "h6":
|
||||
case "h7":
|
||||
$append = "title";
|
||||
$c = count($description) - 1;
|
||||
$description[$c]["value"] =
|
||||
rtrim($description[$c]["value"]);
|
||||
break;
|
||||
}
|
||||
|
||||
$description[] = [
|
||||
"type" => $append,
|
||||
"value" =>
|
||||
trim(
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$tag,
|
||||
true,
|
||||
false
|
||||
)
|
||||
)
|
||||
];
|
||||
|
||||
$start = $tag["endPos"];
|
||||
}
|
||||
|
||||
// shit out remainder
|
||||
$description[] = [
|
||||
"type" => "text",
|
||||
"value" =>
|
||||
trim(
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
substr(
|
||||
$html,
|
||||
$start
|
||||
),
|
||||
true,
|
||||
false
|
||||
)
|
||||
)
|
||||
];
|
||||
}
|
||||
|
||||
return $description;
|
||||
}
|
||||
|
||||
private function titledots($title){
|
||||
|
||||
$substr = substr($title, -3);
|
||||
|
|
|
|||
|
|
@ -16,49 +16,82 @@ class greppr{
|
|||
return [];
|
||||
}
|
||||
|
||||
private function get($proxy, $url, $get = [], $cookie = false){
|
||||
private function get($proxy, $url, $get = [], $cookie = false, $post){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
if($get !== []){
|
||||
$get = http_build_query($get);
|
||||
$url .= "?" . $get;
|
||||
}
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, $url);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
|
||||
if($cookie === false){
|
||||
if($post === false){
|
||||
|
||||
if($get !== []){
|
||||
$get = http_build_query($get);
|
||||
$url .= "?" . $get;
|
||||
}
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1"]
|
||||
);
|
||||
if($cookie === false){
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1"]
|
||||
);
|
||||
}else{
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
"DNT: 1",
|
||||
"Sec-GPC: 1",
|
||||
"Connection: keep-alive",
|
||||
"Referer: https://greppr.org/search",
|
||||
"Cookie: PHPSESSID=$cookie",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: same-origin",
|
||||
"Sec-Fetch-User: ?1",
|
||||
"Priority: u=0, i"]
|
||||
);
|
||||
}
|
||||
}else{
|
||||
|
||||
$get = http_build_query($get);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_POST, true);
|
||||
curl_setopt($curlproc, CURLOPT_POSTFIELDS, $get);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip",
|
||||
"Cookie: PHPSESSID=" . $cookie,
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
"Content-Type: application/x-www-form-urlencoded",
|
||||
"Content-Length: " . strlen($get),
|
||||
"Origin: https://greppr.org",
|
||||
"DNT: 1",
|
||||
"Sec-GPC: 1",
|
||||
"Connection: keep-alive",
|
||||
"Referer: https://greppr.org/",
|
||||
"Cookie: PHPSESSID=$cookie",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1"]
|
||||
"Sec-Fetch-Site: same-origin",
|
||||
"Sec-Fetch-User: ?1",
|
||||
"Priority: u=0, i"]
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -113,7 +146,24 @@ class greppr{
|
|||
|
||||
[$q, $proxy] = $this->backend->get($get["npt"], "web");
|
||||
|
||||
$q = json_decode($q, true);
|
||||
$tokens = json_decode($q, true);
|
||||
|
||||
//
|
||||
// Get paginated page
|
||||
//
|
||||
try{
|
||||
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://greppr.org" . $tokens["get"],
|
||||
[],
|
||||
$tokens["cookie"],
|
||||
false
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
|
||||
}else{
|
||||
|
||||
|
|
@ -124,88 +174,114 @@ class greppr{
|
|||
}
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
}
|
||||
|
||||
// get token
|
||||
// token[0] = static token that changes once a day
|
||||
// token[1] = dynamic token that changes on every request
|
||||
// token[1] = PHPSESSID cookie
|
||||
$tokens = apcu_fetch("greppr_token");
|
||||
|
||||
if(
|
||||
$tokens === false ||
|
||||
$first_attempt === false // force token fetch
|
||||
){
|
||||
|
||||
// we haven't gotten the token yet, get it
|
||||
//
|
||||
// get token
|
||||
//
|
||||
try{
|
||||
|
||||
$response =
|
||||
$html =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://greppr.org",
|
||||
[]
|
||||
[],
|
||||
false,
|
||||
false
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search tokens");
|
||||
}
|
||||
|
||||
$tokens = $this->parse_token($response);
|
||||
//
|
||||
// Parse token
|
||||
//
|
||||
$this->fuckhtml->load($html["data"]);
|
||||
|
||||
$tokens = [];
|
||||
|
||||
$inputs =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName(
|
||||
"input"
|
||||
);
|
||||
|
||||
foreach($inputs as $input){
|
||||
|
||||
if(!isset($input["attributes"]["name"])){
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
switch($input["attributes"]["name"]){
|
||||
|
||||
case "var1":
|
||||
case "var2":
|
||||
case "n":
|
||||
$tokens[$input["attributes"]["name"]] =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$input["attributes"]["value"]
|
||||
);
|
||||
break;
|
||||
|
||||
default:
|
||||
$tokens["req"] =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$input["attributes"]["name"]
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// get cookie
|
||||
preg_match(
|
||||
'/PHPSESSID=([^;]+)/',
|
||||
$html["headers"]["set-cookie"],
|
||||
$cookie
|
||||
);
|
||||
|
||||
if(!isset($cookie[1])){
|
||||
|
||||
// server sent an unexpected cookie
|
||||
throw new Exception("Got malformed cookie");
|
||||
}
|
||||
|
||||
$tokens["cookie"] = $cookie[1];
|
||||
|
||||
if($tokens === false){
|
||||
|
||||
throw new Exception("Failed to grep search tokens");
|
||||
}
|
||||
}
|
||||
|
||||
try{
|
||||
|
||||
if($get["npt"]){
|
||||
//
|
||||
// Get initial search page
|
||||
//
|
||||
try{
|
||||
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://greppr.org/search",
|
||||
[
|
||||
"var1" => $tokens["var1"],
|
||||
"var2" => $tokens["var2"],
|
||||
$tokens["req"] => $search,
|
||||
"n" => $tokens["n"]
|
||||
],
|
||||
$tokens["cookie"],
|
||||
true
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
$params = [
|
||||
$tokens[0] => $q["q"],
|
||||
"s" => $q["s"],
|
||||
"l" => 30,
|
||||
"n" => $tokens[1]
|
||||
];
|
||||
}else{
|
||||
|
||||
$params = [
|
||||
$tokens[0] => $search,
|
||||
"n" => $tokens[1]
|
||||
];
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
|
||||
$searchresults = $this->get(
|
||||
$proxy,
|
||||
"https://greppr.org/search",
|
||||
$params,
|
||||
$tokens[2]
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
|
||||
if(strlen($searchresults["data"]) === 0){
|
||||
|
||||
// redirected to main page, which means we got old token
|
||||
// generate a new one
|
||||
|
||||
// ... unless we just tried to do that
|
||||
if($first_attempt === false){
|
||||
|
||||
throw new Exception("Failed to get a new search token");
|
||||
}
|
||||
|
||||
return $this->web($get, false);
|
||||
}
|
||||
//$html = file_get_contents("scraper/greppr.html");
|
||||
//$this->fuckhtml->load($html);
|
||||
$this->fuckhtml->load($html["data"]);
|
||||
|
||||
// refresh the token with new data (this also triggers fuckhtml load)
|
||||
$this->parse_token($searchresults, $tokens[2]);
|
||||
|
||||
// response object
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"spelling" => [
|
||||
|
|
@ -254,24 +330,16 @@ class greppr{
|
|||
|
||||
if($break === true){
|
||||
|
||||
parse_str(
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$a["attributes"]["href"]
|
||||
),
|
||||
$values
|
||||
);
|
||||
|
||||
$values = array_values($values);
|
||||
|
||||
$out["npt"] =
|
||||
$this->backend->store(
|
||||
json_encode(
|
||||
[
|
||||
"q" => $values[0],
|
||||
"s" => $values[1]
|
||||
]
|
||||
),
|
||||
json_encode([
|
||||
"get" =>
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$a["attributes"]["href"]
|
||||
),
|
||||
"cookie" => $tokens["cookie"]
|
||||
]),
|
||||
"web",
|
||||
$proxy
|
||||
);
|
||||
|
|
@ -360,76 +428,8 @@ class greppr{
|
|||
return $out;
|
||||
}
|
||||
|
||||
private function parse_token($response, $cookie = false){
|
||||
|
||||
$this->fuckhtml->load($response["data"]);
|
||||
|
||||
$scripts =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName("script");
|
||||
|
||||
$found = false;
|
||||
foreach($scripts as $script){
|
||||
|
||||
preg_match(
|
||||
'/window\.location ?= ?\'\/search\?([^=]+).*&n=([0-9]+)/',
|
||||
$script["innerHTML"],
|
||||
$tokens
|
||||
);
|
||||
|
||||
if(isset($tokens[1])){
|
||||
|
||||
$found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if($found === false){
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
$tokens = [
|
||||
$tokens[1],
|
||||
$tokens[2]
|
||||
];
|
||||
|
||||
if($cookie !== false){
|
||||
|
||||
// we already specified a cookie, so use the one we have already
|
||||
$tokens[] = $cookie;
|
||||
apcu_store("greppr_token", $tokens);
|
||||
|
||||
return $tokens;
|
||||
}
|
||||
|
||||
if(!isset($response["headers"]["set-cookie"])){
|
||||
|
||||
// server didn't send a cookie
|
||||
return false;
|
||||
}
|
||||
|
||||
// get cookie
|
||||
preg_match(
|
||||
'/PHPSESSID=([^;]+)/',
|
||||
$response["headers"]["set-cookie"],
|
||||
$cookie
|
||||
);
|
||||
|
||||
if(!isset($cookie[1])){
|
||||
|
||||
// server sent an unexpected cookie
|
||||
return false;
|
||||
}
|
||||
|
||||
$tokens[] = $cookie[1];
|
||||
apcu_store("greppr_token", $tokens);
|
||||
|
||||
return $tokens;
|
||||
}
|
||||
|
||||
private function limitstrlen($text){
|
||||
|
||||
return explode("\n", wordwrap($text, 300, "\n"))[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -235,6 +235,10 @@ $settings = [
|
|||
"value" => "pinterest",
|
||||
"text" => "Pinterest"
|
||||
],
|
||||
[
|
||||
"value" => "cara",
|
||||
"text" => "Cara"
|
||||
],
|
||||
[
|
||||
"value" => "flickr",
|
||||
"text" => "Flickr"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue