Robots.txt update + new htaccess for cache folder.

Updated robots.txt + new htaccess files.

I made the following changes for the robots.txt and .htaccess files.

# robots.txt updated
- Allow theme assets (CSS, JS, Images, Fonts) for SEO
- Allow folder with uploaded images (/content/images)

# .httaccess files added
- /content - Deny all
- /content/images - Allow only jpg|png|gif
- /cache - Deny all
- /system - Deny all
- /system/admin/editor - Allow all
- /system/resources - Allow all
- /themes - Deny all
- /themes/blog/css - Allow all
- /themes/blog/fonts - Allow all
- /themes/blog/images - Allow all
- /themes/blog/js - Allow all
- /themes/clean/css - Allow all
- /themes/default/css  - Allow all
- /themes\default\img - Allow all
- /themes\logs\css - Allow all

cache/.htaccess

@@ -0,0 +1 @@
+deny from all

content/.htaccess

@@ -0,0 +1 @@
+deny from all

content/images/.htaccess

@@ -0,0 +1,4 @@
+Deny from all
+<Files ~ "\.(jpg|png|gif)$">
+   Allow from all
+</Files>

robots.txt

@@ -17,17 +17,30 @@
 # http://www.sxw.org.uk/computing/robots/check.html
 
 User-agent: *
-# Directories
+
+# Disallow directories
 Disallow: /config/
 Disallow: /system/
 Disallow: /themes/
 Disallow: /vendor/
 Disallow: /cache/
-# Files
+
+# Disallow files
 Disallow: /changelog.txt
 Disallow: /composer.json
 Disallow: /composer.lock
 Disallow: /composer.phar
-# Paths 
+
+# Disallow paths
 Disallow: /search/
-Disallow: /admin/
+Disallow: /admin/
+
+# Allow themes
+Allow: /themes/*/css/
+Allow: /themes/*/images/
+Allow: /themes/*/img/
+Allow: /themes/*/js/
+Allow: /themes/*/fonts/
+Allow: /content/images/*.jpg
+Allow: /content/images/*.png
+Allow: /content/images/*.gif

system/.htaccess

@@ -0,0 +1 @@
+deny from all

system/admin/editor/.htaccess

@@ -0,0 +1 @@
+allow from all

system/resources/.htaccess

@@ -0,0 +1 @@
+allow from all

themes/.htaccess

@@ -0,0 +1 @@
+deny from all

themes/blog/css/.htaccess

@@ -0,0 +1 @@
+allow from all

themes/blog/fonts/.htaccess

@@ -0,0 +1 @@
+allow from all

themes/blog/images/.htaccess

@@ -0,0 +1 @@
+allow from all

themes/blog/js/.htaccess

@@ -0,0 +1 @@
+allow from all

themes/clean/css/.htaccess

@@ -0,0 +1 @@
+allow from all

themes/default/css/.htaccess

@@ -0,0 +1 @@
+allow from all

themes/default/img/.htaccess

@@ -0,0 +1 @@
+allow from all

themes/logs/css/.htaccess

@@ -0,0 +1 @@
+allow from all