From 80afb162d97c189efbd2027aa0c456a549be3268 Mon Sep 17 00:00:00 2001 From: Andreas Date: Mon, 24 Aug 2015 16:02:46 +0200 Subject: [PATCH] Robots.txt update + new htaccess for cache folder. Updated robots.txt + new htaccess files. I made the following changes for the robots.txt and .htaccess files. # robots.txt updated - Allow theme assets (CSS, JS, Images, Fonts) for SEO - Allow folder with uploaded images (/content/images) # .httaccess files added - /content - Deny all - /content/images - Allow only jpg|png|gif - /cache - Deny all - /system - Deny all - /system/admin/editor - Allow all - /system/resources - Allow all - /themes - Deny all - /themes/blog/css - Allow all - /themes/blog/fonts - Allow all - /themes/blog/images - Allow all - /themes/blog/js - Allow all - /themes/clean/css - Allow all - /themes/default/css - Allow all - /themes\default\img - Allow all - /themes\logs\css - Allow all --- cache/.htaccess | 1 + content/.htaccess | 1 + content/images/.htaccess | 4 ++++ robots.txt | 21 +++++++++++++++++---- system/.htaccess | 1 + system/admin/editor/.htaccess | 1 + system/resources/.htaccess | 1 + themes/.htaccess | 1 + themes/blog/css/.htaccess | 1 + themes/blog/fonts/.htaccess | 1 + themes/blog/images/.htaccess | 1 + themes/blog/js/.htaccess | 1 + themes/clean/css/.htaccess | 1 + themes/default/css/.htaccess | 1 + themes/default/img/.htaccess | 1 + themes/logs/css/.htaccess | 1 + 16 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 cache/.htaccess create mode 100644 content/.htaccess create mode 100644 content/images/.htaccess create mode 100644 system/.htaccess create mode 100644 system/admin/editor/.htaccess create mode 100644 system/resources/.htaccess create mode 100644 themes/.htaccess create mode 100644 themes/blog/css/.htaccess create mode 100644 themes/blog/fonts/.htaccess create mode 100644 themes/blog/images/.htaccess create mode 100644 themes/blog/js/.htaccess create mode 100644 themes/clean/css/.htaccess create mode 100644 themes/default/css/.htaccess create mode 100644 themes/default/img/.htaccess create mode 100644 themes/logs/css/.htaccess diff --git a/cache/.htaccess b/cache/.htaccess new file mode 100644 index 0000000..3418e55 --- /dev/null +++ b/cache/.htaccess @@ -0,0 +1 @@ +deny from all \ No newline at end of file diff --git a/content/.htaccess b/content/.htaccess new file mode 100644 index 0000000..3418e55 --- /dev/null +++ b/content/.htaccess @@ -0,0 +1 @@ +deny from all \ No newline at end of file diff --git a/content/images/.htaccess b/content/images/.htaccess new file mode 100644 index 0000000..bb3e49d --- /dev/null +++ b/content/images/.htaccess @@ -0,0 +1,4 @@ +Deny from all + + Allow from all + \ No newline at end of file diff --git a/robots.txt b/robots.txt index cb5adb2..7297323 100644 --- a/robots.txt +++ b/robots.txt @@ -17,17 +17,30 @@ # http://www.sxw.org.uk/computing/robots/check.html User-agent: * -# Directories + +# Disallow directories Disallow: /config/ Disallow: /system/ Disallow: /themes/ Disallow: /vendor/ Disallow: /cache/ -# Files + +# Disallow files Disallow: /changelog.txt Disallow: /composer.json Disallow: /composer.lock Disallow: /composer.phar -# Paths + +# Disallow paths Disallow: /search/ -Disallow: /admin/ \ No newline at end of file +Disallow: /admin/ + +# Allow themes +Allow: /themes/*/css/ +Allow: /themes/*/images/ +Allow: /themes/*/img/ +Allow: /themes/*/js/ +Allow: /themes/*/fonts/ +Allow: /content/images/*.jpg +Allow: /content/images/*.png +Allow: /content/images/*.gif \ No newline at end of file diff --git a/system/.htaccess b/system/.htaccess new file mode 100644 index 0000000..3418e55 --- /dev/null +++ b/system/.htaccess @@ -0,0 +1 @@ +deny from all \ No newline at end of file diff --git a/system/admin/editor/.htaccess b/system/admin/editor/.htaccess new file mode 100644 index 0000000..72d0a6e --- /dev/null +++ b/system/admin/editor/.htaccess @@ -0,0 +1 @@ +allow from all \ No newline at end of file diff --git a/system/resources/.htaccess b/system/resources/.htaccess new file mode 100644 index 0000000..72d0a6e --- /dev/null +++ b/system/resources/.htaccess @@ -0,0 +1 @@ +allow from all \ No newline at end of file diff --git a/themes/.htaccess b/themes/.htaccess new file mode 100644 index 0000000..3418e55 --- /dev/null +++ b/themes/.htaccess @@ -0,0 +1 @@ +deny from all \ No newline at end of file diff --git a/themes/blog/css/.htaccess b/themes/blog/css/.htaccess new file mode 100644 index 0000000..72d0a6e --- /dev/null +++ b/themes/blog/css/.htaccess @@ -0,0 +1 @@ +allow from all \ No newline at end of file diff --git a/themes/blog/fonts/.htaccess b/themes/blog/fonts/.htaccess new file mode 100644 index 0000000..72d0a6e --- /dev/null +++ b/themes/blog/fonts/.htaccess @@ -0,0 +1 @@ +allow from all \ No newline at end of file diff --git a/themes/blog/images/.htaccess b/themes/blog/images/.htaccess new file mode 100644 index 0000000..72d0a6e --- /dev/null +++ b/themes/blog/images/.htaccess @@ -0,0 +1 @@ +allow from all \ No newline at end of file diff --git a/themes/blog/js/.htaccess b/themes/blog/js/.htaccess new file mode 100644 index 0000000..72d0a6e --- /dev/null +++ b/themes/blog/js/.htaccess @@ -0,0 +1 @@ +allow from all \ No newline at end of file diff --git a/themes/clean/css/.htaccess b/themes/clean/css/.htaccess new file mode 100644 index 0000000..72d0a6e --- /dev/null +++ b/themes/clean/css/.htaccess @@ -0,0 +1 @@ +allow from all \ No newline at end of file diff --git a/themes/default/css/.htaccess b/themes/default/css/.htaccess new file mode 100644 index 0000000..72d0a6e --- /dev/null +++ b/themes/default/css/.htaccess @@ -0,0 +1 @@ +allow from all \ No newline at end of file diff --git a/themes/default/img/.htaccess b/themes/default/img/.htaccess new file mode 100644 index 0000000..72d0a6e --- /dev/null +++ b/themes/default/img/.htaccess @@ -0,0 +1 @@ +allow from all \ No newline at end of file diff --git a/themes/logs/css/.htaccess b/themes/logs/css/.htaccess new file mode 100644 index 0000000..72d0a6e --- /dev/null +++ b/themes/logs/css/.htaccess @@ -0,0 +1 @@ +allow from all \ No newline at end of file