Merge pull request #388 from itk-dev/feature/robots.txt

Cleaned up robots.txt

Author: rimi-itk

.gitignore

@@ -93,7 +93,6 @@
 /web/autoload.php
 /web/example.gitignore
 /web/index.php
-/web/robots.txt
 /web/update.php
 /web/web.config
 /web/modules/README.txt
@@ -169,7 +168,6 @@
 /web/INSTALL.txt
 /web/LICENSE.txt
 /web/README.txt
-/web/robots.txt
 /web/update.php
 /web/web.config
 

CHANGELOG.md

@@ -8,6 +8,8 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+- [PR-388](https://github.com/itk-dev/os2loop/pull/388)
+  Cleaned up robots.txt
 - [PR-387](https://github.com/itk-dev/os2loop/pull/387)
   Disabled caching of error pages
 - [PR-385](https://github.com/itk-dev/os2loop/pull/385)

composer.json

@@ -170,6 +170,7 @@
         },
         "drupal-scaffold": {
             "file-mapping": {
+                "[web-root]/robots.txt": false,
                 "[web-root]/sites/development.services.yml": false
             },
             "locations": {

composer.lock

@@ -0,0 +1,18 @@
+#
+# robots.txt
+#
+# This file is to prevent the crawling and indexing of certain parts
+# of your site by web crawlers and spiders run by sites like Yahoo!
+# and Google. By telling these "robots" where not to go on your site,
+# you save bandwidth and server resources.
+#
+# This file will be ignored unless it is at the root of your host:
+# Used:    http://example.com/robots.txt
+# Ignored: http://example.com/site/robots.txt
+#
+# For more information about the robots.txt standard, see:
+# http://www.robotstxt.org/robotstxt.html
+
+# We don't want anything to index anything.
+User-agent: *
+Disallow: /