diff --git a/.gitignore b/.gitignore index 1f766bd38..76e17d342 100644 --- a/.gitignore +++ b/.gitignore @@ -93,7 +93,6 @@ /web/autoload.php /web/example.gitignore /web/index.php -/web/robots.txt /web/update.php /web/web.config /web/modules/README.txt @@ -169,7 +168,6 @@ /web/INSTALL.txt /web/LICENSE.txt /web/README.txt -/web/robots.txt /web/update.php /web/web.config diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b304ec8b..6e5033462 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- [PR-388](https://github.com/itk-dev/os2loop/pull/388) + Cleaned up robots.txt - [PR-387](https://github.com/itk-dev/os2loop/pull/387) Disabled caching of error pages - [PR-385](https://github.com/itk-dev/os2loop/pull/385) diff --git a/composer.json b/composer.json index b45228a9b..dcf7265a2 100644 --- a/composer.json +++ b/composer.json @@ -170,6 +170,7 @@ }, "drupal-scaffold": { "file-mapping": { + "[web-root]/robots.txt": false, "[web-root]/sites/development.services.yml": false }, "locations": { diff --git a/composer.lock b/composer.lock index f72016ceb..1e602e7c9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2064f4843b980734c64a1412221c79e2", + "content-hash": "515d39065690231d477b527300c20f8a", "packages": [ { "name": "asm89/stack-cors", diff --git a/web/robots.txt b/web/robots.txt new file mode 100755 index 000000000..5349056ec --- /dev/null +++ b/web/robots.txt @@ -0,0 +1,18 @@ +# +# robots.txt +# +# This file is to prevent the crawling and indexing of certain parts +# of your site by web crawlers and spiders run by sites like Yahoo! +# and Google. By telling these "robots" where not to go on your site, +# you save bandwidth and server resources. +# +# This file will be ignored unless it is at the root of your host: +# Used: http://example.com/robots.txt +# Ignored: http://example.com/site/robots.txt +# +# For more information about the robots.txt standard, see: +# http://www.robotstxt.org/robotstxt.html + +# We don't want anything to index anything. +User-agent: * +Disallow: /