Add robots.txt

build.sh

@@ -8,6 +8,7 @@ npm run build
 
 rsync -ra  public/ "${TARGET}/instance/static"
 cp dist/index.html  "${TARGET}/instance/static/index.html"
+cp robots.txt "${TARGET}/instance/static/robots.txt"
 rsync --delete -ra dist/static/ "${TARGET}/instance/static/static"
 rsync --delete -ra images/ "${TARGET}/instance/static/images"
 rsync --delete -ra sounds/ "${TARGET}/instance/static/sounds"

robots.txt

@@ -0,0 +1,12 @@
+# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
+ User-agent: *
+ Disallow: /web
+ Disallow: /settings
+ Disallow: /login
+ Disallow: /user-search
+ Disallow: /registration
+ Disallow: /cgi-bin
+ Disallow: /pages/DMCA.html
+ Disallow: /oauth
+ Disallow: /search
+ Disallow: /tag