Added robots.txt for productive environments

Author: byRoadrunner

.github/workflows/yarn.yml

@@ -44,7 +44,13 @@ jobs:
         if: ${{ github.ref != 'refs/heads/main' }}
         uses: canastro/copy-file-action@master
         with:
-          source: "template/robots.txt"
+          source: "template/dev/robots.txt"
+          target: "build/robots.txt"
+      - name: Copy robots.txt to productive envs
+        if: ${{ github.ref = 'refs/heads/main' }}
+        uses: canastro/copy-file-action@master
+        with:
+          source: "template/prod/robots.txt"
           target: "build/robots.txt"
       - name: Upload artifact
         uses: actions/upload-artifact@v2

template/robots.txt template/dev/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Disallow: /cloudnet/
+Disallow: /repositories/
+Disallow: /data/