add robots.txt file

adbar · Oct 30, 2023 · 41adf07 · 41adf07
1 parent 8fa6f45
commit 41adf07
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 0 deletions.
diff --git a/docs/conf.py b/docs/conf.py
@@ -139,3 +139,5 @@
 
 html_baseurl = 'https://trafilatura.readthedocs.io/'
 sitemap_url_scheme = "{lang}latest/{link}"
+
+html_extra_path = ['robots.txt']
diff --git a/docs/robots.txt b/docs/robots.txt
@@ -0,0 +1,3 @@
+User-agent: *
+
+Sitemap: https://trafilatura.readthedocs.io/en/latest/sitemap.xml
Original file line number	Diff line number	Diff line change
Expand Up		@@ -139,3 +139,5 @@

		html_baseurl = 'https://trafilatura.readthedocs.io/'
		sitemap_url_scheme = "{lang}latest/{link}"

		html_extra_path = ['robots.txt']
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		User-agent: *

		Sitemap: https://trafilatura.readthedocs.io/en/latest/sitemap.xml