Make sure that the markdown files don't show up the search results

Author: fulopkovacs

public/robots.txt

@@ -0,0 +1,71 @@
+User-agent: Googlebot
+Disallow: /*.md$
+
+User-agent: Bingbot
+Disallow: /*.md$
+
+User-agent: 360Spider
+Disallow: /*.md$
+
+User-agent: Applebot
+Disallow: /*.md$
+
+User-agent: Baiduspider
+Disallow: /*.md$
+
+User-agent: coccocbot
+Disallow: /*.md$
+
+User-agent: Daumoa
+Disallow: /*.md$
+
+User-agent: DuckDuckBot
+Disallow: /*.md$
+
+User-agent: Edge Crawler
+Disallow: /*.md$
+
+User-agent: FAST-WebCrawler
+Disallow: /*.md$
+
+User-agent: iaskspider
+Disallow: /*.md$
+
+User-agent: MojeekBot
+Disallow: /*.md$
+
+User-agent: NaverBot
+Disallow: /*.md$
+
+User-agent: Qwantify
+Disallow: /*.md$
+
+User-agent: Rambler bot
+Disallow: /*.md$
+
+User-agent: Seekport Crawler
+Disallow: /*.md$
+
+User-agent: SeznamBot
+Disallow: /*.md$
+
+User-agent: sogou spider
+Disallow: /*.md$
+
+User-agent: startpage bot
+Disallow: /*.md$
+
+User-agent: Swisscows Crawler
+Disallow: /*.md$
+
+User-agent: Yahoo!
+Disallow: /*.md$
+
+User-agent: Yahoo! JAPAN
+Disallow: /*.md$
+
+User-agent: YandexBot
+Disallow: /*.md$
+
+User-agent: YisouSpider
+Disallow: /*.md$