# robots.txt - robot exclusion file - back-end server version - no robots! # ======================================================================== User-agent: htdig/3.1.6 Disallow: /cgi-bin Disallow: /entrez Disallow: /stat Disallow: /COG Disallow: /Entrez Disallow: /mailman Disallow: /myncbi Disallow: /sutils Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /mapview Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/issues/ Disallow: /pmc/articles/ Disallow: /pmc/ivip/ Disallow: /pmc/articlerender.fcgi Disallow: /pmc/pagerender.fcgi Disallow: /pmc/picrender.fcgi Disallow: /pmc/tocrender.fcgi Disallow: /pmc/*report= Disallow: /pmc/*page= Disallow: /pmc/articles/*/citedby/ Disallow: /pmc/publisherportal/api/ Disallow: /pmc/publisherportal/application/ Disallow: /pmc/publisherportal/download Disallow: /pmc/publisherportal/journalmanager/ Disallow: /labs/pmc/ Disallow: /geo/tools Disallow: /projects/geo/tools Disallow: /geo/download Disallow: /projects/geo/download Disallow: /projects/gtr/gt Disallow: /Structure/flink/flink.cgi Disallow: /Structure/vast/vastsrv.cgi Disallow: /viewvc Disallow: /sars-cov-2 User-agent: Baiduspider Disallow: /cgi-bin Disallow: /entrez Disallow: /stat Disallow: /COG Disallow: /Entrez Disallow: /mailman Disallow: /myncbi Disallow: /sutils Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /mapview Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/issues/ Disallow: /pmc/articles/ Disallow: /pmc/ivip/ Disallow: /pmc/articlerender.fcgi Disallow: /pmc/pagerender.fcgi Disallow: /pmc/picrender.fcgi Disallow: /pmc/tocrender.fcgi Disallow: /pmc/*report= Disallow: /pmc/*page= Disallow: /pmc/articles/*/citedby/ Disallow: /pmc/publisherportal/api/ Disallow: /pmc/publisherportal/application/ Disallow: /pmc/publisherportal/download Disallow: /pmc/publisherportal/journalmanager/ Disallow: /labs/pmc/ Disallow: /geo/tools Disallow: /projects/geo/tools Disallow: /geo/download Disallow: /projects/geo/download Disallow: /projects/gtr/gt Disallow: /Structure/flink/flink.cgi Disallow: /Structure/vast/vastsrv.cgi Disallow: /viewvc Disallow: /sars-cov-2 User-agent: Googlebot Disallow: /cgi-bin Disallow: /stat Disallow: /COG Disallow: /mailman Disallow: /sutils Disallow: /entrez/viewer.fcgi Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /SAGE Disallow: /SNP Disallow: /snp Disallow: /IEB/ToolBox/SB Disallow: /Structure/dgs Disallow: /Structure/VA Disallow: /Structure/cblast/cblast.cgi Disallow: /Coffeebreak Disallow: /gorf Disallow: /mapview Disallow: /myncbi Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /entrez/eutils Disallow: /entrez/utils Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/issues/ Disallow: /pmc/*/bin/ Disallow: /pmc/*/pdf/ Disallow: /pmc/*/equ/ Disallow: /pmc/*.cgi Disallow: /pmc/*report= Disallow: /pmc/*page= Disallow: /pmc/articles/*/citedby/ Disallow: /pmc/publisherportal/api/ Disallow: /pmc/publisherportal/application/ Disallow: /pmc/publisherportal/download Disallow: /pmc/publisherportal/journalmanager/ Disallow: /labs/pmc/ Disallow: /geo/tools Disallow: /projects/geo/tools Disallow: /geo/download Disallow: /projects/geo/download Disallow: /projects/gtr/gt Disallow: /Structure/flink/flink.cgi Disallow: /Structure/vast/vastsrv.cgi Disallow: /viewvc Disallow: /clone/library/genomic/organisms Disallow: /sars-cov-2 Allow: /portal/portal3rc.fcgi/*/*.js Allow: /portal/portal3rc.fcgi/*/*.css User-agent: Teleport Pro Disallow: /cgi-bin Disallow: /entrez Disallow: /stat Disallow: /COG Disallow: /Entrez Disallow: /myncbi Disallow: /mailman Disallow: /sutils Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /mapview Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/issues/ Disallow: /pmc/articles/ Disallow: /pmc/ivip/ Disallow: /pmc/articlerender.fcgi Disallow: /pmc/pagerender.fcgi Disallow: /pmc/picrender.fcgi Disallow: /pmc/tocrender.fcgi Disallow: /pmc/publisherportal/api/ Disallow: /pmc/publisherportal/application/ Disallow: /pmc/publisherportal/download Disallow: /pmc/publisherportal/journalmanager/ Disallow: /labs/pmc/ Disallow: /geo/tools Disallow: /projects/geo/tools Disallow: /geo/download Disallow: /projects/geo/download Disallow: /projects/gtr/gt Disallow: /Structure/flink/flink.cgi Disallow: /Structure/vast/vastsrv.cgi Disallow: /viewvc Disallow: /sars-cov-2 User-agent: yacybot Disallow: * User-agent: Browsershots Disallow: Disallow: /sars-cov-2 User-agent: Slurp Crawl-delay: 1 Disallow: /cgi-bin Disallow: /entrez Disallow: /stat Disallow: /COG Disallow: /Entrez Disallow: /myncbi Disallow: /mailman Disallow: /sutils Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /mapview Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /SAGE Disallow: /SNP Disallow: /snp Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/*/bin/ Disallow: /pmc/*/pdf/ Disallow: /pmc/*/equ/ Disallow: /pmc/*.cgi Disallow: /pmc/*report= Disallow: /pmc/*page= Disallow: /pmc/articles/*/citedby/ Disallow: /pmc/publisherportal/api/ Disallow: /pmc/publisherportal/application/ Disallow: /pmc/publisherportal/download Disallow: /pmc/publisherportal/journalmanager/ Disallow: /labs/pmc/ Disallow: /geo/tools Disallow: /projects/geo/tools Disallow: /geo/download Disallow: /projects/geo/download Disallow: /projects/gtr/gt Disallow: /Structure/flink/flink.cgi Disallow: /Structure/vast/vastsrv.cgi Disallow: /viewvc Disallow: /sars-cov-2 User-agent: Twitterbot Disallow: * Allow: /pmc/articles Allow: /books/NBK* User-agent: Exabot Disallow: /Structure Disallow: /mailman Disallow: /stat Disallow: /sars-cov-2 User-agent: bingbot Disallow: /cgi-bin Disallow: /entrez Disallow: /stat Disallow: /COG Disallow: /Entrez Disallow: /myncbi Disallow: /mailman Disallow: /sutils Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /mapview Disallow: /SAGE Disallow: /SNP Disallow: /snp Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/issues/ Disallow: /pmc/*/bin/ Disallow: /pmc/*/pdf/ Disallow: /pmc/*/equ/ Disallow: /pmc/*.cgi Disallow: /pmc/*report= Disallow: /pmc/*page= Disallow: /pmc/articles/*/citedby/ Disallow: /pmc/publisherportal/api/ Disallow: /pmc/publisherportal/application/ Disallow: /pmc/publisherportal/download Disallow: /pmc/publisherportal/journalmanager/ Disallow: /labs/pmc/ Disallow: /geo/tools Disallow: /projects/geo/tools Disallow: /geo/download Disallow: /projects/geo/download Disallow: /projects/gtr/gt Disallow: /Structure/flink/flink.cgi Disallow: /Structure/vast/vastsrv.cgi Disallow: /viewvc Disallow: /clone/library/genomic/organisms Disallow: /sars-cov-2 Allow: /portal/portal3rc.fcgi/*/*.js Allow: /portal/portal3rc.fcgi/*/*.css User-agent: * Crawl-delay: 5 Disallow: /cgi-bin Disallow: /stat Disallow: /entrez Disallow: /COG Disallow: /Entrez Disallow: /myncbi Disallow: /mailman Disallow: /sutils Disallow: /Taxonomy/Selector Disallow: /Taxonomy/CommonTree Disallow: /entrez/sutils Disallow: /mapview Disallow: /SAGE Disallow: /SNP Disallow: /snp Disallow: /blast/BlastAlign.cgi Disallow: /blast/bl2seq/wblast2.cgi Disallow: /portal Disallow: /pmc/utilities/ Disallow: /pmc/issues/ Disallow: /pmc/articles/ Disallow: /pmc/ivip/ Disallow: /pmc/articlerender.fcgi Disallow: /pmc/pagerender.fcgi Disallow: /pmc/picrender.fcgi Disallow: /pmc/tocrender.fcgi Disallow: /pmc/*report= Disallow: /pmc/*page= Disallow: /pmc/articles/*/citedby/ Disallow: /pmc/publisherportal/api/ Disallow: /pmc/publisherportal/application/ Disallow: /pmc/publisherportal/download Disallow: /pmc/publisherportal/journalmanager/ Disallow: /labs/pmc/ Disallow: /geo/tools Disallow: /projects/geo/tools Disallow: /geo/download Disallow: /projects/geo/download Disallow: /projects/gtr/gt Disallow: /Structure/flink/flink.cgi Disallow: /Structure/vast/vastsrv.cgi Disallow: /viewvc Disallow: /clone/library/genomic/organisms Disallow: /sars-cov-2 Allow: /portal/portal3rc.fcgi/*/*.js Allow: /portal/portal3rc.fcgi/*/*.css Sitemap: https://www.ncbi.nlm.nih.gov/Structure/sitemaps/cdd/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/Structure/sitemaps/genome/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/Structure/sitemaps/mmdb/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/Structure/sitemaps/sparcle/sitemap.xml Sitemap: https://www.ncbi.nlm.nih.gov/Traces/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/core/assets/osiris/files/sitemap.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=assembly/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=biosystems/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=books/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=cdd/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=clinvar/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=gene/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=gtr/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=medgen/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=mesh/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=ncbi/sitemap.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=nlmcatalog/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=orgtrack/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=pmc/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=snp/sitemap_index.xml Sitemap: https://www.ncbi.nlm.nih.gov/escan?p=sra/sitemap_index.xml