User-Agent: twiceler
Disallow: *
User-Agent: lwp-trivial
Disallow: *

User-Agent: *
Crawl-delay: 500
Disallow: /dtd
Disallow: /Resources
Disallow: /Trubrum
Disallow: /vfs
Disallow: /shibase
Disallow: /tred
Disallow: /genomecomp
Disallow: /GenomeComp/Reference
Disallow: /GenomeComp/Archives
Disallow: /GenomeComp/Manual/Images
Disallow: /VFs/blast
Disallow: /VFs/Down
Disallow: /VFs/Figures
Disallow: /VFs/download.htm
Disallow: /VFs/thanks.htm
Disallow: /cgi-bin/VFs/gene.cgi
Disallow: /cgi-bin/VFs/vfs.cgi?VFID=
Disallow: /cgi-bin/VFs/comp_gene.cgi
Disallow: /cgi-bin/VFs/comp_graph.cgi
Disallow: /cgi-bin/VFs/comp_malign.cgi
Disallow: /cgi-bin/VFs/compvfs_save.cgi
Disallow: /cgi-bin/VFs/showhtml.cgi
Disallow: /cgi-bin/VFs/showfig.cgi
Disallow: /ShiBASE/quick_guide.htm
Disallow: /ShiBASE/CGH.htm
Disallow: /ShiBASE/Orth_order.htm
Disallow: /ShiBASE/circular_Sd197.htm
Disallow: /ShiBASE/circular_Sf301.htm
Disallow: /ShiBASE/circular_Sb227.htm
Disallow: /ShiBASE/circular_Ss046.htm
Disallow: /ShiBASE/howto.htm
Disallow: /ShiBASE/Blast.htm
Disallow: /ShiBASE/Shi_align.htm
Disallow: /ShiBASE/general.htm
Disallow: /ShiBASE/IS_elements.htm
Disallow: /ShiBASE/fatalerror.htm
Disallow: /ShiBASE/Search.htm
Disallow: /ShiBASE/nt_comparison.htm
Disallow: /ShiBASE/comparison/
Disallow: /ShiBASE/Circular/
Disallow: /ShiBASE/ClustalTree.class
Disallow: /cgi-bin/ShiBASE
Disallow: /TrED/classif.html
Disallow: /TrED/search.html
Disallow: /TrED/path.html
Disallow: /TrED/array.html
Disallow: /TrED/blast.html
Disallow: /TrED/data.html
Disallow: /TrED/development.html
Disallow: /TrED/view.html
Disallow: /TrED/download/
Disallow: /TrED/applet/
Disallow: /cgi-bin/TrED