1.0Lukas Wojcik - Bloghttps://www.lukaswojcik.com/bloglukyhttps://www.lukaswojcik.com/blog/author/luky/rich600338<blockquote class="wp-embedded-content" data-secret="80yo2gcqgT"><a href="https://www.lukaswojcik.com/blog/en/toolbox/robots-txt-crawler-matrix/">robots.txt Crawler Matrix: which rule wins for which bot</a></blockquote><iframe sandbox="allow-scripts" security="restricted" src="https://www.lukaswojcik.com/blog/en/toolbox/robots-txt-crawler-matrix/embed/#?secret=80yo2gcqgT" width="600" height="338" title="“robots.txt Crawler Matrix: which rule wins for which bot” — Lukas Wojcik - Blog" data-secret="80yo2gcqgT" frameborder="0" marginwidth="0" marginheight="0" scrolling="no" class="wp-embedded-content"></iframe><script> /*! This file is auto-generated */ !function(d,l){"use strict";l.querySelector&&d.addEventListener&&"undefined"!=typeof URL&&(d.wp=d.wp||{},d.wp.receiveEmbedMessage||(d.wp.receiveEmbedMessage=function(e){var t=e.data;if((t||t.secret||t.message||t.value)&&!/[^a-zA-Z0-9]/.test(t.secret)){for(var s,r,n,a=l.querySelectorAll('iframe[data-secret="'+t.secret+'"]'),o=l.querySelectorAll('blockquote[data-secret="'+t.secret+'"]'),c=new RegExp("^https?:$","i"),i=0;i<o.length;i++)o[i].style.display="none";for(i=0;i<a.length;i++)s=a[i],e.source===s.contentWindow&&(s.removeAttribute("style"),"height"===t.message?(1e3<(r=parseInt(t.value,10))?r=1e3:~~r<200&&(r=200),s.height=r):"link"===t.message&&(r=new URL(s.getAttribute("src")),n=new URL(t.value),c.test(n.protocol))&&n.host===r.host&&l.activeElement===s&&(d.top.location.href=t.value))}},d.addEventListener("message",d.wp.receiveEmbedMessage,!1),l.addEventListener("DOMContentLoaded",function(){for(var e,t,s=l.querySelectorAll("iframe.wp-embedded-content"),r=0;r<s.length;r++)(t=(e=s[r]).getAttribute("data-secret"))||(t=Math.random().toString(36).substring(2,12),e.src+="#?secret="+t,e.setAttribute("data-secret",t)),e.contentWindow.postMessage({message:"ready",secret:t},"*")},!1)))}(window,document); //# sourceURL=https://www.lukaswojcik.com/blog/wp-includes/js/wp-embed.min.js </script> Reads a site's robots.txt, resolves every user agent to the group that governs it, and shows for each path whether crawling is allowed and which rule decided it. Includes the part most summaries get wrong: what a 404, a 410 or a 500 on the rules file itself means for a crawler.https://www.lukaswojcik.com/blog/wp-content/uploads/2026/08/og-default.jpg1200630