# ============================================================================ # robots.txt - store code: glmx # # Structure: TWO groups. # Group A - technical rules. Every crawler obeys them, AI bots included. # Group B - allow-list + faceted-navigation limits. Search crawlers only. # # A crawler uses the most specific User-agent token that matches it, and # RFC 9309 2.2.1 merges every group declaring that same token. Googlebot, # bingbot and any unknown crawler match "*", which appears in BOTH groups, # so they get A + B. The AI bots match their own name, which appears in # group A only, so they get the technical rules but NOT the facet limits - # they read pages to answer questions, not to build a ranked index. # # DO NOT swap the group order and DO NOT drop the repeated "User-agent: *" # / "User-agent: adsbot-google" lines in group B. They look redundant and # are not: they are what makes the merge happen. # ============================================================================ Sitemap: https://www.glamira.com.mx/sitemap.xml # ============================================================================ # GROUP A - technical rules, for EVERY crawler (AI assistants included). # ============================================================================ User-agent: * User-agent: adsbot-google User-agent: ChatGPT-User User-agent: GPTBot User-agent: OAI-SearchBot User-agent: PerplexityBot User-agent: Google-Extended User-agent: ClaudeBot # -- 2. Admin & server internals --------------------------------------------- Disallow: /admin/ # DECISION NEEDED - security team (#23740 gate c). This line publishes the # live admin frontName in a world-readable file, which # .claude/rules/security-rules.md forbids. Kept here only so that this # restructure does not change behaviour. Recommended replacement: delete it # and block/noindex the admin path at nginx level instead. Deleting it # WITHOUT that nginx change would leave the admin login crawlable. Disallow: /secured2021/ Disallow: /app/ Disallow: /downloader/ Disallow: /errors/ Disallow: /includes/ Disallow: /lib/ Disallow: /pkginfo/ Disallow: /shell/ Disallow: /testcache/ Disallow: /var/ # Every PHP entry point - covers cron.php, get.php, health_check.php, # api.php, install.php, buildcache.php, so they need no separate lines. Disallow: /*.php$ Disallow: /cron.sh Disallow: /error_log Disallow: /rev.txt Disallow: /README.txt Disallow: /RELEASE_NOTES.txt Disallow: /LICENSE.html Disallow: /LICENSE.txt Disallow: /LICENSE_AFL.txt # -- 3. Non-content endpoints: REST, AJAX, cache ------------------------------ # # Each path appears twice: bare, and with a "/*/" prefix for store views # served from a sub-folder (/en/, /fr/, ...). "*" matches "/" too, but a # bare "/rest/" is anchored at the root so it does NOT cover "/en/rest/". # The pairs are kept adjacent so a missing one is obvious. # # The trailing "*" on every "/*/" rule is load-bearing: without it the rule # is 8 characters, ties with "Allow: /xx/*/*/" from group B, and a tie is # resolved in favour of Allow - which silently disabled /*/rest/, /*/awin/, # /*/gpt3/ and /*/sqzl/ in production. Do not remove it. # ---------------------------------------------------------------------------- Disallow: /rest/ Disallow: /*/rest/* Disallow: /page_cache/ Disallow: /*/page_cache/* Disallow: /page_cache_public/ Disallow: /*/page_cache_public/* Disallow: /relation/ Disallow: /*/relation/* Disallow: /review/ Disallow: /*/review/* Disallow: /megamenu/ Disallow: /*/megamenu/* Disallow: /bulk-request/ Disallow: /*/bulk-request/* Disallow: /catalog/product_compare/ Disallow: /*/catalog/product_compare/* Disallow: /override/catalog_ajax/recentviewed/ Disallow: /*/override/catalog_ajax/recentviewed/* Disallow: /sendfriend/product/send/ Disallow: /*/sendfriend/product/send/* Disallow: /custom/geoip/ Disallow: /*/custom/geoip/* Disallow: /captcha/ Disallow: /*/captcha/* Disallow: /delivery_date/ Disallow: /*/delivery_date/* Disallow: /promo_extend/ Disallow: /*/promo_extend/* Disallow: /data-faq/ Disallow: /*/data-faq/* # -- 4. Third-party integrations & tracking ----------------------------------- Disallow: /awin/ Disallow: /*/awin/* Disallow: /forter/ Disallow: /*/forter/* Disallow: /gpt3/ Disallow: /*/gpt3/* Disallow: /googletagmanager/ Disallow: /*/googletagmanager/* Disallow: /amcookie/ Disallow: /*/amcookie/* Disallow: /sqzl/ Disallow: /*/sqzl/* Disallow: /xcover/ Disallow: /*/xcover/* # -- 5. Checkout & customer account ------------------------------------------- # # "/customer/" already covers /customer/account/ and # /customer/account/login/, so those need no separate lines. # ---------------------------------------------------------------------------- Disallow: /checkout/ Disallow: /*/checkout/* Disallow: /customcheckout/ Disallow: /*/customcheckout/* Disallow: /onestepcheckout/ Disallow: /*/onestepcheckout/* Disallow: /customer/ Disallow: /*/customer/* # -- 6. URL parameters: session IDs and store switching ---------------------- # # These carry no catalog data and a fresh value can be minted on every # request, so they open an unbounded URL space. Blocked for every crawler, # AI assistants included - this is a crawler trap, not an indexing concern. # Display and sorting parameters live in group B instead. # ---------------------------------------------------------------------------- Disallow: /*SID= Disallow: /*___SID= Disallow: /*___store= Disallow: /*___from_store= # ============================================================================ # GROUP B - index hygiene. Search crawlers only, NOT the AI assistants. # Re-declares the same two tokens as group A on purpose (see file header). # ============================================================================ User-agent: * User-agent: adsbot-google # -- 1. Allow-list: resources crawlers MUST be able to fetch ----------------- # # These override every Disallow above. Google and Bing resolve conflicts by # LONGEST matching pattern, so each pattern here must be longer than the # generic facet rule in section 7 below. See the docstring in the builder: # the trailing segments scale with the facet depth of this store. # ---------------------------------------------------------------------------- Allow: /media/* Allow: /static/* Allow: /blog/*/*/ Allow: /*/blog/*/*/ Allow: /productcustomizer/reactdata/ Allow: /*/productcustomizer/reactdata/ # -- 6b. Display, sorting and pagination parameters -------------------------- # # Same catalog data in a different arrangement: duplicate content for a # search index, but still useful information for an AI assistant answering # a question about the catalogue. Kept in group B so the AI bots may fetch # them while Googlebot and bingbot - which merge A + B - still may not. # ---------------------------------------------------------------------------- Disallow: /*amp= Disallow: /*limit= Disallow: /*mode=grid Disallow: /*mode=list Disallow: /*order=relevance Disallow: /*product_list_order=price # -- 7. Faceted navigation: block category pages with 2+ filters ------------- # # URL shape: ////... # /////... # # "*" matches "/" as well, so "/*/*/*/" means "any path with 3+ segments". # Category + 1 filter = 2 segments (crawlable); + 2 filters = 3 (blocked). # A sub-folder store view shifts everything by one segment, so each locale # needs its own pair: Allow N segments (still 1 filter), Disallow N+1 # (2 filters). Longest match decides, so order inside this block does not # matter - the pairs are adjacent purely for readability. # # These pages also carry # from OpenTechiz_LayeredNavigationExtend (Observer/RuleRobots.php). Note # that while a URL is Disallow-ed here crawlers never fetch it and so never # see that meta tag - the rules below are what is actually in force. # ---------------------------------------------------------------------------- Disallow: /*/*/*/ Disallow: /ar/*/*/*/ Allow: /ar/*/*/ Disallow: /cn/*/*/*/ Allow: /cn/*/*/ Disallow: /en/*/*/*/ Allow: /en/*/*/ Disallow: /es/*/*/*/ Allow: /es/*/*/ Disallow: /fr/*/*/*/ Allow: /fr/*/*/ Disallow: /it/*/*/*/ Allow: /it/*/*/ Disallow: /my/*/*/*/ Allow: /my/*/*/ Disallow: /nl/*/*/*/ Allow: /nl/*/*/ Disallow: /pt/*/*/*/ Allow: /pt/*/*/ Disallow: /ru/*/*/*/ Allow: /ru/*/*/ Disallow: /tr/*/*/*/ Allow: /tr/*/*/ Disallow: /tw/*/*/*/ Allow: /tw/*/*/