# robots.txt

#User-agent: Mediapartners-Google
#Disallow:

#
User-agent: *
Disallow: /cgi-bin/
Disallow: /search
Disallow: /goto/
Disallow: /commodity-price-charts?1*
Disallow: /_private/
Disallow: /_vti_bin/
Disallow: /_vti_cnf/
Disallow: /_vti_log/
Disallow: /_vti_pvt/
Disallow: /_vti_txt/
Disallow: /newsletters/
Disallow: /feed/3
Disallow: /newsfeeds/redirect/
Disallow: /salestool
Disallow: /salestool/
Disallow: /articles/sponsored
Disallow: /articles/sponsored/
Disallow: /auth/*
Disallow: /account/*
Sitemap: https://oilprice.com/sitemap_index.xml
Sitemap: https://oilprice.com/sitemap.xml
Sitemap: https://oilprice.com/googlenews.xml

# Original rules for old site
Disallow: /administrator/
Disallow: /cache/
Disallow: /components/
Disallow: /component/
Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /libraries/
Disallow: /media/
Disallow: /modules/
Disallow: /plugins/
Disallow: /templates/
Disallow: /tmp/
Disallow: /xmlrpc/

Disallow: /ajax/csrf
Disallow: /ajax/add_view/*
Disallow: /ajax/return_country
Disallow: /widgets/*
Disallow: /freewidgets/*
Disallow: /opc/*
Disallow: /ajax/get_banner/*
Disallow: /ajax/get_user_blends
Disallow: /ajax/add_pushcrew
Disallow: /ajax/*

#mobile usability problems
#Urgent SEO issues - please confirm receipt 15/02/2017
Disallow: /ajax/get_chart_prices/
Disallow: /commodity-widgets.php
Disallow: /iframes/barcharts
Disallow: /market-intelligence/signup_success/6
Disallow: /newwidgets.php?3
Disallow: /specialreports/8_Mega_Trends.html
Disallow: /widgets/energyproduction.html

Disallow: /*data.json$
Disallow: /*data.json?t=*

Disallow: /*last.json$
Disallow: /*last.json?t=*

Disallow: /Energy/Energy-General/The-Gas-Find-That-Could-Transform-Europes-Energy-Future.html
Disallow: /Energy/Energy-General/This-Could-Be-A-Gamechanger-For-Natural-Gas-In-Europe.html
Disallow: /Energy/Energy-General/How-To-Profit-From-Europes-800-Billion-Energy-Crisis.html
Disallow: /Energy/Energy-General/The-No1-Energy-Stock-for-2024.html
Disallow: /Energy/Energy-General/Forgotten-Gas-Reserves-Could-Be-A-Gamechanger-For-European-Energy.html
Disallow: /Energy/Energy-General/Meet-the-Man-Using-AI-To-Revive-An-Oil-Gas-Play-Supermajors-Left-For-Dead.html
Disallow: /Energy/Natural-Gas/How-Artificial-Intelligence-Could-Trigger-a-Natural-Gas-Boom-in-Europe.html


# translated category/subcategory pages which have no articles on them so have the "wrong" langugue tag
Disallow: /es/Metales/Plata/
Disallow: /ar/%D8%A7%D9%84%D9%85%D8%B9%D8%A7%D8%AF%D9%86/%D9%81%D8%B6%D8%A9/
Disallow: /fr/M%C3%A9taux/Argent/
Disallow: /zh/%E9%87%91%E5%B1%9E/%E9%93%B6%E8%89%B2/
Disallow: /jp/%E9%87%91%E5%B1%9E/%E3%82%B7%E3%83%AB%E3%83%90%E3%83%BC/
Disallow: /es/Platino/
Disallow: /ar/%D8%A7%D9%84%D8%A8%D9%84%D8%A7%D8%AA%D9%8A%D9%86/
Disallow: /fr/Le-platine/
Disallow: /zh/%E7%99%BD%E9%87%91/
Disallow: /jp/%E3%83%97%E3%83%A9%E3%83%81%E3%83%8A/
Disallow: /es/Alimentos/
Disallow: /ar/%D9%85%D9%88%D8%A7%D8%AF-%D8%BA%D8%B0%D8%A7%D8%A6%D9%8A%D8%A9/
Disallow: /fr/Denr%C3%A9es-alimentaires/
Disallow: /zh/%E9%A3%9F%E5%93%81/
Disallow: /jp/%E9%A3%9F%E6%96%99%E5%93%81-%E3%81%97%E3%82%87%E3%81%8F%E3%82%8A%E3%82%87%E3%81%86%E3%81%B2%E3%82%93/
Disallow: /es/This-translation-does-not-provide-enough-context-to-accurately-translate-Could-you-please-provide-more-information-or-clarify-the-content-youd-like-to-be-translated/
Disallow: /ar/%D8%A7%D9%84%D9%86%D8%AD%D8%A7%D8%B3/
Disallow: /fr/Cuivre/
Disallow: /zh/%E9%93%9C/
Disallow: /jp/%E9%8A%85/
Disallow: /es/Energ%C3%ADa-alternativa/Celdas-de-combustible/
Disallow: /ar/%D8%A7%D9%84%D8%B7%D8%A7%D9%82%D8%A9-%D8%A7%D9%84%D9%85%D8%AA%D8%AC%D8%AF%D8%AF%D8%A9/%D8%AE%D9%84%D8%A7%D9%8A%D8%A7-%D8%A7%D9%84%D9%88%D9%82%D9%88%D8%AF/
Disallow: /fr/%C3%89nergie-renouvelable/Piles-%C3%A0-combustible/
Disallow: /zh/%E5%8F%AF%E5%86%8D%E7%94%9F%E8%83%BD%E6%BA%90/%E7%87%83%E6%96%99%E7%94%B5%E6%B1%A0/
Disallow: /jp/%E4%BB%A3%E6%9B%BF%E3%82%A8%E3%83%8D%E3%83%AB%E3%82%AE%E3%83%BC/%E7%87%83%E6%96%99%E9%9B%BB%E6%B1%A0/
Disallow: /es/Energ%C3%ADa-alternativa/Energ%C3%ADa-Mareomotriz/
Disallow: /ar/%D8%A7%D9%84%D8%B7%D8%A7%D9%82%D8%A9-%D8%A7%D9%84%D9%85%D8%AA%D8%AC%D8%AF%D8%AF%D8%A9/%D8%B7%D8%A7%D9%82%D8%A9-%D9%85%D8%AF-%D8%A7%D9%84%D8%AA%D9%8A%D8%A7%D8%B1%D8%A7%D8%AA/
Disallow: /fr/%C3%89nergie-renouvelable/%C3%89nergie-mar%C3%A9motrice/
Disallow: /zh/%E5%8F%AF%E5%86%8D%E7%94%9F%E8%83%BD%E6%BA%90/%E6%BD%AE%E6%B1%90%E8%83%BD%E9%87%8F/
Disallow: /jp/%E4%BB%A3%E6%9B%BF%E3%82%A8%E3%83%8D%E3%83%AB%E3%82%AE%E3%83%BC/%E6%BD%AE%E5%8A%9B%E3%82%A8%E3%83%8D%E3%83%AB%E3%82%AE%E3%83%BC/
Disallow: /es/El-Medio-Ambiente/Derrames-de-petr%C3%B3leo/
Disallow: /ar/%D8%A7%D9%84%D8%A8%D9%8A%D8%A6%D8%A9/%D8%AA%D8%B3%D8%B1%D8%A8-%D8%A7%D9%84%D9%86%D9%81%D8%B7/
Disallow: /fr/LEnvironnement/D%C3%A9versements-de-p%C3%A9trole/
Disallow: /zh/%E7%8E%AF%E5%A2%83/%E7%9F%B3%E6%B2%B9%E6%B3%84%E6%BC%8F/
Disallow: /jp/%E7%92%B0%E5%A2%83-Kanky%C5%8D/%E7%9F%B3%E6%B2%B9%E6%B5%81%E5%87%BA%E4%BA%8B%E6%95%85/
Disallow: /es/Geopol%C3%ADtica/Indonesia/
Disallow: /ar/%D8%A7%D9%84%D8%AC%D8%BA%D8%B1%D8%A7%D9%81%D9%8A%D8%A7-%D8%A7%D9%84%D8%B3%D9%8A%D8%A7%D8%B3%D9%8A%D8%A9/%D8%A5%D9%86%D8%AF%D9%88%D9%86%D9%8A%D8%B3%D9%8A%D8%A7/
Disallow: /fr/G%C3%A9opolitique/LIndon%C3%A9sie/
Disallow: /zh/%E5%9C%B0%E7%BC%98%E6%94%BF%E6%B2%BB/%E5%8D%B0%E5%BA%A6%E5%B0%BC%E8%A5%BF%E4%BA%9A/
Disallow: /jp/%E5%9C%B0%E6%94%BF%E5%AD%A6/%E3%82%A4%E3%83%B3%E3%83%89%E3%83%8D%E3%82%B7%E3%82%A2/
Disallow: /es/Geopol%C3%ADtica/Am%C3%A9rica-del-Norte/
Disallow: /ar/%D8%A7%D9%84%D8%AC%D8%BA%D8%B1%D8%A7%D9%81%D9%8A%D8%A7-%D8%A7%D9%84%D8%B3%D9%8A%D8%A7%D8%B3%D9%8A%D8%A9/%D8%A3%D9%85%D8%B1%D9%8A%D9%83%D8%A7-%D8%A7%D9%84%D8%B4%D9%85%D8%A7%D9%84%D9%8A%D8%A9/
Disallow: /fr/G%C3%A9opolitique/Am%C3%A9rique-du-Nord/
Disallow: /zh/%E5%9C%B0%E7%BC%98%E6%94%BF%E6%B2%BB/%E5%8C%97%E7%BE%8E%E6%B4%B2/
Disallow: /jp/%E5%9C%B0%E6%94%BF%E5%AD%A6/%E5%8C%97%E3%82%A2%E3%83%A1%E3%83%AA%E3%82%AB/
Disallow: /es/Geopol%C3%ADtica/Sudam%C3%A9rica/
Disallow: /ar/%D8%A7%D9%84%D8%AC%D8%BA%D8%B1%D8%A7%D9%81%D9%8A%D8%A7-%D8%A7%D9%84%D8%B3%D9%8A%D8%A7%D8%B3%D9%8A%D8%A9/%D8%A3%D9%85%D8%B1%D9%8A%D9%83%D8%A7-%D8%A7%D9%84%D8%AC%D9%86%D9%88%D8%A8%D9%8A%D8%A9/
Disallow: /fr/G%C3%A9opolitique/Am%C3%A9rique-du-Sud/
Disallow: /zh/%E5%9C%B0%E7%BC%98%E6%94%BF%E6%B2%BB/%E5%8D%97%E7%BE%8E%E6%B4%B2/
Disallow: /jp/%E5%9C%B0%E6%94%BF%E5%AD%A6/%E5%8D%97%E3%82%A2%E3%83%A1%E3%83%AA%E3%82%AB/
Disallow: /es/Finanzas/Informes-de-Inversi%C3%B3n-y-Trading/
Disallow: /ar/%D8%A7%D9%84%D9%85%D8%A7%D9%84%D9%8A%D8%A9/%D8%AA%D9%82%D8%A7%D8%B1%D9%8A%D8%B1-%D8%A7%D9%84%D8%A7%D8%B3%D8%AA%D8%AB%D9%85%D8%A7%D8%B1-%D9%88%D8%A7%D9%84%D8%AA%D8%AF%D8%A7%D9%88%D9%84/
Disallow: /fr/%C3%89conomie/Rapports-sur-lInvestissement-et-le-Trading/
Disallow: /zh/%E8%B4%A2%E7%BB%8F/%E6%8A%95%E8%B5%84%E4%B8%8E%E4%BA%A4%E6%98%93%E6%8A%A5%E5%91%8A/
Disallow: /jp/%E8%B2%A1%E5%8B%99/%E6%8A%95%E8%B3%87%E3%81%A8%E5%8F%96%E5%BC%95%E3%83%AC%E3%83%9D%E3%83%BC%E3%83%88/


User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
#User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: ClaudeWeb
User-agent: cohere-ai
User-agent: Diffbot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: Google-Extended
User-agent: GPTBot
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: Meta-ExternalAgent
#User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
#User-agent: PerplexityBot
User-agent: Scrapy
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: YouBot
Disallow: /