gov.uk valuation and analysis

User-agent: *
Disallow: /*/print$
# Don't allow indexing of user needs pages
Disallow: /info/*
Sitemap: https://www.gov.uk/sitemap.xml
# https://ahrefs.com/robot/ crawls the site frequently
User-agent: AhrefsBot
Crawl-delay: 10
# https://www.deepcrawl.com/bot/ makes lots of requests. Ideally
# we'd slow it down rather than blocking it but it doesn't mention
# whether or not it supports crawl-delay.
User-agent: deepcrawl
Disallow: /
# Complaints of 429 'Too many requests' seem to be coming from SharePoint servers
# (https://social.msdn.microsoft.com/Forums/en-US/3ea268ed-58a6-4166-ab40-d3f4fc55fef4)
# The robot doesn't recognise its User-Agent string, see the MS support article:
# https://support.microsoft.com/en-us/help/3019711/the-sharepoint-server-crawler-ignores-directives-in-robots-txt
User-agent: MS Search 6.0 Robot
Disallow: /
# Google's crawler was sending requests for each variation of query param for the sectors page of licence-finder

gov.uk has Semrush global rank of 383,422. gov.uk has an estimated worth of US$ 27,604,841, based on its estimated Ads revenue. gov.uk receives approximately 3,185,174 unique visitors each day. Its web server is located in United States, with IP address 151.101.0.144. According to SiteAdvisor, gov.uk is safe to visit.

HTTP/1.1 301 Moved Permanently
Server: Varnish
Retry-After: 0
Location: https://gov.uk/
Fastly-Backend-Name: force_ssl
Content-Length: 0
Accept-Ranges: bytes
Date: Sat, 23 Oct 2021 16:26:06 GMT
X-Varnish: 3728459309
Via: 1.1 varnish
Connection: close

HTTP/2 301 
server: Varnish
retry-after: 0
location: https://www.gov.uk/
strict-transport-security: max-age=63072000; preload
accept-ranges: bytes
date: Sat, 23 Oct 2021 16:26:06 GMT
x-varnish: 3621910968
via: 1.1 varnish
content-length: 0

HTTP/2 200 
content-type: text/html; charset=utf-8
server: nginx
cache-control: max-age=1800, public
content-security-policy-report-only: default-src https: 'self' *.publishing.service.gov.uk www.gov.uk *.dev.gov.uk; img-src 'self' data: *.publishing.service.gov.uk www.gov.uk *.dev.gov.uk www.google-analytics.com ssl.google-analytics.com stats.g.doubleclick.net www.googletagmanager.com assets.digital.cabinet-office.gov.uk lux.speedcurve.com; script-src 'self' *.publishing.service.gov.uk www.gov.uk *.dev.gov.uk www.google-analytics.com ssl.google-analytics.com stats.g.doubleclick.net www.googletagmanager.com www.gstatic.com www.signin.service.gov.uk *.ytimg.com www.youtube.com www.youtube-nocookie.com hmrc-uk.digital.nuance.com 'unsafe-inline'; style-src 'self' *.publishing.service.gov.uk www.gov.uk *.dev.gov.uk www.gstatic.com 'unsafe-inline'; font-src 'self' *.publishing.service.gov.uk www.gov.uk *.dev.gov.uk data:; connect-src 'self' *.publishing.service.gov.uk www.gov.uk *.dev.gov.uk www.google-analytics.com ssl.google-analytics.com stats.g.doubleclick.net www.googletagmanager.com www.tax.service.gov.uk hmrc-uk.digital.nuance.com hmpowebchat.klick2contact.com www.signin.service.gov.uk lux.speedcurve.com; object-src 'none'; frame-src 'self' *.publishing.service.gov.uk www.gov.uk *.dev.gov.uk www.youtube.com www.youtube-nocookie.com; report-uri https://jhpno0hk6b.execute-api.eu-west-2.amazonaws.com/production
etag: W/"3ebabf762d2b2efc3e1af1a41561d17b"
link: ; rel=preload; as=style; nopush,; rel=preload; as=style; nopush,; rel=preload; as=script; nopush
permissions-policy: interest-cohort=()
strict-transport-security: max-age=31536000; preload
via: 1.1 router, 1.1 varnish, 1.1 varnish
x-frame-options: ALLOWALL
x-request-id: 5b8cfe6a-496f-4693-82f0-006544936040
fastly-backend-name: origin
accept-ranges: bytes
date: Sat, 23 Oct 2021 16:26:07 GMT
age: 1012
x-served-by: cache-chi21172-CHI
x-cache: HIT, MISS
x-cache-hits: 0
x-timer: S1635006367.695714,VS0,VE654
vary: Accept-Encoding, Accept-Encoding
content-length: 81589