(define-constraint-type HEADER-ROBOTS-ALLOWED
(:header
:documentation "Succeeds when robots are allowed on the URI host.")
(constraint activity url)
(ecase (robot-exclusion-status activity url)
(:excluded nil)
(:allowed t)
(:unknown
(let ((exclusion-url (robot-exclusion-url url)))
(multiple-value-bind (headers status-code)
(get-resource-headers activity exclusion-url)
(case status-code
(404 (note-robot-exclusion-status
activity (host-object url) :allowed)
t)
(t (note-robot-exclusion-status
activity (host-object url) :excluded)
nil)))))))
| Previous slide | Next slide | Back to the first slide | View Graphic Version |