public class RobotsDirectives extends Object implements Serializable
Modifier and Type | Field and Description |
---|---|
protected ConcurrentSkipListSet<String> |
allows |
protected float |
crawlDelay |
protected ConcurrentSkipListSet<String> |
disallows |
Constructor and Description |
---|
RobotsDirectives() |
Modifier and Type | Method and Description |
---|---|
void |
addAllow(String path) |
void |
addDisallow(String path) |
boolean |
allows(String path) |
static void |
autoregisterTo(AutoKryo kryo) |
float |
getCrawlDelay() |
protected int |
longestPrefixLength(ConcurrentSkipListSet<String> prefixSet,
String str) |
void |
setCrawlDelay(float i) |
protected ConcurrentSkipListSet<String> disallows
protected ConcurrentSkipListSet<String> allows
protected float crawlDelay
public boolean allows(String path)
protected int longestPrefixLength(ConcurrentSkipListSet<String> prefixSet, String str)
prefixSet
- str
- prefixSet
that prefixes str
, or zero
if no entry prefixes str
public void addDisallow(String path)
public void addAllow(String path)
public void setCrawlDelay(float i)
public float getCrawlDelay()
public static void autoregisterTo(AutoKryo kryo)
Copyright © 2003-2014 Internet Archive. All Rights Reserved.