public class QuotaEnforcer extends Processor
Modifier and Type | Field and Description |
---|---|
protected Frontier |
frontier |
protected static int |
GROUP |
protected static int |
HOST |
protected static String[][] |
keys |
protected static int |
RESPONSE_KB |
protected static int |
RESPONSES |
protected static int |
SERVER |
protected ServerCache |
serverCache |
protected static int |
SUCCESS_KB |
protected static int |
SUCCESSES |
Constructor and Description |
---|
QuotaEnforcer() |
Modifier and Type | Method and Description |
---|---|
protected boolean |
applyQuota(CrawlURI curi,
String key,
long actual)
Apply the quota specified by the given key against the actual
value provided.
|
protected boolean |
checkQuotas(CrawlURI curi,
FetchStats.HasFetchStats hasStats,
int CAT)
Check all quotas for the given substats and category (server, host, or
group).
|
boolean |
getForceRetire() |
Frontier |
getFrontier() |
long |
getGroupMaxAllKb() |
long |
getGroupMaxFetchResponses() |
long |
getGroupMaxFetchSuccesses() |
long |
getGroupMaxSuccessKb() |
long |
getHostMaxAllKb() |
long |
getHostMaxFetchResponses() |
long |
getHostMaxFetchSuccesses() |
long |
getHostMaxSuccessKb() |
ServerCache |
getServerCache() |
long |
getServerMaxAllKb() |
long |
getServerMaxFetchResponses() |
long |
getServerMaxFetchSuccesses() |
long |
getServerMaxSuccessKb() |
protected void |
innerProcess(CrawlURI puri)
Actually performs the process.
|
protected ProcessResult |
innerProcessResult(CrawlURI puri) |
void |
setForceRetire(boolean force) |
void |
setFrontier(Frontier frontier) |
void |
setGroupMaxAllKb(long max) |
void |
setGroupMaxFetchResponses(long max) |
void |
setGroupMaxFetchSuccesses(long max) |
void |
setGroupMaxSuccessKb(long max) |
void |
setHostMaxAllKb(long max) |
void |
setHostMaxFetchResponses(long max) |
void |
setHostMaxFetchSuccesses(long max) |
void |
setHostMaxSuccessKb(long max) |
void |
setServerCache(ServerCache serverCache) |
void |
setServerMaxAllKb(long max) |
void |
setServerMaxFetchResponses(long max) |
void |
setServerMaxFetchSuccesses(long max) |
void |
setServerMaxSuccessKb(long max) |
protected boolean |
shouldProcess(CrawlURI puri)
Determines whether the given uri should be processed by this
processor.
|
doCheckpoint, finishCheckpoint, flattenVia, fromCheckpointJson, getBeanName, getEnabled, getKeyedProperties, getRecordedSize, getShouldProcessRule, getURICount, hasHttpAuthenticationCredential, innerRejectProcess, isRunning, isSuccess, process, report, setBeanName, setEnabled, setRecoveryCheckpoint, setShouldProcessRule, start, startCheckpoint, stop, toCheckpointJson
protected static final int SERVER
protected static final int HOST
protected static final int GROUP
protected static final int SUCCESSES
protected static final int SUCCESS_KB
protected static final int RESPONSES
protected static final int RESPONSE_KB
protected static final String[][] keys
protected ServerCache serverCache
protected Frontier frontier
public long getServerMaxFetchSuccesses()
public void setServerMaxFetchSuccesses(long max)
public long getServerMaxSuccessKb()
public void setServerMaxSuccessKb(long max)
public long getServerMaxFetchResponses()
public void setServerMaxFetchResponses(long max)
public long getServerMaxAllKb()
public void setServerMaxAllKb(long max)
public long getHostMaxFetchSuccesses()
public void setHostMaxFetchSuccesses(long max)
public long getHostMaxSuccessKb()
public void setHostMaxSuccessKb(long max)
public long getHostMaxFetchResponses()
public void setHostMaxFetchResponses(long max)
public long getHostMaxAllKb()
public void setHostMaxAllKb(long max)
public long getGroupMaxFetchSuccesses()
public void setGroupMaxFetchSuccesses(long max)
public long getGroupMaxSuccessKb()
public void setGroupMaxSuccessKb(long max)
public long getGroupMaxFetchResponses()
public void setGroupMaxFetchResponses(long max)
public long getGroupMaxAllKb()
public void setGroupMaxAllKb(long max)
public boolean getForceRetire()
public void setForceRetire(boolean force)
public ServerCache getServerCache()
public void setServerCache(ServerCache serverCache)
public Frontier getFrontier()
public void setFrontier(Frontier frontier)
protected boolean shouldProcess(CrawlURI puri)
Processor
shouldProcess
in class Processor
puri
- the URI to testprotected void innerProcess(CrawlURI puri)
Processor
#ENABLED
, the
#DECIDE_RULES
and the #shouldProcess(ProcessorURI)
tests.innerProcess
in class Processor
puri
- the URI to processprotected ProcessResult innerProcessResult(CrawlURI puri)
innerProcessResult
in class Processor
protected boolean checkQuotas(CrawlURI curi, FetchStats.HasFetchStats hasStats, int CAT)
curi
- CrawlURI to mark up with resultshasStats
- holds CrawlSubstats with actual values to testCAT
- category index (SERVER, HOST, GROUP) to quota settings keysprotected boolean applyQuota(CrawlURI curi, String key, long actual)
curi
- CrawlURI whose processing is subject to a potential quota
limitationquotaKey
- settings key to get applicable quotaactual
- current value to compare to quotaCopyright © 2003-2014 Internet Archive. All Rights Reserved.