public abstract class Scoper extends Processor implements org.springframework.context.Lifecycle
Modifier and Type | Field and Description |
---|---|
protected Logger |
fileLogger |
protected boolean |
isRunning |
protected CrawlerLoggerModule |
loggerModule |
protected DecideRule |
scope |
beanName, kp, recoveryCheckpoint, uriCount
Constructor and Description |
---|
Scoper()
Constructor.
|
Modifier and Type | Method and Description |
---|---|
CrawlerLoggerModule |
getLoggerModule() |
boolean |
getLogToFile() |
DecideRule |
getScope() |
protected boolean |
isInScope(CrawlURI caUri)
Schedule the given
CrawlURI with the Frontier. |
boolean |
isRunning() |
protected void |
outOfScope(CrawlURI caUri)
Called when a CrawlURI is ruled out of scope.
|
void |
setLoggerModule(CrawlerLoggerModule loggerModule) |
void |
setLogToFile(boolean enabled) |
void |
setScope(DecideRule scope) |
void |
start() |
void |
stop() |
doCheckpoint, finishCheckpoint, flattenVia, fromCheckpointJson, getBeanName, getEnabled, getKeyedProperties, getRecordedSize, getShouldProcessRule, getURICount, hasHttpAuthenticationCredential, innerProcess, innerProcessResult, innerRejectProcess, isSuccess, process, report, setBeanName, setEnabled, setRecoveryCheckpoint, setShouldProcessRule, shouldProcess, startCheckpoint, toCheckpointJson
protected DecideRule scope
protected Logger fileLogger
protected CrawlerLoggerModule loggerModule
protected boolean isRunning
public DecideRule getScope()
public void setScope(DecideRule scope)
public boolean getLogToFile()
public void setLogToFile(boolean enabled)
public CrawlerLoggerModule getLoggerModule()
public void setLoggerModule(CrawlerLoggerModule loggerModule)
public void start()
public boolean isRunning()
public void stop()
protected boolean isInScope(CrawlURI caUri)
CrawlURI
with the Frontier.caUri
- The CrawlURI to be scheduled.protected void outOfScope(CrawlURI caUri)
caUri
- CrawlURI that is out of scope.Copyright © 2003-2014 Internet Archive. All Rights Reserved.