public abstract class SetBasedUriUniqFilter extends Object implements UriUniqFilter, Serializable
UriUniqFilter.CrawlUriReceiver
Modifier and Type | Field and Description |
---|---|
protected long |
duplicateCount |
protected long |
duplicatesAtLastSample |
protected PrintWriter |
profileLog |
protected UriUniqFilter.CrawlUriReceiver |
receiver |
Constructor and Description |
---|
SetBasedUriUniqFilter() |
Modifier and Type | Method and Description |
---|---|
void |
add(String key,
CrawlURI value)
Add given uri, if not already present.
|
void |
addForce(String key,
CrawlURI value)
Add given uri, all the way through to underlying destination, even
if already present.
|
void |
addNow(String key,
CrawlURI value)
Immediately add uri.
|
void |
close()
Close down any allocated resources.
|
long |
count() |
void |
forget(String key,
CrawlURI value)
Forget item was seen
|
void |
note(String key)
Note item as seen, without passing through to receiver.
|
long |
pending()
Count of items added, but not yet filtered in or out.
|
protected void |
profileLog(String key) |
long |
requestFlush()
Request that any pending items be added/dropped.
|
protected abstract boolean |
setAdd(CharSequence key) |
protected abstract long |
setCount() |
void |
setDestination(UriUniqFilter.CrawlUriReceiver receiver)
Receiver of uniq URIs.
|
void |
setProfileLog(File logfile)
Set a File to receive a log for replay profiling.
|
protected abstract boolean |
setRemove(CharSequence key) |
protected UriUniqFilter.CrawlUriReceiver receiver
protected PrintWriter profileLog
protected long duplicateCount
protected long duplicatesAtLastSample
protected abstract boolean setAdd(CharSequence key)
protected abstract boolean setRemove(CharSequence key)
protected abstract long setCount()
public long count()
count
in interface UriUniqFilter
public long pending()
UriUniqFilter
pending
in interface UriUniqFilter
public void setDestination(UriUniqFilter.CrawlUriReceiver receiver)
UriUniqFilter
setDestination
in interface UriUniqFilter
receiver
- Object that will be passed items. Must implement
HasUriReceiver interface.protected void profileLog(String key)
public void add(String key, CrawlURI value)
UriUniqFilter
add
in interface UriUniqFilter
key
- Usually a canonicalized version of value
.
This is the key used doing lookups, forgets and insertions on the
already included list.value
- item to add.public void addNow(String key, CrawlURI value)
UriUniqFilter
addNow
in interface UriUniqFilter
key
- Usually a canonicalized version of uri
.
This is the key used doing lookups, forgets and insertions on the
already included list.value
- item to add.public void addForce(String key, CrawlURI value)
UriUniqFilter
addForce
in interface UriUniqFilter
key
- Usually a canonicalized version of uri
.
This is the key used doing lookups, forgets and insertions on the
already included list.value
- item to add.public void note(String key)
UriUniqFilter
note
in interface UriUniqFilter
key
- Usually a canonicalized version of an URI
.
This is the key used doing lookups, forgets and insertions on the
already included list.public void forget(String key, CrawlURI value)
UriUniqFilter
forget
in interface UriUniqFilter
key
- Usually a canonicalized version of an URI
.
This is the key used doing lookups, forgets and insertions on the
already included list.value
- item to add.public long requestFlush()
UriUniqFilter
requestFlush
in interface UriUniqFilter
public void close()
UriUniqFilter
close
in interface UriUniqFilter
public void setProfileLog(File logfile)
UriUniqFilter
setProfileLog
in interface UriUniqFilter
Copyright © 2003-2014 Internet Archive. All Rights Reserved.