项目：search
文件：CommonTermsQueryTest.java
/**
* populates a writer with random stuff. this must be fully reproducable with
* the seed!
*/
public static void createrandomIndex(int numdocs,RandomIndexWriter writer,long seed) throws IOException {
Random random = new Random(seed);
// primary source for our data is from linefiledocs,its realistic.
LineFileDocs lineFileDocs = new LineFileDocs(random,false); // no docvalues in 4x
// Todo: we should add other fields that use things like docs&freqs but omit
// positions,// because linefiledocs doesn't cover all the possibilities.
for (int i = 0; i < numdocs; i++) {
writer.addDocument(lineFileDocs.nextDoc());
}
lineFileDocs.close();
}
public void buildindex(Directory dir) throws IOException {
Random random = random();
MockAnalyzer analyzer = new MockAnalyzer(random());
analyzer.setMaxTokenLength(TestUtil.nextInt(random(),1,IndexWriter.MAX_TERM_LENGTH));
IndexWriterConfig config = newIndexWriterConfig(analyzer);
Similarity provider = new MySimProvider();
config.setSimilarity(provider);
RandomIndexWriter writer = new RandomIndexWriter(random,dir,config);
final LineFileDocs docs = new LineFileDocs(random,defaultCodecSupportsdocValues());
int num = atLeast(100);
for (int i = 0; i < num; i++) {
Document doc = docs.nextDoc();
int boost = random().nextInt(255);
Field f = new TextField(byteTestField,"" + boost,Field.Store.YES);
f.setBoost(boost);
doc.add(f);
writer.addDocument(doc);
doc.removeField(byteTestField);
if (rarely()) {
writer.commit();
}
}
writer.commit();
writer.close();
docs.close();
}
项目：NYBC
文件：CommonTermsQueryTest.java
/**
* populates a writer with random stuff. this must be fully reproducable with
* the seed!
*/
public static void createrandomIndex(int numdocs,// because linefiledocs doesn't cover all the possibilities.
for (int i = 0; i < numdocs; i++) {
writer.addDocument(lineFileDocs.nextDoc());
}
lineFileDocs.close();
}
public void buildindex(Directory dir) throws IOException {
Random random = random();
IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,new MockAnalyzer(random()));
Similarity provider = new MySimProvider();
config.setSimilarity(provider);
RandomIndexWriter writer = new RandomIndexWriter(random,Field.Store.YES);
f.setBoost(boost);
doc.add(f);
writer.addDocument(doc);
doc.removeField(byteTestField);
if (rarely()) {
writer.commit();
}
}
writer.commit();
writer.close();
docs.close();
}
项目：NYBC
文件：TestDuelingCodecs.java
/**
* populates a writer with random stuff. this must be fully reproducable with the seed!
*/
public static void createrandomIndex(int numdocs,its realistic.
LineFileDocs lineFileDocs = new LineFileDocs(random);
// Todo: we should add other fields that use things like docs&freqs but omit positions,// because linefiledocs doesn't cover all the possibilities.
for (int i = 0; i < numdocs; i++) {
Document document = lineFileDocs.nextDoc();
// grab the title and add some SortedSet instances for fun
String title = document.get("titletokenized");
String split[] = title.split("\\s+");
for (String trash : split) {
document.add(new SortedSetDocValuesField("sortedset",new BytesRef(trash)));
}
writer.addDocument(document);
}
lineFileDocs.close();
}
项目：Maskana-Gestor-de-Conocimiento
文件：CommonTermsQueryTest.java
/**
* populates a writer with random stuff. this must be fully reproducable with
* the seed!
*/
public static void createrandomIndex(int numdocs,// because linefiledocs doesn't cover all the possibilities.
for (int i = 0; i < numdocs; i++) {
writer.addDocument(lineFileDocs.nextDoc());
}
lineFileDocs.close();
}
public void buildindex(Directory dir) throws IOException {
Random random = random();
IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,Field.Store.YES);
f.setBoost(boost);
doc.add(f);
writer.addDocument(doc);
doc.removeField(byteTestField);
if (rarely()) {
writer.commit();
}
}
writer.commit();
writer.close();
docs.close();
}
项目：Maskana-Gestor-de-Conocimiento
文件：TestDuelingCodecs.java
/**
* populates a writer with random stuff. this must be fully reproducable with the seed!
*/
public static void createrandomIndex(int numdocs,new BytesRef(trash)));
}
// add a numeric dv field sometimes
document.removeFields("sparsenumeric");
if (random.nextInt(4) == 2) {
document.add(new NumericDocValuesField("sparsenumeric",random.nextInt()));
}
writer.addDocument(document);
}
lineFileDocs.close();
}
项目：elasticsearch_my
文件：TranslogTests.java
public void testTranslogopsCountIsCorrect() throws IOException {
List<Translog.Location> locations = new ArrayList<>();
int numOps = randomIntBetween(100,200);
LineFileDocs lineFileDocs = new LineFileDocs(random()); // writes pretty big docs so we cross buffer boarders regularly
for (int opsAdded = 0; opsAdded < numOps; opsAdded++) {
locations.add(translog.add(new Translog.Index("test","" + opsAdded,lineFileDocs.nextDoc().toString().getBytes(Charset.forName("UTF-8")))));
Translog.Snapshot snapshot = this.translog.newSnapshot();
assertEquals(opsAdded + 1,snapshot.totalOperations());
for (int i = 0; i < opsAdded; i++) {
assertEquals("expected operation" + i + " to be in the current translog but wasn't",translog.currentFileGeneration(),locations.get(i).generation);
Translog.Operation next = snapshot.next();
assertNotNull("operation " + i + " must be non-null",next);
}
}
}
public void testLineDocs() throws IOException {
Random r = random();
LineFileDocs lineFileDocs = new LineFileDocs(r);
for (int i = 0; i < 10; i++) {
int numDocs = TestUtil.nextInt(r,200);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
for (int j = 0; j < numDocs; j++) {
String s = lineFileDocs.nextDoc().get("body");
bos.write(s.getBytes(StandardCharsets.UTF_8));
}
doTest(bos.toByteArray());
}
lineFileDocs.close();
}
public void testLineDocsThreads() throws Exception {
final Random r = random();
int threadCount = TestUtil.nextInt(r,2,6);
Thread[] threads = new Thread[threadCount];
final CountDownLatch startingGun = new CountDownLatch(1);
for (int tid=0; tid < threadCount; tid++) {
final long seed = r.nextLong();
threads[tid] = new Thread() {
@Override
public void run() {
try {
Random r = new Random(seed);
startingGun.await();
LineFileDocs lineFileDocs = new LineFileDocs(r);
for (int i = 0; i < 10; i++) {
int numDocs = TestUtil.nextInt(r,200);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
for (int j = 0; j < numDocs; j++) {
String s = lineFileDocs.nextDoc().get("body");
bos.write(s.getBytes(StandardCharsets.UTF_8));
}
doTest(bos.toByteArray());
}
lineFileDocs.close();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
threads[tid].start();
}
startingGun.countDown();
for (Thread t : threads) {
t.join();
}
}
public void testMixed() throws IOException {
Random r = random();
LineFileDocs lineFileDocs = new LineFileDocs(r);
for (int i = 0; i < 2; ++i) {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
int prevInt = r.nextInt();
long prevLong = r.nextLong();
while (bos.size() < 400000) {
switch (r.nextInt(4)) {
case 0:
addInt(r,prevInt,bos);
break;
case 1:
addLong(r,prevLong,bos);
break;
case 2:
addString(lineFileDocs,bos);
break;
case 3:
addBytes(r,bos);
break;
default:
throw new IllegalStateException("Random is broken");
}
}
doTest(bos.toByteArray());
}
}
项目：search
文件：ShardSearchingTestBase.java
@Override
public void run() {
try {
final LineFileDocs docs = new LineFileDocs(random(),defaultCodecSupportsdocValues());
int numDocs = 0;
while (System.nanoTime() < endTimeNanos) {
final int what = random().nextInt(3);
final NodeState node = nodes[random().nextInt(nodes.length)];
if (numDocs == 0 || what == 0) {
node.writer.addDocument(docs.nextDoc());
numDocs++;
} else if (what == 1) {
node.writer.updateDocument(new Term("docid",""+random().nextInt(numDocs)),docs.nextDoc());
numDocs++;
} else {
node.writer.deleteDocuments(new Term("docid",""+random().nextInt(numDocs)));
}
// Todo: doc blocks too
if (random().nextInt(17) == 12) {
node.writer.commit();
}
if (random().nextInt(17) == 12) {
nodes[random().nextInt(nodes.length)].reopen();
}
}
} catch (Throwable t) {
System.out.println("Failed:");
t.printstacktrace(System.out);
throw new RuntimeException(t);
}
}
项目：search
文件：TestReuseDocsEnum.java
/**
* populates a writer with random stuff. this must be fully reproducable with
* the seed!
*/
public static void createrandomIndex(int numdocs,Random random) throws IOException {
LineFileDocs lineFileDocs = new LineFileDocs(random);
for (int i = 0; i < numdocs; i++) {
writer.addDocument(lineFileDocs.nextDoc());
}
lineFileDocs.close();
}
public void testFloatnorms() throws IOException {
Directory dir = newDirectory();
MockAnalyzer analyzer = new MockAnalyzer(random());
analyzer.setMaxTokenLength(TestUtil.nextInt(random(),IndexWriter.MAX_TERM_LENGTH));
IndexWriterConfig config = newIndexWriterConfig(analyzer);
Similarity provider = new MySimProvider();
config.setSimilarity(provider);
RandomIndexWriter writer = new RandomIndexWriter(random(),config);
final LineFileDocs docs = new LineFileDocs(random());
int num = atLeast(100);
for (int i = 0; i < num; i++) {
Document doc = docs.nextDoc();
float nextFloat = random().nextFloat();
Field f = new TextField(floatTestField,"" + nextFloat,Field.Store.YES);
f.setBoost(nextFloat);
doc.add(f);
writer.addDocument(doc);
doc.removeField(floatTestField);
if (rarely()) {
writer.commit();
}
}
writer.commit();
writer.close();
AtomicReader open = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
NumericDocValues norms = open.getnormValues(floatTestField);
assertNotNull(norms);
for (int i = 0; i < open.maxDoc(); i++) {
Document document = open.document(i);
float expected = Float.parseFloat(document.get(floatTestField));
assertEquals(expected,Float.intBitsToFloat((int)norms.get(i)),0.0f);
}
open.close();
dir.close();
docs.close();
}
项目：search
文件：TestFlushByRamOrCountsPolicy.java
public IndexThread(AtomicInteger pendingDocs,int numThreads,IndexWriter writer,LineFileDocs docs,boolean doRandomCommit) {
this.pendingDocs = pendingDocs;
this.writer = writer;
iwc = writer.getConfig();
this.docs = docs;
this.doRandomCommit = doRandomCommit;
}
项目：search
文件：TestDuelingCodecs.java
/**
* populates a writer with random stuff. this must be fully reproducable with the seed!
*/
public static void createrandomIndex(int numdocs,random.nextInt()));
}
// add sortednumeric sometimes
document.removeFields("sparsesortednum");
if (random.nextInt(5) == 1) {
document.add(new SortednumericDocValuesField("sparsesortednum",random.nextLong()));
if (random.nextBoolean()) {
document.add(new SortednumericDocValuesField("sparsesortednum",random.nextLong()));
}
}
writer.addDocument(document);
}
lineFileDocs.close();
}
项目：NYBC
文件：ShardSearchingTestBase.java
@Override
public void run() {
try {
final LineFileDocs docs = new LineFileDocs(random(),""+random().nextInt(numDocs)));
}
// Todo: doc blocks too
if (random().nextInt(17) == 12) {
node.writer.commit();
}
if (random().nextInt(17) == 12) {
nodes[random().nextInt(nodes.length)].reopen();
}
}
} catch (Throwable t) {
System.out.println("Failed:");
t.printstacktrace(System.out);
throw new RuntimeException(t);
}
}
项目：NYBC
文件：TestReuseDocsEnum.java
/**
* populates a writer with random stuff. this must be fully reproducable with
* the seed!
*/
public static void createrandomIndex(int numdocs,Random random) throws IOException {
LineFileDocs lineFileDocs = new LineFileDocs(random);
for (int i = 0; i < numdocs; i++) {
writer.addDocument(lineFileDocs.nextDoc());
}
lineFileDocs.close();
}
public void testFloatnorms() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,new MockAnalyzer(random()));
Similarity provider = new MySimProvider();
config.setSimilarity(provider);
RandomIndexWriter writer = new RandomIndexWriter(random(),0.0f);
}
open.close();
dir.close();
docs.close();
}
项目：NYBC
文件：TestFlushByRamOrCountsPolicy.java
public IndexThread(AtomicInteger pendingDocs,boolean doRandomCommit) {
this.pendingDocs = pendingDocs;
this.writer = writer;
iwc = writer.getConfig();
this.docs = docs;
this.doRandomCommit = doRandomCommit;
}
项目：Maskana-Gestor-de-Conocimiento
文件：ShardSearchingTestBase.java
@Override
public void run() {
try {
final LineFileDocs docs = new LineFileDocs(random(),""+random().nextInt(numDocs)));
}
// Todo: doc blocks too
if (random().nextInt(17) == 12) {
node.writer.commit();
}
if (random().nextInt(17) == 12) {
nodes[random().nextInt(nodes.length)].reopen();
}
}
} catch (Throwable t) {
System.out.println("Failed:");
t.printstacktrace(System.out);
throw new RuntimeException(t);
}
}
项目：Maskana-Gestor-de-Conocimiento
文件：TestReuseDocsEnum.java
/**
* populates a writer with random stuff. this must be fully reproducable with
* the seed!
*/
public static void createrandomIndex(int numdocs,Random random) throws IOException {
LineFileDocs lineFileDocs = new LineFileDocs(random);
for (int i = 0; i < numdocs; i++) {
writer.addDocument(lineFileDocs.nextDoc());
}
lineFileDocs.close();
}
项目：Maskana-Gestor-de-Conocimiento
文件：TestFlushByRamOrCountsPolicy.java
public IndexThread(AtomicInteger pendingDocs,boolean doRandomCommit) {
this.pendingDocs = pendingDocs;
this.writer = writer;
iwc = writer.getConfig();
this.docs = docs;
this.doRandomCommit = doRandomCommit;
}
private void addString(LineFileDocs lineFileDocs,ByteArrayOutputStream bos) throws IOException {
String s = lineFileDocs.nextDoc().get("body");
bos.write(s.getBytes(StandardCharsets.UTF_8));
}
项目：search
文件：TestFreeTextSuggester.java
@Ignore
public void testWiki() throws Exception {
final LineFileDocs lfd = new LineFileDocs(null,"/lucenedata/enwiki/enwiki-20120502-lines-1k.txt",false);
// Skip header:
lfd.nextDoc();
FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
sug.build(new InputIterator() {
private int count;
@Override
public long weight() {
return 1;
}
@Override
public Comparator<BytesRef> getComparator() {
return null;
}
@Override
public BytesRef next() {
Document doc;
try {
doc = lfd.nextDoc();
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
if (doc == null) {
return null;
}
if (count++ == 10000) {
return null;
}
return new BytesRef(doc.get("body"));
}
@Override
public BytesRef payload() {
return null;
}
@Override
public boolean hasPayloads() {
return false;
}
@Override
public Set<BytesRef> contexts() {
return null;
}
@Override
public boolean hasContexts() {
return false;
}
});
if (VERBOSE) {
System.out.println(sug.ramBytesUsed() + " bytes");
List<LookupResult> results = sug.lookup("general r",10);
System.out.println("results:");
for(LookupResult result : results) {
System.out.println(" " + result);
}
}
}
项目：search
文件：TestFlushByRamOrCountsPolicy.java
@BeforeClass
public static void beforeClass() throws Exception {
lineDocFile = new LineFileDocs(random(),defaultCodecSupportsdocValues());
}
项目：search
文件：TestNRTCachingDirectory.java
public void testNRTAndCommit() throws Exception {
Directory dir = newDirectory();
NRTCachingDirectory cachedDir = new NRTCachingDirectory(dir,2.0,25.0);
MockAnalyzer analyzer = new MockAnalyzer(random());
analyzer.setMaxTokenLength(TestUtil.nextInt(random(),IndexWriter.MAX_TERM_LENGTH));
IndexWriterConfig conf = newIndexWriterConfig(analyzer);
RandomIndexWriter w = new RandomIndexWriter(random(),cachedDir,conf);
final LineFileDocs docs = new LineFileDocs(random(),defaultCodecSupportsdocValues());
final int numDocs = TestUtil.nextInt(random(),100,400);
if (VERBOSE) {
System.out.println("TEST: numDocs=" + numDocs);
}
final List<BytesRef> ids = new ArrayList<>();
DirectoryReader r = null;
for(int docCount=0;docCount<numDocs;docCount++) {
final Document doc = docs.nextDoc();
ids.add(new BytesRef(doc.get("docid")));
w.addDocument(doc);
if (random().nextInt(20) == 17) {
if (r == null) {
r = DirectoryReader.open(w.w,false);
} else {
final DirectoryReader r2 = DirectoryReader.openIfChanged(r);
if (r2 != null) {
r.close();
r = r2;
}
}
assertEquals(1+docCount,r.numDocs());
final IndexSearcher s = newSearcher(r);
// Just make sure search can run; we can't assert
// totHits since it Could be 0
TopDocs hits = s.search(new TermQuery(new Term("body","the")),10);
// System.out.println("tot hits " + hits.totalHits);
}
}
if (r != null) {
r.close();
}
// Close should force cache to clear since all files are sync'd
w.close();
final String[] cachedFiles = cachedDir.listCachedFiles();
for(String file : cachedFiles) {
System.out.println("FAIL: cached file " + file + " remains after sync");
}
assertEquals(0,cachedFiles.length);
r = DirectoryReader.open(dir);
for(BytesRef id : ids) {
assertEquals(1,r.docFreq(new Term("docid",id)));
}
r.close();
cachedDir.close();
docs.close();
}
项目：NYBC
文件：TestFlushByRamOrCountsPolicy.java
@BeforeClass
public static void beforeClass() throws Exception {
lineDocFile = new LineFileDocs(random(),defaultCodecSupportsdocValues());
}
项目：NYBC
文件：TestNRTCachingDirectory.java
public void testNRTAndCommit() throws Exception {
Directory dir = newDirectory();
NRTCachingDirectory cachedDir = new NRTCachingDirectory(dir,25.0);
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,new MockAnalyzer(random()));
RandomIndexWriter w = new RandomIndexWriter(random(),defaultCodecSupportsdocValues());
final int numDocs = _TestUtil.nextInt(random(),400);
if (VERBOSE) {
System.out.println("TEST: numDocs=" + numDocs);
}
final List<BytesRef> ids = new ArrayList<BytesRef>();
DirectoryReader r = null;
for(int docCount=0;docCount<numDocs;docCount++) {
final Document doc = docs.nextDoc();
ids.add(new BytesRef(doc.get("docid")));
w.addDocument(doc);
if (random().nextInt(20) == 17) {
if (r == null) {
r = DirectoryReader.open(w.w,r.numDocs());
final IndexSearcher s = new IndexSearcher(r);
// Just make sure search can run; we can't assert
// totHits since it Could be 0
TopDocs hits = s.search(new TermQuery(new Term("body",id)));
}
r.close();
cachedDir.close();
docs.close();
}
项目：Maskana-Gestor-de-Conocimiento
文件：TestFreeTextSuggester.java
@Ignore
public void testWiki() throws Exception {
final LineFileDocs lfd = new LineFileDocs(null,false);
// Skip header:
lfd.nextDoc();
FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
sug.build(new InputIterator() {
private int count;
@Override
public long weight() {
return 1;
}
@Override
public Comparator<BytesRef> getComparator() {
return null;
}
@Override
public BytesRef next() {
Document doc;
try {
doc = lfd.nextDoc();
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
if (doc == null) {
return null;
}
if (count++ == 10000) {
return null;
}
return new BytesRef(doc.get("body"));
}
@Override
public BytesRef payload() {
return null;
}
@Override
public boolean hasPayloads() {
return false;
}
});
if (VERBOSE) {
System.out.println(sug.sizeInBytes() + " bytes");
List<LookupResult> results = sug.lookup("general r",10);
System.out.println("results:");
for(LookupResult result : results) {
System.out.println(" " + result);
}
}
}
项目：Maskana-Gestor-de-Conocimiento
文件：TestFlushByRamOrCountsPolicy.java
@BeforeClass
public static void beforeClass() throws Exception {
lineDocFile = new LineFileDocs(random(),defaultCodecSupportsdocValues());
}
项目：Maskana-Gestor-de-Conocimiento
文件：TestNRTCachingDirectory.java
public void testNRTAndCommit() throws Exception {
Directory dir = newDirectory();
NRTCachingDirectory cachedDir = new NRTCachingDirectory(dir,id)));
}
r.close();
cachedDir.close();
docs.close();
}
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 dio@foxmail.com 举报，一经查实，本站将立刻删除。
