TestDocument.cpp

/*------------------------------------------------------------------------------
* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
*
* Distributable under the terms of either the Apache License (Version 2.0) or
* the GNU Lesser General Public License, as specified in the COPYING file.
------------------------------------------------------------------------------*/
#define ANKERL_NANOBENCH_IMPLEMENT
#include "test.h"

#include "CLucene/document/FieldSelector.h"
#include "CLucene/index/SDocumentWriter.h"
#include <nanobench.h>
#include <memory>

//an in memory input stream for testing binary data
class MemReader : public CL_NS(util)::Reader {
    signed char *value;
    size_t len;
    int64_t pos;

public:
    MemReader(const char *value, const int32_t length = -1) {
        if (length >= 0)
            this->len = length;
        else
            this->len = strlen(value);
        this->pos = 0;
        this->value = _CL_NEWARRAY(signed char, this->len);
        memcpy(this->value, value, this->len);
    }
    virtual ~MemReader() {
        _CLDELETE_ARRAY(this->value);
    }

    int32_t read(const signed char *&start, int32_t min, int32_t max) {
        start = this->value + pos;
        int32_t r = max > min ? max : min;
        if (len - pos < r)
            r = len - pos;
        pos += r;
        return r;
    }
    int64_t position() {
        return pos;
    }
    int64_t skip(int64_t ntoskip) {
        int64_t s = ntoskip;
        if (len - pos < s)
            s = len - pos;

        this->pos += s;
        return s;
    }
    size_t size() {
        return len;
    }
};

void TestReaderValueField(CuTest *tc) {
    RAMDirectory dir;

    SimpleAnalyzer<char> analyzer;
    IndexWriter w(&dir, &analyzer, true);
    w.setUseCompoundFile(false);
    auto field_name = lucene::util::Misc::_charToWide("f3");

    Document doc;
    auto field = _CLNEW Field(field_name, Field::INDEX_TOKENIZED | Field::STORE_NO);

    auto char_string_reader = std::make_unique<lucene::util::SStringReader<char>>();
    char_string_reader->init("value1", 6, true);
    auto stream = analyzer.tokenStream(field->name(), char_string_reader.get());
    field->setValue(stream);
    doc.add(*field);

    w.addDocument(&doc);
    w.close();

    IndexSearcher searcher(&dir);
    auto t1 = std::make_unique<Term>(_T("f3"), _T("value1"));
    auto query1 =std::make_unique<TermQuery>(t1.get());
    Hits *hits1 = searcher.search(query1.get());
    CLUCENE_ASSERT(1 == hits1->length());
    _CLDELETE(stream)
    _CLDELETE(hits1)
    _CLDELETE_ARRAY(field_name)
}

void TestMultiSetValueField(CuTest *tc) {
    RAMDirectory dir;

    SimpleAnalyzer<char> analyzer;
    IndexWriter w(&dir, &analyzer, true);
    w.setUseCompoundFile(false);
    auto field_name = lucene::util::Misc::_charToWide("f3");

    Document doc;
    auto field = _CLNEW Field(field_name, Field::INDEX_TOKENIZED | Field::STORE_NO);
    auto char_string_reader = std::make_unique<lucene::util::SStringReader<char>>();
    char_string_reader->init("value1", 6, false);
    auto stream = analyzer.tokenStream(field->name(), char_string_reader.get());
    field->setValue(stream);
    char_string_reader->init("value2", 6, false);
    auto stream2 = analyzer.tokenStream(field->name(), char_string_reader.get());
    field->setValue(stream2);

    doc.add(*field);

    w.addDocument(&doc);
    w.close();

    IndexSearcher searcher(&dir);
    auto t1 = std::make_unique<Term>(_T("f3"), _T("value1"));
    auto query1 = std::make_unique<TermQuery>(t1.get());
    Hits *hits1 = searcher.search(query1.get());
    CLUCENE_ASSERT(0 == hits1->length());
    auto t2 = std::make_unique<Term>(_T("f3"), _T("value2"));
    auto query2 = std::make_unique<TermQuery>(t2.get());
    Hits *hits2 = searcher.search(query2.get());
    CLUCENE_ASSERT(1 == hits2->length());

    doc.clear();
    _CLDELETE(stream)
    _CLDELETE(stream2)
    _CLDELETE(hits1)
    _CLDELETE(hits2)
    _CLDELETE_ARRAY(field_name)
}

void TestMultiAddValueField(CuTest *tc) {
    RAMDirectory dir;
    auto field_name = lucene::util::Misc::_charToWide("f3");

    SimpleAnalyzer<char> analyzer;
    IndexWriter w(&dir, &analyzer, true);
    w.setUseCompoundFile(false);

    Document doc;
    auto field1 = _CLNEW Field(field_name, Field::INDEX_TOKENIZED | Field::STORE_NO);
    auto char_string_reader = std::make_unique<lucene::util::SStringReader<char>>();
    char_string_reader->init("value1", 6, false);
    auto stream = analyzer.tokenStream(field1->name(), char_string_reader.get());
    field1->setValue(stream);
    doc.add(*field1);

    auto field2 = _CLNEW Field(field_name, Field::INDEX_TOKENIZED | Field::STORE_NO);
    auto char_string_reader2 = std::make_unique<lucene::util::SStringReader<char>>();
    char_string_reader2->init("value2", 6, false);
    auto stream2 = analyzer.tokenStream(field2->name(), char_string_reader2.get());
    field2->setValue(stream2);
    doc.add(*field2);

    try {
        w.addDocument(&doc);
        w.close();
    } catch (CLuceneError& ae) {
        std::cout <<ae.what() << std::endl;
        throw ae;
    }

    auto t1 = std::make_unique<Term>(_T("f3"), _T("value1"));
    auto query1 = std::make_unique<TermQuery>(t1.get());
    IndexSearcher searcher(&dir);
    Hits *hits1 = searcher.search(query1.get());

    CLUCENE_ASSERT(1 == hits1->length());
    auto t2 = std::make_unique<Term>(_T("f3"), _T("value2"));
    auto query2 = std::make_unique<TermQuery>(t2.get());
    Hits *hits2 = searcher.search(query2.get());
    CLUCENE_ASSERT(1 == hits2->length());
    doc.removeFields(_T("f3"));
    CLUCENE_ASSERT(doc.getFields()->size() == 0);

    _CLDELETE(hits1);
    _CLDELETE(hits2);
    _CLDELETE(stream);
    _CLDELETE(stream2);
    _CLDELETE_ARRAY(field_name)
}

void TestFields(CuTest *tc) {
    Field *f = _CLNEW Field(_T("test"), _T("value"), Field::INDEX_TOKENIZED);
    CLUCENE_ASSERT(f->isIndexed() && f->isTokenized());
    CLUCENE_ASSERT(!f->isStored() && !f->isBinary() && !f->getOmitNorms());
    _CLDELETE(f);

    f = _CLNEW Field(_T("test"), _T("value"), Field::STORE_YES | Field::INDEX_NONORMS);
    //CLUCENE_ASSERT(f->isIndexed());
    //CLUCENE_ASSERT(!f->isTokenized());
    CLUCENE_ASSERT(f->getOmitNorms());
    CLUCENE_ASSERT(f->isStored() && !f->isBinary());
    _CLDELETE(f);

    Document doc;
    doc.add(*_CLNEW Field(_T("f1"), _T("value"), Field::INDEX_TOKENIZED));
    doc.add(*_CLNEW Field(_T("f2"), _T("value"), Field::INDEX_TOKENIZED));
    doc.add(*_CLNEW Field(_T("f3"), _T("value1"), Field::INDEX_TOKENIZED));
    doc.add(*_CLNEW Field(_T("f3"), _T("value2"), Field::INDEX_TOKENIZED));
    doc.add(*_CLNEW Field(_T("f4"), _T("value"), Field::INDEX_TOKENIZED));
    CLUCENE_ASSERT(doc.getFields()->size() == 5);

    _CLLDELETE(doc.fields());//just fetch the fields (to test deprecated loads)

    doc.removeField(_T("f3"));
    CLUCENE_ASSERT(doc.getFields()->size() == 4);

    //test deprecated enumerator
    DocumentFieldEnumeration *e = doc.fields();
    int count = 0;
    while (e->hasMoreElements()) {
        Field *field4 = e->nextElement();
        CLUCENE_ASSERT(field4 != NULL);
        count++;
    }
    CLUCENE_ASSERT(count == 4);
    _CLDELETE(e);

    doc.add(*_CLNEW Field(_T("f3"), _T("value3"), Field::INDEX_TOKENIZED));
    CLUCENE_ASSERT(doc.getFields()->size() == 5);

    //test deprecated enumerator
    e = doc.fields();
    count = 0;
    while (e->hasMoreElements()) {
        Field *field5 = e->nextElement();
        CLUCENE_ASSERT(field5 != NULL);
        count++;
    }
    CLUCENE_ASSERT(count == 5);
    _CLDELETE(e);

    doc.removeFields(_T("f3"));
    CLUCENE_ASSERT(doc.getFields()->size() == 3);

    doc.removeFields(_T("f4"));
    CLUCENE_ASSERT(doc.getFields()->size() == 2);

    doc.removeFields(_T("f1"));
    CLUCENE_ASSERT(doc.getFields()->size() == 1);

    doc.removeFields(_T("f2"));
    CLUCENE_ASSERT(doc.getFields()->size() == 0);
}

/*
  void TestDateTools(CuTest *tc) {
	  TCHAR* t = CL_NS(document)::DateTools::timeToString( Misc::currentTimeMillis() , CL_NS(document)::DateTools::MILLISECOND_FORMAT);
	  _CLDELETE_ARRAY(t);

	  TCHAR buf[30];
	  const TCHAR* xpt = _T("19700112102054321");
	  int64_t vv = (int64_t)987654321;
	  CL_NS(document)::DateTools::timeToString( vv , CL_NS(document)::DateTools::MILLISECOND_FORMAT, buf, 30);

	  if ( _tcscmp(buf,xpt) != 0 ) {
			CuFail(tc, _T("timeToString failed\n"), buf, xpt);
	  }
	  _CLDELETE_ARRAY(t);
  }
  */

void TestFieldSelectors(CuTest *tc) {
    RAMDirectory dir;
    const TCHAR *longStrValue = _T("too long a field...");
    {
        WhitespaceAnalyzer<TCHAR> a;
        IndexWriter w(&dir, &a, true);
        for (int i = 0; i < 3; i++) {
            Document doc;
            doc.add(*_CLNEW Field(_T("f1"), _T("value1"), Field::STORE_NO | Field::INDEX_TOKENIZED));
            doc.add(*_CLNEW Field(_T("f2"), _T("value2"), Field::STORE_NO | Field::INDEX_TOKENIZED));
            doc.add(*_CLNEW Field(_T("f3"), _T("value3"), Field::STORE_NO | Field::INDEX_TOKENIZED));
            doc.add(*_CLNEW Field(_T("f4"), _T("value4"), Field::STORE_NO | Field::INDEX_TOKENIZED));
            doc.add(*_CLNEW Field(_T("f5"), longStrValue, Field::STORE_NO | Field::INDEX_TOKENIZED));

            w.addDocument(&doc);
        }
        w.flush();
    }

    IndexReader *reader = IndexReader::open(&dir);
    MapFieldSelector fieldsToLoad;
    fieldsToLoad.add(_T("f2"), FieldSelector::LOAD);
    fieldsToLoad.add(_T("f3"), FieldSelector::LAZY_LOAD);
    fieldsToLoad.add(_T("f5"), FieldSelector::SIZE);
    Document doc;
    CLUCENE_ASSERT(reader->document(0, doc, &fieldsToLoad));
    CLUCENE_ASSERT(doc.getFields()->size() == 3);
    CuAssertStrEquals(tc, _T("check f2"), _T("value2"), doc.get(_T("f2")));
    CuAssertStrEquals(tc, _T("check f3"), _T("value3"), doc.get(_T("f3")));

    Field *byteField = doc.getField(_T("f5"));
    const ValueArray<uint8_t> &bytes = *byteField->binaryValue();
    uint32_t shouldBeInt = 2 * _tcslen(longStrValue);
    ValueArray<uint8_t> shouldBe(4);
    shouldBe[0] = (uint8_t) (shouldBeInt >> 24);
    shouldBe[1] = (uint8_t) (shouldBeInt >> 16);
    shouldBe[2] = (uint8_t) (shouldBeInt >> 8);
    shouldBe[3] = (uint8_t) shouldBeInt;
    CLUCENE_ASSERT(byteField != NULL);
    CLUCENE_ASSERT(memcmp(shouldBe.values, bytes.values, 4) == 0);

    _CLDELETE(reader);
    _CL_LDECREF(&dir);//derefence since we are on the stack...
}

void _TestDocumentWithOptions(CuTest *tc, int storeBit, FieldSelector::FieldSelectorResult fieldSelectorBit) {
    char factbook[1024];
    strcpy(factbook, clucene_data_location);
    strcat(factbook, "/reuters-21578/feldman-cia-worldfactbook-data.txt");
    CuAssert(tc, _T("Factbook file does not exist"), Misc::dir_Exists(factbook));

    Document doc;
    Field *f;
    const TCHAR *_ts, *_ts2;
    const ValueArray<uint8_t> *strm;
    RAMDirectory ram;

    const char *areaderString = "a binary field";
    const TCHAR *treaderString = _T("a string reader field");
    size_t readerStringLen = strlen(areaderString);

    SimpleAnalyzer<TCHAR> an;
    IndexWriter writer(&ram, &an, true);//no analyzer needed since we are not indexing...

    ValueArray<uint8_t> b((uint8_t *) strdup(areaderString), strlen(areaderString));
    //use binary utf8
    doc.add(*_CLNEW Field(_T("binaryField"), &b,
                          Field::TERMVECTOR_NO | storeBit | Field::INDEX_NO, true));
    writer.addDocument(&doc);
    doc.clear();

    //use reader
    doc.add(*_CLNEW Field(_T("readerField"), _CLNEW StringReader(treaderString),
                          Field::TERMVECTOR_NO | storeBit | Field::INDEX_NO));
    writer.addDocument(&doc);
    doc.clear();

    //done adding a few documents, now try and add a few more...
    //writer.optimize();

    //use big file
    doc.add(*_CLNEW Field(_T("fileField"),
                          _CLNEW FileReader(factbook, SimpleInputStreamReader::ASCII),
                          Field::TERMVECTOR_NO | storeBit | Field::INDEX_NO));
    writer.addDocument(&doc);
    doc.clear();

    //another optimise...
    //writer.optimize();
    writer.close();

    IndexReader *reader = IndexReader::open(&ram);

    MapFieldSelector fieldsToLoad;
    fieldsToLoad.add(_T("fileField"), fieldSelectorBit);
    fieldsToLoad.add(_T("readerField"), fieldSelectorBit);
    fieldsToLoad.add(_T("binaryField"), fieldSelectorBit);

    //now check binary field
    reader->document(0, doc);
    f = doc.getField(_T("binaryField"));
    strm = f->binaryValue();

    CuAssertIntEquals(tc, _T("Check binary length is correct"), readerStringLen, b.length);
    for (size_t i = 0; i < readerStringLen; i++) {
        CuAssertIntEquals(tc, _T("Check binary values are the same"), (*strm)[i], areaderString[i]);
    }
    doc.clear();

    //and check reader stream
    reader->document(1, doc);
    f = doc.getField(_T("readerField"));
    _ts = f->stringValue();
    CuAssertStrEquals(tc, _T("Check readerField length is correct"), treaderString, _ts);
    doc.clear();

    //now check the large field field
    reader->document(2, doc);
    f = doc.getField(_T("fileField"));
    _ts = f->stringValue();
    FileReader fbStream(factbook, FileReader::ASCII);

    int i = 0;
    _ts2 = NULL;
    do {
        int32_t rd = fbStream.read((const void**)&_ts2, 1, 1);
        if (rd == -1)
            break;
        CLUCENE_ASSERT(rd == 1);
        CLUCENE_ASSERT(_ts[i] == *_ts2);
        i++;
    } while (true);
    CLUCENE_ASSERT(i == _tcslen(_ts));
    doc.clear();

    reader->close();
    _CLDELETE(reader);
    _CL_LDECREF(&ram);//this is in the stack...
}


void TestBinaryDocument(CuTest *tc) {
    _TestDocumentWithOptions(tc, Field::STORE_YES, FieldSelector::LOAD);
}
void TestCompressedDocument(CuTest *tc) {
    _TestDocumentWithOptions(tc, Field::STORE_COMPRESS, FieldSelector::LOAD);
}
void TestLazyBinaryDocument(CuTest *tc) {
    _TestDocumentWithOptions(tc, Field::STORE_YES, FieldSelector::LAZY_LOAD);
}
void TestLazyCompressedDocument(CuTest *tc) {
    _TestDocumentWithOptions(tc, Field::STORE_COMPRESS, FieldSelector::LAZY_LOAD);
}

void TestSetFieldBench(CuTest *tc) {

    ankerl::nanobench::Bench().epochIterations(1).run("TestSetFieldBench", [&] {
        RAMDirectory dir;

        SimpleAnalyzer<TCHAR> analyzer;
        IndexWriter w(&dir, &analyzer, true);
        Document doc;
        auto field_name = lucene::util::Misc::_charToWide("f3");

        auto field = _CLNEW Field(field_name, Field::INDEX_TOKENIZED | Field::STORE_NO);
        doc.add(*field);

        for (int i = 0; i < 10000; ++i) {
            auto value1 = lucene::util::Misc::_charToWide("value1");
            field->setValue(value1, false);
            w.addDocument(&doc);
        }
        w.close();
        doc.clear();
    });
}

string generateRandomString(int length) {
    static const char alphanum[] =
            "0123456789"
            "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
            "abcdefghijklmnopqrstuvwxyz";
    string randomString;
    random_device rd;
    mt19937 gen(rd());
    uniform_int_distribution<> dis(0, sizeof(alphanum) - 2);
    for (int i = 0; i < length; ++i) {
        randomString += alphanum[dis(gen)];
    }
    return randomString;
}

const int32_t MAX_FIELD_LEN = 0x7FFFFFFFL;
const int32_t MAX_BUFFER_DOCS = 100000000;
const int32_t MERGE_FACTOR = 100000000;
void TestAddDocument(CuTest *tc) {
    try {
        RAMDirectory dir;
        SimpleAnalyzer<char> sanalyzer;
        IndexWriter w(&dir, &sanalyzer, true);
        w.setUseCompoundFile(false);
        w.setMaxBufferedDocs(MAX_BUFFER_DOCS);
        w.setRAMBufferSizeMB(256);
        w.setMaxFieldLength(MAX_FIELD_LEN);
        w.setMergeFactor(MERGE_FACTOR);
        w.setDocumentWriter(_CLNEW SDocumentsWriter<char>(w.getDirectory(), &w));
        Document doc;
        auto field_name = lucene::util::Misc::_charToWide("f3");
        auto field = _CLNEW Field(field_name, Field::INDEX_TOKENIZED | Field::STORE_NO);
        doc.add(*field);

        for (int i = 0; i <= 2000000; i++) {
            std::string value1 = "value1";
            if (i > 0) value1 = generateRandomString(2000);
            auto stringReader = _CLNEW lucene::util::SStringReader<char>(
                    value1.c_str(), strlen(value1.c_str()), false);
            auto stream = sanalyzer.reusableTokenStream(field_name, stringReader);

            field->setValue(stream);
            w.addDocument(&doc, &sanalyzer);
        }
        IndexSearcher searcher(&dir);
        Term* t2 = _CLNEW Term(_T("f3"), _T("value1"));
        auto* query2 = _CLNEW TermQuery(t2);
        Hits* hits2 = searcher.search(query2);
        CLUCENE_ASSERT(1 == hits2->length());
        doc.clear();
        w.close();
    } catch (CLuceneError& ae) {
        std::cout <<ae.what() << std::endl;
        throw ae;
    }
}

void TestNewFieldBench(CuTest *tc) {

    ankerl::nanobench::Bench().epochIterations(1).run("TestNewFieldBench", [&] {
        RAMDirectory dir;
        //std::string value = "value1";
        //std::string field_name = "f3";
        //auto field_name_w = std::wstring(field_name.begin(), field_name.end());
        auto field_name = lucene::util::Misc::_charToWide("f3");

        //auto field_name_w = lucene::util::Misc::_charToWide(field_name.c_str());

        SimpleAnalyzer<TCHAR> analyzer;
        IndexWriter w(&dir, &analyzer, true);
        Document doc;
        for (int i = 0; i < 10000; ++i) {
            //wchar_t value1[value.length()];
            //lucene::util::Misc::_cpycharToWide(value.c_str(), value1, value.length());
            //auto value1 = std::wstring(value).c_str();
            auto value1 = lucene::util::Misc::_charToWide("value1");
            auto field = _CLNEW Field(field_name, value1, Field::INDEX_TOKENIZED | Field::STORE_NO, false);
            doc.add(*field);
            w.addDocument(&doc);
            doc.clear();
            //_CLDELETE_ARRAY(value1);
        }
        w.close();
        //_CLDELETE(field);
        //_CLDELETE_ARRAY(field_name_w);
        //doc.add(*);
    });
}

CuSuite *testdocument(void) {
    CuSuite *suite = CuSuiteNew(_T("CLucene Document Test"));

    SUITE_ADD_TEST(suite, TestFields);
    SUITE_ADD_TEST(suite, TestMultiSetValueField);
    SUITE_ADD_TEST(suite, TestMultiAddValueField);
    SUITE_ADD_TEST(suite, TestReaderValueField);
    //SUITE_ADD_TEST(suite, TestAddDocument);
    return suite;
}