Skip to content

search index compatibility and other minor changes #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
snoopdave merged 4 commits into from
Oct 1, 2020
Merged

search index compatibility and other minor changes #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b5086b9
rebuild index in background if lucene throws an IllegalArgumentExcept…
mbien Jun 9, 2020
7e3238d
code cleanup.
mbien Jun 9, 2020
6abff01
added lucene-backward-codecs dependency for index compatibility betwe…
mbien Jun 9, 2020
c124e3b
removed redundant synchronized map wrapper since cache is only access…
mbien Jun 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions app/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,22 +39,22 @@ limitations under the License.
<java-mail.version>1.4.7</java-mail.version>
<jstl.version>1.2</jstl.version>
<angular.version>1.7.8</angular.version>
<ant.version>1.10.7</ant.version>
<asm.version>7.2</asm.version>
<ant.version>1.10.8</ant.version>
<asm.version>8.0.1</asm.version>
<commons-validator.version>1.6</commons-validator.version>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<commons-httpclient.version>3.1</commons-httpclient.version>
<commons-codec.version>1.13</commons-codec.version>
<eclipse-link.version>2.7.5</eclipse-link.version>
<guice.version>4.2.2</guice.version>
<commons-codec.version>1.14</commons-codec.version>
<eclipse-link.version>2.7.7</eclipse-link.version>
<guice.version>4.2.3</guice.version>
<log4j.version>1.2.17</log4j.version>
<log4j2.version>2.10.0</log4j2.version>
<lucene.version>8.3.0</lucene.version>
<lucene.version>8.5.1</lucene.version>
<oauth-core.version>20100527</oauth-core.version>
<maven-war.version>3.2.3</maven-war.version>
<maven-surefire.version>2.17</maven-surefire.version>
<maven-antrun.version>1.0b3</maven-antrun.version>
<rome.version>1.12.2</rome.version>
<rome.version>1.13.1</rome.version>
<slf4j.version>1.7.26</slf4j.version>
<spring.version>4.1.4.RELEASE</spring.version>
<spring.security.version>3.2.5.RELEASE</spring.security.version>
Expand Down Expand Up @@ -264,13 +264,21 @@ limitations under the License.
<artifactId>jquery-validation</artifactId>
<version>1.19.0</version>
</dependency>

<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<scope>compile</scope>
<version>${lucene.version}</version>
</dependency>

<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
<scope>compile</scope>
<version>${lucene.version}</version>
</dependency>

<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
Expand Down
115 changes: 42 additions & 73 deletions app/src/main/java/org/apache/roller/weblogger/business/search/IndexManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.io.IOException;

import java.lang.reflect.InvocationTargetException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
Expand Down Expand Up @@ -60,30 +61,22 @@
*/
@com.google.inject.Singleton
public class IndexManagerImpl implements IndexManager {
// ~ Static fields/initializers
// =============================================

private IndexReader reader;
private final Weblogger roller;

static Log mLogger = LogFactory.getFactory().getInstance(
IndexManagerImpl.class);

// ~ Instance fields
// ========================================================
private final static Log mLogger = LogFactory.getFactory().getInstance(IndexManagerImpl.class);

private boolean searchEnabled = true;

File indexConsistencyMarker;
private final String indexDir;

private String indexDir = null;
private final File indexConsistencyMarker;

private boolean inconsistentAtStartup = false;

private final ReadWriteLock rwl = new ReentrantReadWriteLock();

// ~ Constructors
// ===========================================================

/**
* Creates a new lucene index manager. This should only be created once.
Expand Down Expand Up @@ -125,15 +118,11 @@ public void initialize() throws InitializationException {
// only initialize the index if search is enabled
if (this.searchEnabled) {

// 1. If inconsistency marker exists.
// Delete index
// 2. if we're using RAM index
// load ram index wrapper around index
//
// delete index if inconsistency marker exists
if (indexConsistencyMarker.exists()) {
getFSDirectory(true);
inconsistentAtStartup = true;
mLogger.debug("Index inconsistent: marker exists");
inconsistentAtStartup = true;
deleteIndex();
} else {
try {
File makeIndexDir = new File(indexDir);
Expand All @@ -153,26 +142,26 @@ public void initialize() throws InitializationException {
// test if the index is readable, if the version is outdated or it fails we rebuild.
try {
synchronized(this) {
reader = DirectoryReader.open(getFSDirectory(false));
reader = DirectoryReader.open(getIndexDirectory());
}
} catch (IOException ex) {
mLogger.warn("Error opening search index, scheduling rebuild.", ex);
getFSDirectory(true);
} catch (IOException | IllegalArgumentException ex) { // IAE for incompatible codecs
mLogger.warn("Failed to open search index, scheduling rebuild.", ex);
inconsistentAtStartup = true;
deleteIndex();
}
} else {
mLogger.debug("Creating index");
inconsistentAtStartup = true;

createIndex(getFSDirectory(true));
deleteIndex();
createIndex(getIndexDirectory());
}

if (isInconsistentAtStartup()) {
if (inconsistentAtStartup) {
mLogger.info("Index was inconsistent. Rebuilding index in the background...");
try {
rebuildWebsiteIndex();
} catch (WebloggerException e) {
mLogger.error("ERROR: scheduling re-index operation");
} catch (WebloggerException ex) {
mLogger.error("ERROR: scheduling re-index operation", ex);
}
} else {
mLogger.info("Index initialized and ready for use.");
Expand All @@ -181,48 +170,34 @@ public void initialize() throws InitializationException {

}

// ~ Methods
// ================================================================

@Override
public void rebuildWebsiteIndex() throws WebloggerException {
scheduleIndexOperation(new RebuildWebsiteIndexOperation(roller, this,
null));
scheduleIndexOperation(new RebuildWebsiteIndexOperation(roller, this, null));
}

@Override
public void rebuildWebsiteIndex(Weblog website) throws WebloggerException {
scheduleIndexOperation(new RebuildWebsiteIndexOperation(roller, this,
website));
scheduleIndexOperation(new RebuildWebsiteIndexOperation(roller, this, website));
}

@Override
public void removeWebsiteIndex(Weblog website) throws WebloggerException {
scheduleIndexOperation(new RemoveWebsiteIndexOperation(roller, this,
website));
scheduleIndexOperation(new RemoveWebsiteIndexOperation(roller, this, website));
}

@Override
public void addEntryIndexOperation(WeblogEntry entry)
throws WebloggerException {
AddEntryOperation addEntry = new AddEntryOperation(roller, this, entry);
scheduleIndexOperation(addEntry);
public void addEntryIndexOperation(WeblogEntry entry) throws WebloggerException {
scheduleIndexOperation(new AddEntryOperation(roller, this, entry));
}

@Override
public void addEntryReIndexOperation(WeblogEntry entry)
throws WebloggerException {
ReIndexEntryOperation reindex = new ReIndexEntryOperation(roller, this,
entry);
scheduleIndexOperation(reindex);
public void addEntryReIndexOperation(WeblogEntry entry) throws WebloggerException {
scheduleIndexOperation(new ReIndexEntryOperation(roller, this, entry));
}

@Override
public void removeEntryIndexOperation(WeblogEntry entry)
throws WebloggerException {
RemoveEntryOperation removeOp = new RemoveEntryOperation(roller, this,
entry);
executeIndexOperationNow(removeOp);
public void removeEntryIndexOperation(WeblogEntry entry) throws WebloggerException {
executeIndexOperationNow(new RemoveEntryOperation(roller, this, entry));
}

public ReadWriteLock getReadWriteLock() {
Expand Down Expand Up @@ -314,7 +289,13 @@ public synchronized IndexReader getSharedIndexReader() {
* @return Directory The directory containing the index, or null if error.
*/
public Directory getIndexDirectory() {
return getFSDirectory(false);

try {
return FSDirectory.open(Path.of(indexDir));
} catch (IOException e) {
mLogger.error("Problem accessing index directory", e);
}
return null;
}

private boolean indexExists() {
Expand All @@ -326,31 +307,19 @@ private boolean indexExists() {
return false;
}

private FSDirectory getFSDirectory(boolean delete) {

FSDirectory directory = null;

try {

directory = FSDirectory.open(Path.of(indexDir));

if (delete && directory != null) {
// clear old files
String[] files = directory.listAll();
for (int i = 0; i < files.length; i++) {
File file = new File(indexDir, files[i]);
if (!file.delete()) {
throw new IOException("couldn't delete " + files[i]);
}
}

private void deleteIndex() {

try(FSDirectory directory = FSDirectory.open(Path.of(indexDir))) {

String[] files = directory.listAll();
for (String file : files) {
Files.delete(Path.of(indexDir, file));
}

} catch (IOException e) {
mLogger.error("Problem accessing index directory", e);
} catch (IOException ex) {
mLogger.error("Problem accessing index directory", ex);
}

return directory;

}

private void createIndex(Directory dir) {
Expand Down
6 changes: 1 addition & 5 deletions app/src/main/java/org/apache/roller/weblogger/util/cache/ExpiringLRUCacheImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,7 @@ public synchronized void put(String key, Object value) {
public synchronized Object get(String key) {

Object value = null;
ExpiringCacheEntry entry = null;

synchronized(this) {
entry = (ExpiringCacheEntry) super.get(key);
}
ExpiringCacheEntry entry = (ExpiringCacheEntry) super.get(key);

if (entry != null) {

Expand Down
16 changes: 10 additions & 6 deletions app/src/main/java/org/apache/roller/weblogger/util/cache/LRUCacheImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

package org.apache.roller.weblogger.util.cache;

import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
Expand All @@ -31,8 +30,8 @@
*/
public class LRUCacheImpl implements Cache {

private String id = null;
private Map cache = null;
private final String id;
private final Map cache;

// for metrics
protected double hits = 0;
Expand All @@ -44,18 +43,18 @@ public class LRUCacheImpl implements Cache {

protected LRUCacheImpl(String id) {

this.id = id;
this.cache = Collections.synchronizedMap(new LRULinkedHashMap(100));
this(id, 100);
}


protected LRUCacheImpl(String id, int maxsize) {

this.id = id;
this.cache = Collections.synchronizedMap(new LRULinkedHashMap(maxsize));
this.cache = new LRULinkedHashMap(maxsize);
}


@Override
public String getId() {
return this.id;
}
Expand All @@ -64,6 +63,7 @@ public String getId() {
/**
* Store an entry in the cache.
*/
@Override
public synchronized void put(String key, Object value) {

this.cache.put(key, value);
Expand All @@ -74,6 +74,7 @@ public synchronized void put(String key, Object value) {
/**
* Retrieve an entry from the cache.
*/
@Override
public synchronized Object get(String key) {

Object obj = this.cache.get(key);
Expand All @@ -89,13 +90,15 @@ public synchronized Object get(String key) {
}


@Override
public synchronized void remove(String key) {

this.cache.remove(key);
removes++;
}


@Override
public synchronized void clear() {

this.cache.clear();
Expand All @@ -109,6 +112,7 @@ public synchronized void clear() {
}


@Override
public Map<String, Object> getStats() {

Map<String, Object> stats = new HashMap<String, Object>();
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ limitations under the License.
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.5.2</version>
<version>5.6.2</version>
<scope>test</scope>
</dependency>
</dependencies>
Expand Down