Introduce an index version for the ticket index
In order to be able to update the index definition, the ticket index
is assigned a version number, 2. This way the definiton can be updated
and compatability with existing index files can be checked.
The actual index is stored in a directory of name `indexVersion_codecVersion`.
This wayit is veriy easy to check if an index of a certain version exists on the
filesystem. It allows to have multiple indexes of different versions present,
so that a downgrade of the software is possible without having to reindex
again. Of coure, this is only possible if no new tickets were created since these
would be missing in the old index.
A new class `LuceneIndexStore` is introduced, which abstracts away the versioned
index directory. The idea is, that this provides one place to keep the Lucene
codec version and to allow to code compatibility rules into this class, so that
older indices can still be used if they are compatible.
diff --git a/src/main/java/com/gitblit/tickets/TicketIndexer.java b/src/main/java/com/gitblit/tickets/TicketIndexer.java
index 8aab74b..bc08fc8 100644
--- a/src/main/java/com/gitblit/tickets/TicketIndexer.java
+++ b/src/main/java/com/gitblit/tickets/TicketIndexer.java
@@ -62,7 +62,7 @@
import com.gitblit.models.TicketModel.Attachment;
import com.gitblit.models.TicketModel.Patchset;
import com.gitblit.models.TicketModel.Status;
-import com.gitblit.utils.FileUtils;
+import com.gitblit.utils.LuceneIndexStore;
import com.gitblit.utils.StringUtils;
/**
@@ -110,6 +110,8 @@
priority(Type.INT),
severity(Type.INT);
+ final static int INDEX_VERSION = 2;
+
final Type fieldType;
Lucene(Type fieldType) {
@@ -169,14 +171,15 @@
private final Logger log = LoggerFactory.getLogger(getClass());
- private final File luceneDir;
+ private final LuceneIndexStore indexStore;
private IndexWriter writer;
private IndexSearcher searcher;
public TicketIndexer(IRuntimeManager runtimeManager) {
- this.luceneDir = runtimeManager.getFileOrFolder(Keys.tickets.indexFolder, "${baseFolder}/tickets/lucene");
+ File luceneDir = runtimeManager.getFileOrFolder(Keys.tickets.indexFolder, "${baseFolder}/tickets/lucene");
+ this.indexStore = new LuceneIndexStore(luceneDir, Lucene.INDEX_VERSION);
}
/**
@@ -192,7 +195,7 @@
*/
public void deleteAll() {
close();
- FileUtils.delete(luceneDir);
+ indexStore.delete();
}
/**
@@ -441,12 +444,9 @@
private IndexWriter getWriter() throws IOException {
if (writer == null) {
- Directory directory = FSDirectory.open(luceneDir.toPath());
+ indexStore.create();
- if (!luceneDir.exists()) {
- luceneDir.mkdirs();
- }
-
+ Directory directory = FSDirectory.open(indexStore.getPath());
StandardAnalyzer analyzer = new StandardAnalyzer();
IndexWriterConfig config = new IndexWriterConfig(analyzer);
config.setOpenMode(OpenMode.CREATE_OR_APPEND);
diff --git a/src/main/java/com/gitblit/utils/LuceneIndexStore.java b/src/main/java/com/gitblit/utils/LuceneIndexStore.java
new file mode 100644
index 0000000..0ccfd2e
--- /dev/null
+++ b/src/main/java/com/gitblit/utils/LuceneIndexStore.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2017 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.gitblit.utils;
+
+import java.io.File;
+import java.nio.file.Path;
+
+/**
+ * @author Florian Zschocke
+ *
+ * @since 1.9.0
+ */
+public class LuceneIndexStore
+{
+
+ public static final int LUCENE_CODEC_VERSION = 54;
+
+ protected File indexFolder;
+
+ /**
+ * Constructor for a base folder that contains the version specific index folders
+ * and an index version.
+ *
+ * @param luceneFolder
+ * Path to the base folder for the Lucene indices, i.e. the common "lucene" directory.
+ * @param indexVersion
+ * Version of the index definition
+ */
+ public LuceneIndexStore(File luceneFolder, int indexVersion)
+ {
+ this.indexFolder = new File(luceneFolder, indexVersion + "_" + LUCENE_CODEC_VERSION);
+ }
+
+
+
+ /**
+ * Create the Lucene index directory for this index version and Lucene codec version
+ */
+ public void create()
+ {
+ if (! indexFolder.exists()) {
+ indexFolder.mkdirs();
+ }
+ }
+
+
+ /**
+ * Delete the Lucene index directory for this index version and Lucene codec version
+ *
+ * @return True if the directory could successfully be deleted.
+ */
+ public boolean delete()
+ {
+ if (indexFolder.exists()) {
+ return FileUtils.delete(indexFolder);
+ }
+ return true;
+ }
+
+
+
+ /**
+ * @return The Path to the index folder
+ */
+ public Path getPath()
+ {
+ return indexFolder.toPath();
+ }
+
+
+
+ /**
+ * Check if an index of the respective version, or compatible, already exists.
+ *
+ * @return True if an index exists, False otherwise
+ */
+ public boolean hasIndex()
+ {
+ return indexFolder.exists() &&
+ indexFolder.isDirectory() &&
+ (indexFolder.list().length > 1);
+ }
+
+}
diff --git a/src/test/java/com/gitblit/utils/LuceneIndexStoreTest.java b/src/test/java/com/gitblit/utils/LuceneIndexStoreTest.java
new file mode 100644
index 0000000..4d66c5d
--- /dev/null
+++ b/src/test/java/com/gitblit/utils/LuceneIndexStoreTest.java
@@ -0,0 +1,245 @@
+/*
+ * Copyright 2017 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.gitblit.utils;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+
+/**
+ * @author Florian Zschocke
+ *
+ */
+public class LuceneIndexStoreTest
+{
+
+ private final int LUCENE_VERSION = LuceneIndexStore.LUCENE_CODEC_VERSION;
+
+ @Rule
+ public TemporaryFolder baseFolder = new TemporaryFolder();
+
+ private String getIndexDir(int version)
+ {
+ return version + "_" + LUCENE_VERSION;
+ }
+
+
+
+ @Test
+ public void testCreate()
+ {
+ int version = 0;
+ File luceneFolder = new File(baseFolder.getRoot(), "tickets/lucene");
+ assertFalse("Precondition failure: directory exists already", luceneFolder.exists());
+
+ LuceneIndexStore li = new LuceneIndexStore(luceneFolder, version);
+ li.create();
+
+ File luceneDir = new File(luceneFolder, getIndexDir(version));
+ assertTrue(luceneDir.exists());
+ }
+
+ @Test
+ public void testCreateIndexDir()
+ {
+ int version = 111222;
+ File luceneFolder = null;
+ try {
+ luceneFolder = baseFolder.newFolder("tickets", "lucene");
+ }
+ catch (IOException e) {
+ fail("Failed in setup of folder: " + e);
+ }
+ assertTrue("Precondition failure: directory does not exist", luceneFolder.exists());
+
+ LuceneIndexStore li = new LuceneIndexStore(luceneFolder, version);
+ li.create();
+
+ File luceneDir = new File(luceneFolder, getIndexDir(version));
+ assertTrue(luceneDir.exists());
+ assertTrue(luceneDir.isDirectory());
+
+ // Make sure nothing else was created.
+ assertEquals(0, luceneDir.list().length);
+ assertEquals(1, luceneDir.getParentFile().list().length);
+ assertEquals(1, luceneDir.getParentFile().getParentFile().list().length);
+ }
+
+ @Test
+ public void testCreateIfNecessary()
+ {
+ int version = 1;
+ File luceneFolder = new File(baseFolder.getRoot(), "tickets/lucene");
+ File luceneDir = null;
+ try {
+ luceneDir = baseFolder.newFolder("tickets", "lucene", getIndexDir(version));
+ }
+ catch (IOException e) {
+ fail("Failed in setup of folder: " + e);
+ }
+ assertTrue("Precondition failure: directory does not exist", luceneDir.exists());
+
+ LuceneIndexStore li = new LuceneIndexStore(luceneFolder, version);
+ li.create();
+
+ assertTrue(luceneDir.exists());
+ assertTrue(luceneDir.isDirectory());
+
+ // Make sure nothing else was created.
+ assertEquals(0, luceneDir.list().length);
+ assertEquals(1, luceneDir.getParentFile().list().length);
+ assertEquals(1, luceneDir.getParentFile().getParentFile().list().length);
+ }
+
+ @Test
+ public void testDelete()
+ {
+ int version = 111222333;
+ File luceneFolder = new File(baseFolder.getRoot(), "repo1/lucene");
+ File luceneDir = null;
+ try {
+ luceneDir = baseFolder.newFolder("repo1", "lucene", getIndexDir(version));
+ }
+ catch (IOException e) {
+ fail("Failed in setup of folder: " + e);
+ }
+ assertTrue("Precondition failure: index directory does not exist", luceneDir.exists());
+
+ LuceneIndexStore li = new LuceneIndexStore(luceneFolder, version);
+ assertTrue(li.delete());
+
+ assertFalse(luceneDir.exists());
+ assertTrue(luceneFolder.exists());
+ }
+
+ @Test
+ public void testDeleteNotExist()
+ {
+ int version = 0;
+
+ File luceneFolder = null;
+ try {
+ luceneFolder = baseFolder.newFolder("repo1", "lucene");
+ }
+ catch (IOException e) {
+ fail("Failed in setup of folder: " + e);
+ }
+ File luceneDir = new File(luceneFolder, getIndexDir(version));
+ assertFalse("Precondition failure: index directory exists already", luceneDir.exists());
+
+ LuceneIndexStore li = new LuceneIndexStore(luceneFolder, version);
+ assertTrue(li.delete());
+
+ assertFalse(luceneDir.exists());
+ assertTrue(luceneFolder.exists());
+ }
+
+ @Test
+ public void testDeleteWithFiles()
+ {
+ int version = 111222333;
+
+ File luceneFolder = new File(baseFolder.getRoot(), "tickets/lucene");
+ File luceneDir = null;
+
+ File otherDir = new File(baseFolder.getRoot(), "tickets/lucene/" + version + "_10");
+ File dbFile = null;
+ try {
+ luceneDir = baseFolder.newFolder("tickets", "lucene", getIndexDir(version));
+ File file = new File(luceneDir, "_file1");
+ file.createNewFile();
+ file = new File(luceneDir, "_file2.db");
+ file.createNewFile();
+ file = new File(luceneDir, "conf.conf");
+ file.createNewFile();
+
+ otherDir.mkdirs();
+ dbFile = new File(otherDir, "_file2.db");
+ dbFile.createNewFile();
+ file = new File(otherDir, "conf.conf");
+ file.createNewFile();
+ }
+ catch (IOException e) {
+ fail("Failed in setup of folder: " + e);
+ }
+ assertTrue("Precondition failure: index directory does not exist", luceneDir.exists());
+ assertTrue("Precondition failure: other index directory does not exist", otherDir.exists());
+
+ LuceneIndexStore li = new LuceneIndexStore(luceneFolder, version);
+ li.delete();
+
+ assertFalse(luceneDir.exists());
+ assertTrue(luceneFolder.exists());
+ assertTrue(otherDir.exists());
+ assertTrue(dbFile.exists());
+ }
+
+
+
+
+ @Test
+ public void testGetPath() throws IOException
+ {
+ int version = 2;
+ File luceneFolder = baseFolder.newFolder("tickets", "lucene");
+ LuceneIndexStore li = new LuceneIndexStore(luceneFolder, version);
+ Path dir = li.getPath();
+ File luceneDir = new File(luceneFolder, getIndexDir(version));
+ assertEquals(luceneDir.toPath(), dir);
+ }
+
+
+
+ @Test
+ public void testHasIndex() throws IOException
+ {
+ int version = 0;
+ File luceneFolder = new File(baseFolder.getRoot(), "ticktock/lucene");
+
+ LuceneIndexStore li = new LuceneIndexStore(luceneFolder, version);
+ assertFalse(li.hasIndex());
+
+ baseFolder.newFolder("ticktock");
+ li = new LuceneIndexStore(luceneFolder, version);
+ assertFalse(li.hasIndex());
+
+ baseFolder.newFolder("ticktock", "lucene");
+ li = new LuceneIndexStore(luceneFolder, version);
+ assertFalse(li.hasIndex());
+
+ File luceneDir = baseFolder.newFolder("ticktock", "lucene", getIndexDir(version));
+ li = new LuceneIndexStore(luceneFolder, version);
+ assertFalse(li.hasIndex());
+
+ new File(luceneDir, "write.lock").createNewFile();
+ li = new LuceneIndexStore(luceneFolder, version);
+ assertFalse(li.hasIndex());
+
+ new File(luceneDir, "segments_1").createNewFile();
+ li = new LuceneIndexStore(luceneFolder, version);
+ assertTrue(li.hasIndex());
+
+ }
+
+}
