| #!/usr/bin/python3 |
| |
| # Copyright (C) 2019 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import abc |
| import argparse |
| import enum |
| import os.path |
| import subprocess |
| import sys |
| |
| import requests |
| |
| from git_config_parser import GitConfigParser |
| from init_config import InitConfig |
| from log import get_logger |
| |
| LOG = get_logger("reindex") |
| |
| |
| class IndexType(enum.Enum): |
| LUCENE = enum.auto() |
| ELASTICSEARCH = enum.auto() |
| |
| |
| class GerritAbstractReindexer(abc.ABC): |
| def __init__(self, gerrit_site_path, config): |
| self.gerrit_site_path = gerrit_site_path |
| self.index_config_path = "%s/index/gerrit_index.config" % self.gerrit_site_path |
| self.init_config = config |
| |
| self.configured_indices = self._parse_gerrit_index_config() |
| |
| @abc.abstractmethod |
| def _get_indices(self): |
| pass |
| |
| def _parse_gerrit_index_config(self): |
| indices = dict() |
| if os.path.exists(self.index_config_path): |
| config = GitConfigParser(self.index_config_path) |
| options = config.list() |
| for opt in options: |
| name, version = opt["subsection"].rsplit("_", 1) |
| # TODO (Thomas): Properly handle multiple versions of the same index, |
| # which may be present due to online-reindexing in progress. |
| indices[name] = { |
| "version": int(version), |
| "ready": opt["value"].lower() == "true", |
| } |
| return indices |
| |
| def _get_unready_indices(self): |
| unready_indices = [] |
| for index, index_attrs in self.configured_indices.items(): |
| if not index_attrs["ready"]: |
| LOG.info("Index %s not ready.", index) |
| unready_indices.append(index) |
| return unready_indices |
| |
| def _check_index_versions(self): |
| indices = self._get_indices() |
| |
| if not indices: |
| return False |
| |
| for index, index_attrs in self.configured_indices.items(): |
| if index_attrs["version"] is not indices[index]: |
| return False |
| return True |
| |
| def reindex(self, indices=None): |
| LOG.info("Starting to reindex.") |
| command = "java -jar /var/war/gerrit.war reindex -d %s" % self.gerrit_site_path |
| |
| if indices: |
| command += " ".join([" --index %s" % i for i in indices]) |
| |
| reindex_process = subprocess.run(command.split(), stdout=subprocess.PIPE) |
| |
| if reindex_process.returncode > 0: |
| LOG.error( |
| "An error occured, when reindexing Gerrit indices. Exit code: %d", |
| reindex_process.returncode, |
| ) |
| sys.exit(1) |
| |
| LOG.info("Finished reindexing.") |
| |
| def start(self, is_forced): |
| if is_forced: |
| self.reindex() |
| return |
| |
| if not self.configured_indices: |
| LOG.info("gerrit_index.config does not exist. Creating all indices.") |
| self.reindex() |
| return |
| |
| unready_indices = self._get_unready_indices() |
| if unready_indices: |
| self.reindex(unready_indices) |
| |
| if not self._check_index_versions(): |
| LOG.info("Not all indices are up-to-date.") |
| self.reindex() |
| return |
| |
| LOG.info("Skipping reindexing.") |
| |
| |
| class GerritLuceneReindexer(GerritAbstractReindexer): |
| def _get_indices(self): |
| file_list = os.listdir(os.path.join(self.gerrit_site_path, "index")) |
| file_list.remove("gerrit_index.config") |
| lucene_indices = dict() |
| for index in file_list: |
| try: |
| (name, version) = index.split("_") |
| lucene_indices[name] = int(version) |
| except ValueError: |
| LOG.debug("Ignoring invalid file in index-directory: %s", index) |
| return lucene_indices |
| |
| |
| class GerritElasticSearchReindexer(GerritAbstractReindexer): |
| def _get_elasticsearch_config(self): |
| es_config = dict() |
| gerrit_config = GitConfigParser( |
| os.path.join(self.gerrit_site_path, "etc", "gerrit.config") |
| ) |
| es_config["prefix"] = gerrit_config.get( |
| "elasticsearch.prefix", default="" |
| ).lower() |
| es_config["server"] = gerrit_config.get( |
| "elasticsearch.server", default="" |
| ).lower() |
| return es_config |
| |
| def _get_indices(self): |
| es_config = self._get_elasticsearch_config() |
| url = "{url}/{prefix}*".format( |
| url=es_config["server"], prefix=es_config["prefix"] |
| ) |
| try: |
| response = requests.get(url) |
| except requests.exceptions.SSLError: |
| response = requests.get(url, verify=self.init_config.ca_cert_path) |
| |
| es_indices = dict() |
| for index, _ in response.json().items(): |
| try: |
| index = index.replace(es_config["prefix"], "", 1) |
| (name, version) = index.split("_") |
| es_indices[name] = int(version) |
| except ValueError: |
| LOG.debug("Found unknown index: %s", index) |
| |
| return es_indices |
| |
| |
| def get_reindexer(gerrit_site_path, config): |
| gerrit_config = GitConfigParser( |
| os.path.join(gerrit_site_path, "etc", "gerrit.config") |
| ) |
| index_type = gerrit_config.get("index.type", default=IndexType.LUCENE.name) |
| |
| if IndexType[index_type.upper()] is IndexType.LUCENE: |
| return GerritLuceneReindexer(gerrit_site_path, config) |
| |
| if IndexType[index_type.upper()] is IndexType.ELASTICSEARCH: |
| return GerritElasticSearchReindexer(gerrit_site_path, config) |
| |
| raise RuntimeError("Unknown index type %s." % index_type) |
| |
| |
| # pylint: disable=C0103 |
| if __name__ == "__main__": |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| "-s", |
| "--site", |
| help="Path to Gerrit site", |
| dest="site", |
| action="store", |
| default="/var/gerrit", |
| required=True, |
| ) |
| parser.add_argument( |
| "-f", |
| "--force", |
| help="Reindex even if indices are ready.", |
| dest="force", |
| action="store_true", |
| ) |
| parser.add_argument( |
| "-c", |
| "--config", |
| help="Path to configuration file for init process.", |
| dest="config", |
| action="store", |
| required=True, |
| ) |
| args = parser.parse_args() |
| |
| config = InitConfig().parse(args.config) |
| |
| reindexer = get_reindexer(args.site, config) |
| reindexer.start(args.force) |