Add existing load-testing installation This change adds the load testing installation as suggested in [1]. [1] https://gerrit-review.googlesource.com/c/k8s-gerrit/+/222854 Change-Id: I06b035eb0b748cc6d645129f72b633de449d7333
diff --git a/container/Dockerfile b/container/Dockerfile new file mode 100644 index 0000000..c88a9c0 --- /dev/null +++ b/container/Dockerfile
@@ -0,0 +1,13 @@ +FROM python:3.7-stretch + +WORKDIR /usr/src/app + +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + +COPY tools/ ./ + +RUN pip install pipenv && \ + pipenv install --system + +ENTRYPOINT [ "python", "./start_test.py" ]
diff --git a/container/tools/Pipfile b/container/tools/Pipfile new file mode 100644 index 0000000..84a8cec --- /dev/null +++ b/container/tools/Pipfile
@@ -0,0 +1,14 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" +gitpython = "*" +numpy = "*" + +[requires] +python_version = "3.7"
diff --git a/container/tools/Pipfile.lock b/container/tools/Pipfile.lock new file mode 100644 index 0000000..b187834 --- /dev/null +++ b/container/tools/Pipfile.lock
@@ -0,0 +1,108 @@ +{ + "_meta": { + "hash": { + "sha256": "ad115c17ea23b1608c7391cf91c7ea9f39bd2da751fd77ef53328db95223ead7" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + ], + "version": "==2019.3.9" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "gitdb2": { + "hashes": [ + "sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2", + "sha256:e3a0141c5f2a3f635c7209d56c496ebe1ad35da82fe4d3ec4aaa36278d70648a" + ], + "version": "==2.0.5" + }, + "gitpython": { + "hashes": [ + "sha256:563221e5a44369c6b79172f455584c9ebbb122a13368cc82cb4b5addff788f82", + "sha256:8237dc5bfd6f1366abeee5624111b9d6879393d84745a507de0fda86043b65a8" + ], + "index": "pypi", + "version": "==2.1.11" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "numpy": { + "hashes": [ + "sha256:0e2eed77804b2a6a88741f8fcac02c5499bba3953ec9c71e8b217fad4912c56c", + "sha256:1c666f04553ef70fda54adf097dbae7080645435fc273e2397f26bbf1d127bbb", + "sha256:1f46532afa7b2903bfb1b79becca2954c0a04389d19e03dc73f06b039048ac40", + "sha256:315fa1b1dfc16ae0f03f8fd1c55f23fd15368710f641d570236f3d78af55e340", + "sha256:3d5fcea4f5ed40c3280791d54da3ad2ecf896f4c87c877b113576b8280c59441", + "sha256:48241759b99d60aba63b0e590332c600fc4b46ad597c9b0a53f350b871ef0634", + "sha256:4b4f2924b36d857cf302aec369caac61e43500c17eeef0d7baacad1084c0ee84", + "sha256:54fe3b7ed9e7eb928bbc4318f954d133851865f062fa4bbb02ef8940bc67b5d2", + "sha256:5a8f021c70e6206c317974c93eaaf9bc2b56295b6b1cacccf88846e44a1f33fc", + "sha256:754a6be26d938e6ca91942804eb209307b73f806a1721176278a6038869a1686", + "sha256:771147e654e8b95eea1293174a94f34e2e77d5729ad44aefb62fbf8a79747a15", + "sha256:78a6f89da87eeb48014ec652a65c4ffde370c036d780a995edaeb121d3625621", + "sha256:7fde5c2a3a682a9e101e61d97696687ebdba47637611378b4127fe7e47fdf2bf", + "sha256:80d99399c97f646e873dd8ce87c38cfdbb668956bbc39bc1e6cac4b515bba2a0", + "sha256:88a72c1e45a0ae24d1f249a529d9f71fe82e6fa6a3fd61414b829396ec585900", + "sha256:a4f4460877a16ac73302a9c077ca545498d9fe64e6a81398d8e1a67e4695e3df", + "sha256:a61255a765b3ac73ee4b110b28fccfbf758c985677f526c2b4b39c48cc4b509d", + "sha256:ab4896a8c910b9a04c0142871d8800c76c8a2e5ff44763513e1dd9d9631ce897", + "sha256:abbd6b1c2ef6199f4b7ca9f818eb6b31f17b73a6110aadc4e4298c3f00fab24e", + "sha256:b16d88da290334e33ea992c56492326ea3b06233a00a1855414360b77ca72f26", + "sha256:b78a1defedb0e8f6ae1eb55fa6ac74ab42acc4569c3a2eacc2a407ee5d42ebcb", + "sha256:cfef82c43b8b29ca436560d51b2251d5117818a8d1fb74a8384a83c096745dad", + "sha256:d160e57731fcdec2beda807ebcabf39823c47e9409485b5a3a1db3a8c6ce763e" + ], + "index": "pypi", + "version": "==1.16.3" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "index": "pypi", + "version": "==2.21.0" + }, + "smmap2": { + "hashes": [ + "sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde", + "sha256:29a9ffa0497e7f2be94ca0ed1ca1aa3cd4cf25a1f6b4f5f87f74b46ed91d609a" + ], + "version": "==2.0.5" + }, + "urllib3": { + "hashes": [ + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" + ], + "version": "==1.24.2" + } + }, + "develop": {} +}
diff --git a/container/tools/actions/__init__.py b/container/tools/actions/__init__.py new file mode 100644 index 0000000..5c055f5 --- /dev/null +++ b/container/tools/actions/__init__.py
@@ -0,0 +1,23 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .clone_project import CloneProjectAction +from .create_project import CreateProjectAction +from .fetch_project import FetchProjectAction +from .push_change import PushChangeAction +from .push_commit import PushCommitAction +from .query_change_files import QueryChangeFilesAction +from .query_changes import QueryChangesAction +from .query_projects import QueryProjectsAction +from .review_change import ReviewChangeAction
diff --git a/container/tools/actions/abstract.py b/container/tools/actions/abstract.py new file mode 100644 index 0000000..1718527 --- /dev/null +++ b/container/tools/actions/abstract.py
@@ -0,0 +1,46 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import logging +import random + +import numpy as np + +class AbstractAction(abc.ABC): + + def __init__(self, url, user, pwd, probability=1.0): + self.url = url + self.user = user + self.pwd = pwd + self.probability = probability + self.was_executed = False + self.failed = False + + self.log = logging.getLogger("ActionLogger") + + @abc.abstractmethod + def execute(self): + pass + + def _log_result(self, message=""): + self.log.info("{action} {result} {message}".format( + action = self.__class__.__name__, + result = "FAILED" if self.failed else "OK", + message = message + )) + + def _is_executed(self): + return np.random.choice( + (True, False), 1, p=(self.probability, 1 - self.probability))
diff --git a/container/tools/actions/abstract_push.py b/container/tools/actions/abstract_push.py new file mode 100644 index 0000000..c84135b --- /dev/null +++ b/container/tools/actions/abstract_push.py
@@ -0,0 +1,108 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import enum +import os +import random +import shutil +import string +import traceback + +import git + +from . import abstract + +class AbstractPushAction(abstract.AbstractAction): + + def __init__(self, refspec, project_name, probability=0.2): + super().__init__(url=None, user=None, pwd=None, probability=probability) + self.project_name = project_name + self.CHANGE_TYPES = [ + self._add_file, + self._modify_file, + self._delete_file + ] + self.local_repo_path = os.path.join("/tmp", self.project_name) + self.repo = git.Repo(self.local_repo_path) + self.refspec = refspec + + def _push(self): + self.repo.git.checkout('origin/master') + if os.path.exists(self.local_repo_path) and self._is_executed(): + try: + num_commits = random.randint(1, 5) + for _ in range(num_commits): + self._create_commit() + self.repo.remotes.origin.push(refspec=self.refspec) + self.was_executed = True + self._log_result("Pushed %d commits to project %s using refspec %s" % ( + num_commits, + self.project_name, + self.refspec)) + except Exception: + self.failed = True + self._log_result(traceback.format_exc().replace("\n", " ")) + + def _create_commit(self): + change_type_choice = random.choice(self.CHANGE_TYPES) + change_type_choice() + self.repo.index.commit("%s\n\n" % self._create_random_string(16)) + + def _add_file(self): + while True: + file_name = self._create_random_string(8) + file_path = os.path.join("/tmp", self.project_name, file_name) + if os.path.exists(file_path): + continue + with open(file_path, "w+") as f: + self.log.info("Adding file %s to commit", file_name) + f.write(self._create_random_string(random.randint(1, 2000))) + self.repo.index.add([file_name]) + break + + def _modify_file(self): + files = self._get_files_in_repo_workdir() + if len(files) <= 0: + self.log.info("Repository does not contain any files yet. Adding instead of modifying") + self._add_file() + return + file_name = random.choice(files) + file_path = os.path.join("/tmp", self.project_name, file_name) + with open(file_path, "a+") as f: + self.log.info("Modifying file %s for commit", file_name) + f.write(self._create_random_string(random.randint(1, 2000))) + self.repo.index.add([file_name]) + + def _delete_file(self): + files = self._get_files_in_repo_workdir() + if len(files) <= 0: + self.log.info("Repository does not contain any files yet. Adding instead of deleting") + self._add_file() + return + file_name = random.choice(files) + self.log.info("Deleting file %s to commit", file_name) + os.remove(os.path.join("/tmp", self.project_name, file_name)) + self.repo.index.remove([file_name]) + + def _get_files_in_repo_workdir(self): + files = list() + for (dirpath, dirnames, filenames) in os.walk(os.path.join("/tmp", self.project_name)): + if ".git" not in dirpath: + files += [os.path.join(dirpath, file) for file in filenames] + return files + + def _create_random_string(self, length): + allowed_symbols = string.ascii_letters + string.digits + return "".join( + [random.choice(allowed_symbols) for n in range(length)])
diff --git a/container/tools/actions/clone_project.py b/container/tools/actions/clone_project.py new file mode 100644 index 0000000..91f6514 --- /dev/null +++ b/container/tools/actions/clone_project.py
@@ -0,0 +1,61 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import stat +import traceback + +import git +import requests + +from . import abstract + +class CloneProjectAction(abstract.AbstractAction): + + def __init__(self, url, user, pwd, project_name, probability=0.02): + super().__init__(url, user, pwd, probability) + self.project_name = project_name + + def execute(self): + if self._is_executed(): + try: + local_repo_path = os.path.join("/tmp", self.project_name) + if os.path.exists(local_repo_path): + shutil.rmtree(local_repo_path) + repo = git.Repo.clone_from(self._assemble_url(), local_repo_path) + self._install_commit_hook() + self.was_executed = True + self._log_result(self.project_name) + except Exception: + self.failed = True + self._log_result(traceback.format_exc().replace("\n", " ")) + + + + def _install_commit_hook(self): + hook_path = os.path.join("/tmp", self.project_name, ".git/hooks/commit-msg") + + url = "%s/tools/hooks/commit-msg" % self.url + r = requests.get(url) + + with open(hook_path, 'wb+') as f: + f.write(r.content) + + os.chmod(hook_path, os.stat(hook_path).st_mode | stat.S_IEXEC) + + + def _assemble_url(self): + url = "%s/%s.git" % (self.url, self.project_name) + return url.replace("//", "//%s:%s@" % (self.user, self.pwd))
diff --git a/container/tools/actions/create_project.py b/container/tools/actions/create_project.py new file mode 100644 index 0000000..36947cf --- /dev/null +++ b/container/tools/actions/create_project.py
@@ -0,0 +1,52 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import string +import traceback + +import requests + +from . import abstract + +PROJECT_NAME_LENGTH = 16 + +class CreateProjectAction(abstract.AbstractAction): + + def __init__(self, url, user, pwd, probability=0.00002): + super().__init__(url, user, pwd, probability) + self.project_name = self._get_random_project_name() + + def execute(self): + if self._is_executed(): + try: + rest_url = self._assemble_url() + response = requests.put( + rest_url, + auth=(self.user, self.pwd), + json={"create_empty_commit": "true"}) + self.was_executed = True + self._log_result(self.project_name) + return self.project_name + except Exception: + self.failed = True + self._log_result(traceback.format_exc().replace("\n", " ")) + + def _assemble_url(self): + return "%s/a/projects/%s" % (self.url, self.project_name) + + def _get_random_project_name(self): + allowed_symbols = string.ascii_letters + string.digits + return "".join( + [random.choice(allowed_symbols) for n in range(PROJECT_NAME_LENGTH)])
diff --git a/container/tools/actions/fetch_project.py b/container/tools/actions/fetch_project.py new file mode 100644 index 0000000..25e6583 --- /dev/null +++ b/container/tools/actions/fetch_project.py
@@ -0,0 +1,40 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os.path +import shutil +import traceback + +import git + +from . import abstract + +class FetchProjectAction(abstract.AbstractAction): + + def __init__(self, project_name, probability=0.1): + super().__init__(url=None, user=None, pwd=None, probability=probability) + self.project_name = project_name + + def execute(self): + local_repo_path = os.path.join("/tmp", self.project_name) + if os.path.exists(local_repo_path) and self._is_executed(): + try: + repo = git.Repo(local_repo_path) + for remote in repo.remotes: + remote.fetch() + self.was_executed = True + self._log_result(self.project_name) + except Exception: + self.failed = True + self._log_result(traceback.format_exc().replace("\n", " "))
diff --git a/container/tools/actions/push_change.py b/container/tools/actions/push_change.py new file mode 100644 index 0000000..7b639c2 --- /dev/null +++ b/container/tools/actions/push_change.py
@@ -0,0 +1,23 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import abstract_push + +class PushChangeAction(abstract_push.AbstractPushAction): + + def __init__(self, project_name, probability=0.2): + super().__init__("HEAD:refs/for/master", project_name, probability=probability) + + def execute(self): + self._push()
diff --git a/container/tools/actions/push_commit.py b/container/tools/actions/push_commit.py new file mode 100644 index 0000000..181cd98 --- /dev/null +++ b/container/tools/actions/push_commit.py
@@ -0,0 +1,26 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import abstract_push +from .fetch_project import FetchProjectAction + +class PushCommitAction(abstract_push.AbstractPushAction): + + def __init__(self, project_name, probability=0.2): + super().__init__("HEAD:master", project_name, probability=probability) + + def execute(self): + action = FetchProjectAction(self.project_name, 1.0) + action.execute() + self._push()
diff --git a/container/tools/actions/query_change_files.py b/container/tools/actions/query_change_files.py new file mode 100644 index 0000000..f482eb7 --- /dev/null +++ b/container/tools/actions/query_change_files.py
@@ -0,0 +1,47 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import random +import traceback + +import requests + +from . import abstract + +class QueryChangeFilesAction(abstract.AbstractAction): + + def __init__(self, change_id, url, user, pwd, probability=0.2): + super().__init__(url, user, pwd, probability) + self.change_id = change_id + self.revision_id = 1 + + def execute(self): + if self._is_executed(): + try: + response = requests.get( + self._assemble_url(), + auth=(self.user, self.pwd)) + files = list(json.loads(response.text.split("\n",1)[1]).keys()) + self._log_result(files) + return files + except Exception: + self.failed = True + self._log_result(traceback.format_exc().replace("\n", " ")) + + def _assemble_url(self): + return "%s/a/changes/%s/revisions/%s/files" % ( + self.url, + self.change_id, + self.revision_id)
diff --git a/container/tools/actions/query_changes.py b/container/tools/actions/query_changes.py new file mode 100644 index 0000000..e14fee7 --- /dev/null +++ b/container/tools/actions/query_changes.py
@@ -0,0 +1,49 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import random +import traceback + +import requests + +from . import abstract + +class QueryChangesAction(abstract.AbstractAction): + + def __init__(self, url, user, pwd, probability=0.2): + super().__init__(url, user, pwd, probability) + + def execute(self): + if self._is_executed(): + try: + rest_url = self._assemble_url() + response = requests.get( + rest_url, + auth=(self.user, self.pwd)) + self.was_executed = True + except Exception: + self.failed = True + self._log_result(traceback.format_exc().replace("\n", " ")) + return None + + try: + change = random.choice(json.loads(response.text.split("\n",1)[1])) + self._log_result(change["change_id"]) + return change + except: + return None + + def _assemble_url(self): + return "%s/a/changes/?q=status:open&n=100" % (self.url)
diff --git a/container/tools/actions/query_projects.py b/container/tools/actions/query_projects.py new file mode 100644 index 0000000..509b7e2 --- /dev/null +++ b/container/tools/actions/query_projects.py
@@ -0,0 +1,51 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import random +import traceback + +import requests + +from . import abstract + +DISALLOWED_PROJECTS = ["All-Projects", "All-Users"] + +class QueryProjectsAction(abstract.AbstractAction): + + def __init__(self, url, user, pwd, probability=0.05): + super().__init__(url, user, pwd, probability) + + def execute(self): + if self._is_executed(): + selected_project = None + try: + rest_url = self._assemble_url() + response = requests.get( + rest_url, + auth=(self.user, self.pwd)) + self.was_executed = True + projects = list(json.loads(response.text.split("\n",1)[1]).keys()) + for project in DISALLOWED_PROJECTS: + projects.remove(project) + selected_project = random.choice(projects) + self._log_result(selected_project) + except Exception: + self.failed = True + self._log_result(traceback.format_exc().replace("\n", " ")) + + return selected_project + + def _assemble_url(self): + return "%s/a/projects/?d" % (self.url)
diff --git a/container/tools/actions/review_change.py b/container/tools/actions/review_change.py new file mode 100644 index 0000000..18b845e --- /dev/null +++ b/container/tools/actions/review_change.py
@@ -0,0 +1,80 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import random +import traceback + +import requests + +from . import abstract +from .query_changes import QueryChangesAction +from .query_change_files import QueryChangeFilesAction + +class ReviewChangeAction(abstract.AbstractAction): + + def __init__(self, url, user, pwd, probability=0.3): + super().__init__(url, user, pwd, probability) + self.change_id = self._get_change_id() + self.revision_id = 1 + + def execute(self): + if self._is_executed() and self.change_id: + try: + rest_url = self._assemble_review_url() + response = requests.post( + rest_url, + auth=(self.user, self.pwd), + json=self._assemble_body()) + self.was_executed = True + self._log_result() + except Exception: + self.failed = True + self._log_result(traceback.format_exc().replace("\n", " ")) + + def _get_change_id(self): + try: + return QueryChangesAction( + self.url, self.user, self.pwd, 1.0).execute()["change_id"] + except: + return None + + def _assemble_review_url(self): + return "%s/a/changes/%s/revisions/%s/review" % ( + self.url, + self.change_id, + self.revision_id) + + def _assemble_body(self): + file_to_comment = random.choice(self._list_files()) + label = random.randint(-2, 2) + return { + "tag": "loadtest", + "message": "Yet another comment.", + "labels": { + "Code-Review": label + }, + "comments": { + file_to_comment: [ + { + "line": 1, + "message": "Gibberish!" + } + ] + } + } + + def _list_files(self): + return QueryChangeFilesAction( + self.change_id, self.url, self.user, self.pwd, 1.0).execute()
diff --git a/container/tools/start_test.py b/container/tools/start_test.py new file mode 100755 index 0000000..19a86c2 --- /dev/null +++ b/container/tools/start_test.py
@@ -0,0 +1,190 @@ +#!/usr/bin/python3 + +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import random +import time + +import numpy as np + +from actions import * + +LOG_PATH = '/var/logs/loadtester.log' + +class LoadTestInstance: + + def __init__(self, url, user, pwd, test_duration=None): + self.url = url + self.user = user + self.pwd = pwd + + self.timeout = time.time() + test_duration if test_duration else None + + self.owned_projects = set() + self.cloned_projects = set() + + self._create_initial_projects(1) + + self.log = logging.getLogger("ActionLogger") + + def run(self): + while True: + if self.timeout and time.time() >= self.timeout: + break + + self._wait_random_seconds(1, 10) + + self._exec_create_project_action() + self._exec_list_projects_action() + + if self.owned_projects: + self._exec_clone_project_action() + + if self.cloned_projects: + self._exec_fetch_project_action() + self._exec_push_commit_action() + self._exec_push_change_action() + + self._exec_query_changes_action() + self._exec_review_change_action() + + def _create_initial_projects(self, num_init_projects): + for _ in range(num_init_projects): + self.owned_projects.add(CreateProjectAction( + self.url, + self.user, + self.pwd, + 1.0).execute()) + + def _wait_random_seconds(self, min, max): + wait_duration = random.randint(min, max) + time.sleep(wait_duration) + + def _choose_from_list_poisson(self, l): + probabilities = np.random.poisson(20, len(l)) + probabilities = probabilities / np.sum(probabilities) + return np.random.choice(l, 1, p=probabilities).tolist()[0] + + def _exec_create_project_action(self): + action = CreateProjectAction(self.url, self.user, self.pwd) + project_name = action.execute() + if not action.failed and project_name: + self.owned_projects.add(project_name) + + def _exec_list_projects_action(self): + action = QueryProjectsAction(self.url, self.user, self.pwd) + project_name = action.execute() + if not action.failed and project_name: + self.owned_projects.add(project_name) + + def _exec_clone_project_action(self): + action = CloneProjectAction( + self.url, + self.user, + self.pwd, + self._choose_from_list_poisson(list(self.owned_projects))) + action.execute() + if not action.failed and action.was_executed: + self.cloned_projects.add(action.project_name) + + def _exec_fetch_project_action(self): + action = FetchProjectAction( + self._choose_from_list_poisson(list(self.cloned_projects))) + action.execute() + + def _exec_push_commit_action(self): + action = PushCommitAction( + self._choose_from_list_poisson(list(self.cloned_projects))) + action.execute() + + def _exec_push_change_action(self): + action = PushChangeAction( + self._choose_from_list_poisson(list(self.cloned_projects))) + action.execute() + + def _exec_query_changes_action(self): + action = QueryChangesAction( + self.url, + self.user, + self.pwd) + action.execute() + + def _exec_review_change_action(self): + action = ReviewChangeAction( + self.url, + self.user, + self.pwd) + action.execute() + + +if __name__ == "__main__": + + os.makedirs(os.path.dirname(LOG_PATH), exist_ok=True) + + log_format = '%(asctime)s %(message)s' + + logging.basicConfig( + level=logging.DEBUG, + format=log_format, + filename=LOG_PATH, + filemode='w' + ) + + handler = logging.StreamHandler() + handler.setLevel(logging.DEBUG) + handler.setFormatter(logging.Formatter(log_format)) + logging.getLogger('ActionLogger').addHandler(handler) + + parser = argparse.ArgumentParser() + parser.add_argument( + "-U", + "--url", + help="Gerrit base url", + dest="url", + action="store", + required=True) + + parser.add_argument( + "-u", + "--user", + help="Gerrit user", + dest="user", + action="store", + default="admin") + + parser.add_argument( + "-p", + "--password", + help="Gerrit password", + dest="pwd", + action="store", + default="secret") + + parser.add_argument( + "-d", + "--duration", + help="Test duration in seconds", + dest="duration", + action="store", + type=int, + default=None) + + args = parser.parse_args() + + test = LoadTestInstance(args.url, args.user, args.pwd, args.duration) + test.run()
diff --git a/efk/elasticsearch.yaml b/efk/elasticsearch.yaml new file mode 100644 index 0000000..b8fd740 --- /dev/null +++ b/efk/elasticsearch.yaml
@@ -0,0 +1,86 @@ +# Versions supported by Gerrit: https://www.gerritcodereview.com/elasticsearch.html +appVersion: "6.5.4" + +image: + # Use only repositories with `-oss`-prefix, if you do not own a commercial + # elasticsearch license! + repository: "docker.elastic.co/elasticsearch/elasticsearch-oss" + tag: "6.5.4" + pullPolicy: "IfNotPresent" + +initImage: + repository: "busybox" + tag: "latest" + pullPolicy: "Always" + +cluster: + name: "elasticsearch-gerrit-index" + # Xpack is commercial and has to be deactivated (not included in oss-images + # anyway) + xpackEnable: false + env: + MINIMUM_MASTER_NODES: "2" + plugins: [] + +client: + name: client + replicas: 2 + serviceType: ClusterIP + heapSize: "512m" + antiAffinity: "soft" + resources: + limits: + cpu: "1" + memory: "1024Mi" + requests: + cpu: "25m" + memory: "512Mi" + ingress: + enabled: false + +master: + name: master + exposeHttp: false + replicas: 3 + heapSize: "512m" + persistence: + enabled: true + size: "4Gi" + antiAffinity: "soft" + resources: + limits: + cpu: "1" + memory: "1024Mi" + requests: + cpu: "25m" + memory: "512Mi" + podManagementPolicy: OrderedReady + updateStrategy: + type: OnDelete + +data: + name: data + exposeHttp: false + replicas: 2 + heapSize: "1536m" + persistence: + enabled: true + size: "30Gi" + terminationGracePeriodSeconds: 3600 + antiAffinity: "soft" + resources: + limits: + cpu: "1" + memory: "3Gi" + requests: + cpu: "25m" + memory: "1536Mi" + podManagementPolicy: OrderedReady + updateStrategy: + type: OnDelete + hooks: + drain: + enabled: true + +sysctlInitContainer: + enabled: true
diff --git a/efk/fluentbit.yaml b/efk/fluentbit.yaml new file mode 100644 index 0000000..67c52fc --- /dev/null +++ b/efk/fluentbit.yaml
@@ -0,0 +1,19 @@ +image: + fluent_bit: + repository: fluent/fluent-bit + tag: 1.0.6 + +backend: + type: es + es: + host: es-index-elasticsearch-client.elasticsearch.svc.cluster.local + time_key: "@timestamp_fb" + port: 9200 + +parsers: + enabled: true + regex: + - name: loadtester + regex: '^(?<time>\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2},\d{3})\s(?<action>[^\s]+)\s(?<result>(?:OK)|(?:FAILED))\s(?<message>.*)$' + timeKey: time + timeFormat: "%Y-%d-%m %H:%M:%S.%f"
diff --git a/efk/kibana.yaml b/efk/kibana.yaml new file mode 100644 index 0000000..8937447 --- /dev/null +++ b/efk/kibana.yaml
@@ -0,0 +1,22 @@ +image: + repository: "docker.elastic.co/kibana/kibana-oss" + tag: "6.5.4" + +env: + ELASTICSEARCH_URL: http://es-index-elasticsearch-client.elasticsearch.svc.cluster.local:9200 +files: + kibana.yml: + server.name: kibana + server.host: "0" + elasticsearch.url: http://es-index-elasticsearch-client.elasticsearch.svc.cluster.local:9200 +service: + type: ClusterIP + +dashboardImport: + enabled: false + timeout: 60 + xpackauth: + enabled: false + dashboards: + loadbalancer: |- + {}
diff --git a/load-tester.deployment.yaml b/load-tester.deployment.yaml new file mode 100644 index 0000000..4c8ec73 --- /dev/null +++ b/load-tester.deployment.yaml
@@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gerrit-load-tester +spec: + replicas: 15 + selector: + matchLabels: + app: gerrit-load-tester + template: + metadata: + annotations: + fluentbit.io/parser: loadtester + labels: + app: gerrit-load-tester + spec: + containers: + - name: gerrit-load-tester + image: k8sgerrit/loadtest + args: + - "-U" + - "http://gerrit.example.com" + resources: + limits: + memory: "128Mi" + cpu: "500m"