Merge "docs: attention set: 3.3 is no longer upcoming" into stable-3.3
diff --git a/Documentation/rest-api-changes.txt b/Documentation/rest-api-changes.txt
index 6b73482..84e9c7b 100644
--- a/Documentation/rest-api-changes.txt
+++ b/Documentation/rest-api-changes.txt
@@ -150,6 +150,12 @@
The `S` or `start` query parameter can be supplied to skip a number
of changes from the list.
+Administrators can use the `skip-visibility` query parameter to skip visibility filtering.
+This can be used to ensure that no changes are missed e.g. when querying for changes which
+need to be reindexed. Without this parameter query results the user has no permission to read
+are filtered out. REST requests with the skip-visibility option are rejected when the current
+user doesn't have the ADMINISTRATE_SERVER capability.
+
Clients are allowed to specify more than one query by setting the `q`
parameter multiple times. In this case the result is an array of
arrays, one per query in the same order the queries were given in.
diff --git a/contrib/reindex/.flake8 b/contrib/reindex/.flake8
new file mode 100644
index 0000000..151557f
--- /dev/null
+++ b/contrib/reindex/.flake8
@@ -0,0 +1,9 @@
+[flake8]
+max-line-length=100
+ignore=
+ # E203 whitespace before ':'
+ E203,
+ # W503: Line break before binary operator
+ W503,
+ # W504: Line break after binary operator
+ W504
diff --git a/contrib/reindex/.gitignore b/contrib/reindex/.gitignore
new file mode 100644
index 0000000..fd8c78f
--- /dev/null
+++ b/contrib/reindex/.gitignore
@@ -0,0 +1 @@
+changes-to-reindex.list
diff --git a/contrib/reindex/Pipfile b/contrib/reindex/Pipfile
new file mode 100644
index 0000000..21ffd90
--- /dev/null
+++ b/contrib/reindex/Pipfile
@@ -0,0 +1,19 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pygerrit2 = "*"
+requests = "*"
+tqdm = "*"
+
+[dev-packages]
+flake8 = "*"
+black = "*"
+
+[requires]
+python_version = "3.9"
+
+[pipenv]
+allow_prereleases = true
diff --git a/contrib/reindex/Pipfile.lock b/contrib/reindex/Pipfile.lock
new file mode 100644
index 0000000..bb7cc2d
--- /dev/null
+++ b/contrib/reindex/Pipfile.lock
@@ -0,0 +1,248 @@
+{
+ "_meta": {
+ "hash": {
+ "sha256": "37be5a74a22d0e084ebfe168bfdcd7bcaa87ad7b42be66b1d9fbff5e936ebe72"
+ },
+ "pipfile-spec": 6,
+ "requires": {
+ "python_version": "3.9"
+ },
+ "sources": [
+ {
+ "name": "pypi",
+ "url": "https://pypi.org/simple",
+ "verify_ssl": true
+ }
+ ]
+ },
+ "default": {
+ "certifi": {
+ "hashes": [
+ "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
+ "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
+ ],
+ "version": "==2020.12.5"
+ },
+ "chardet": {
+ "hashes": [
+ "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
+ "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+ "version": "==4.0.0"
+ },
+ "idna": {
+ "hashes": [
+ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+ "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==2.10"
+ },
+ "pbr": {
+ "hashes": [
+ "sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9",
+ "sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00"
+ ],
+ "markers": "python_version >= '2.6'",
+ "version": "==5.5.1"
+ },
+ "pygerrit2": {
+ "hashes": [
+ "sha256:d12cff5cc514dd61281d997ea86771e7f818030c3d2ef230b25bb14dae7d3f86"
+ ],
+ "index": "pypi",
+ "version": "==2.0.14"
+ },
+ "requests": {
+ "hashes": [
+ "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
+ "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
+ ],
+ "index": "pypi",
+ "version": "==2.25.1"
+ },
+ "tqdm": {
+ "hashes": [
+ "sha256:38b658a3e4ecf9b4f6f8ff75ca16221ae3378b2e175d846b6b33ea3a20852cf5",
+ "sha256:d4f413aecb61c9779888c64ddf0c62910ad56dcbe857d8922bb505d4dbff0df1"
+ ],
+ "index": "pypi",
+ "version": "==4.54.1"
+ },
+ "urllib3": {
+ "hashes": [
+ "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
+ "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+ "version": "==1.26.2"
+ }
+ },
+ "develop": {
+ "appdirs": {
+ "hashes": [
+ "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41",
+ "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"
+ ],
+ "version": "==1.4.4"
+ },
+ "black": {
+ "hashes": [
+ "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"
+ ],
+ "index": "pypi",
+ "version": "==20.8b1"
+ },
+ "click": {
+ "hashes": [
+ "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
+ "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+ "version": "==7.1.2"
+ },
+ "flake8": {
+ "hashes": [
+ "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839",
+ "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b"
+ ],
+ "index": "pypi",
+ "version": "==3.8.4"
+ },
+ "mccabe": {
+ "hashes": [
+ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+ "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+ ],
+ "version": "==0.6.1"
+ },
+ "mypy-extensions": {
+ "hashes": [
+ "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+ "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+ ],
+ "version": "==0.4.3"
+ },
+ "pathspec": {
+ "hashes": [
+ "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd",
+ "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"
+ ],
+ "version": "==0.8.1"
+ },
+ "pycodestyle": {
+ "hashes": [
+ "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367",
+ "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==2.6.0"
+ },
+ "pyflakes": {
+ "hashes": [
+ "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92",
+ "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==2.2.0"
+ },
+ "regex": {
+ "hashes": [
+ "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538",
+ "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4",
+ "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc",
+ "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa",
+ "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444",
+ "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1",
+ "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af",
+ "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8",
+ "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9",
+ "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88",
+ "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba",
+ "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364",
+ "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e",
+ "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7",
+ "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0",
+ "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31",
+ "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683",
+ "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee",
+ "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b",
+ "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884",
+ "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c",
+ "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e",
+ "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562",
+ "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85",
+ "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c",
+ "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6",
+ "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d",
+ "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b",
+ "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70",
+ "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b",
+ "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b",
+ "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f",
+ "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0",
+ "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5",
+ "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5",
+ "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f",
+ "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e",
+ "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512",
+ "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d",
+ "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917",
+ "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f"
+ ],
+ "version": "==2020.11.13"
+ },
+ "toml": {
+ "hashes": [
+ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
+ "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+ ],
+ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==0.10.2"
+ },
+ "typed-ast": {
+ "hashes": [
+ "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355",
+ "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919",
+ "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d",
+ "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa",
+ "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652",
+ "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75",
+ "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c",
+ "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01",
+ "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d",
+ "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1",
+ "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907",
+ "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c",
+ "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3",
+ "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d",
+ "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b",
+ "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614",
+ "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c",
+ "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb",
+ "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395",
+ "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b",
+ "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41",
+ "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6",
+ "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34",
+ "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe",
+ "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072",
+ "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298",
+ "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91",
+ "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4",
+ "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f",
+ "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"
+ ],
+ "version": "==1.4.1"
+ },
+ "typing-extensions": {
+ "hashes": [
+ "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
+ "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
+ "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+ ],
+ "version": "==3.7.4.3"
+ }
+ }
+}
diff --git a/contrib/reindex/README.md b/contrib/reindex/README.md
new file mode 100644
index 0000000..acb9588
--- /dev/null
+++ b/contrib/reindex/README.md
@@ -0,0 +1,63 @@
+# Incremental reindexing during upgrade of large gerrit site
+
+In order to shorten the downtime needed to reindex changes during a
+Gerrit upgrade the following strategy can be used:
+
+- index preparation
+ - create a full consistent backup
+ - note down the timestamp when the backup was created (backup-time)
+ - create a complete copy of the production system from the backup
+ - upgrade this copy to the new Gerrit version
+ - online reindex this copy
+- upgrade of the production system
+ - make system unavailable so that users can't reach it anymore
+ e.g. by changing port numbers (downtime starts)
+ - take a full backup
+ - run
+
+ ``` bash
+ ./reindex.py -u gerrit-url -s backup-time
+ ```
+
+ to write the list of changes which have been created or modified
+ since the backup for the index preparation was created to a file
+ "changes-to-reindex.list"
+ - upgrade the production system to the new gerrit version skipping
+ reindexing
+ - copy the bulk of the new index from the copy system to the
+ production system
+ - run
+
+ ``` bash
+ ./reindex.py -u gerrit-url
+ ```
+
+ this reindexes all changes which have been created or modified after
+ the backup was taken reading these changes from the file
+ "changes-to-reindex.list"
+ - smoketest the system
+ - make the production system available to the users again
+ (downtime ends)
+
+## Online help
+
+For help on all available options run
+
+``` bash
+./reindex -h
+```
+
+## Python environment
+
+Prerequisites:
+
+- python 3.9
+- pipenv
+
+Install virtual python environment and run the script
+
+``` bash
+pipenv sync
+pipenv shell
+./reindex <options>
+```
diff --git a/contrib/reindex/reindex.py b/contrib/reindex/reindex.py
new file mode 100755
index 0000000..266f5ec
--- /dev/null
+++ b/contrib/reindex/reindex.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+from argparse import ArgumentParser, RawTextHelpFormatter
+from itertools import islice
+import getpass
+import logging
+import os
+
+from pygerrit2 import GerritRestAPI, HTTPBasicAuth, HTTPBasicAuthFromNetrc
+from tqdm import tqdm
+
+EPILOG = """\
+To query the list of changes which have been created or modified since the
+given timestamp and write them to a file "changes-to-reindex.list" run
+$ ./reindex.py -u gerrit-url -s timestamp
+
+To reindex the list of changes in file "changes-to-reindex.list" run
+$ ./reindex.py -u gerrit-url
+"""
+
+
+def _parse_options():
+ parser = ArgumentParser(
+ formatter_class=RawTextHelpFormatter,
+ epilog=EPILOG,
+ )
+ parser.add_argument(
+ "-u",
+ "--url",
+ dest="url",
+ help="gerrit url",
+ )
+ parser.add_argument(
+ "-s",
+ "--since",
+ dest="time",
+ help=(
+ "changes modified after the given 'TIME', inclusive. Must be in the\n"
+ "format '2006-01-02[ 15:04:05[.890][ -0700]]', omitting the time defaults\n"
+ "to 00:00:00 and omitting the timezone defaults to UTC."
+ ),
+ )
+ parser.add_argument(
+ "-f",
+ "--file",
+ default="changes-to-reindex.list",
+ dest="file",
+ help=(
+ "file path to store list of changes if --since is given,\n"
+ "otherwise file path to read list of changes from"
+ ),
+ )
+ parser.add_argument(
+ "-c",
+ "--chunk",
+ default=100,
+ dest="chunksize",
+ help="chunk size defining how many changes are reindexed per request",
+ type=int,
+ )
+ parser.add_argument(
+ "--cert",
+ dest="cert",
+ type=str,
+ help="path to file containing custom ca certificates to trust",
+ )
+ parser.add_argument(
+ "-v",
+ "--verbose",
+ dest="verbose",
+ action="store_true",
+ help="verbose debugging output",
+ )
+ parser.add_argument(
+ "-n",
+ "--netrc",
+ default=True,
+ dest="netrc",
+ action="store_true",
+ help=(
+ "read credentials from .netrc, default to environment variables\n"
+ "USERNAME and PASSWORD, otherwise prompt for credentials interactively"
+ ),
+ )
+ return parser.parse_args()
+
+
+def _chunker(iterable, chunksize):
+ it = map(lambda s: s.strip(), iterable)
+ while True:
+ chunk = list(islice(it, chunksize))
+ if not chunk:
+ return
+ yield chunk
+
+
+class Reindexer:
+ """Class for reindexing Gerrit changes"""
+
+ def __init__(self):
+ self.options = _parse_options()
+ self._init_logger()
+ credentials = self._authenticate()
+ if self.options.cert:
+ certs = os.path.expanduser(self.options.cert)
+ self.api = GerritRestAPI(
+ url=self.options.url, auth=credentials, verify=certs
+ )
+ else:
+ self.api = GerritRestAPI(url=self.options.url, auth=credentials)
+
+ def _init_logger(self):
+ self.logger = logging.getLogger("Reindexer")
+ self.logger.setLevel(logging.DEBUG)
+ h = logging.StreamHandler()
+ if self.options.verbose:
+ h.setLevel(logging.DEBUG)
+ else:
+ h.setLevel(logging.INFO)
+ formatter = logging.Formatter("%(message)s")
+ h.setFormatter(formatter)
+ self.logger.addHandler(h)
+
+ def _authenticate(self):
+ username = password = None
+ if self.options.netrc:
+ auth = HTTPBasicAuthFromNetrc(url=self.options.url)
+ username = auth.username
+ password = auth.password
+ if not username:
+ username = os.environ.get("USERNAME")
+ if not password:
+ password = os.environ.get("PASSWORD")
+ while not username:
+ username = input("user: ")
+ while not password:
+ password = getpass.getpass("password: ")
+ auth = HTTPBasicAuth(username, password)
+ return auth
+
+ def _query(self):
+ start = 0
+ more_changes = True
+ while more_changes:
+ query = f"since:{self.options.time}&start={start}&skip-visibility"
+ for change in self.api.get(f"changes/?q={query}"):
+ more_changes = change.get("_more_changes") is not None
+ start += 1
+ yield change.get("_number")
+ break
+
+ def _query_to_file(self):
+ self.logger.debug(
+ f"writing changes since {self.options.time} to file {self.options.file}:"
+ )
+ with open(self.options.file, "w") as output:
+ for id in self._query():
+ self.logger.debug(id)
+ output.write(f"{id}\n")
+
+ def _reindex_chunk(self, chunk):
+ self.logger.debug(f"indexing {chunk}")
+ response = self.api.post(
+ "/config/server/index.changes",
+ chunk,
+ )
+ self.logger.debug(f"response: {response}")
+
+ def _reindex(self):
+ self.logger.debug(f"indexing changes from file {self.options.file}")
+ with open(self.options.file, "r") as f:
+ with tqdm(unit="changes", desc="Indexed") as pbar:
+ for chunk in _chunker(f, self.options.chunksize):
+ self._reindex_chunk(chunk)
+ pbar.update(len(chunk))
+
+ def execute(self):
+ if self.options.time:
+ self._query_to_file()
+ else:
+ self._reindex()
+
+
+def main():
+ reindexer = Reindexer()
+ reindexer.execute()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/java/com/google/gerrit/httpd/restapi/RestApiServlet.java b/java/com/google/gerrit/httpd/restapi/RestApiServlet.java
index 9533850..3b4aa01 100644
--- a/java/com/google/gerrit/httpd/restapi/RestApiServlet.java
+++ b/java/com/google/gerrit/httpd/restapi/RestApiServlet.java
@@ -108,6 +108,7 @@
import com.google.gerrit.server.RequestInfo;
import com.google.gerrit.server.RequestListener;
import com.google.gerrit.server.audit.ExtendedHttpAuditEvent;
+import com.google.gerrit.server.cache.PerThreadCache;
import com.google.gerrit.server.change.ChangeFinder;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.group.GroupAuditService;
@@ -327,7 +328,7 @@
try (TraceContext traceContext = enableTracing(req, res)) {
List<IdString> path = splitPath(req);
- try {
+ try (PerThreadCache ignored = PerThreadCache.create()) {
RequestInfo requestInfo = createRequestInfo(traceContext, requestUri(req), path);
globals.requestListeners.runEach(l -> l.onRequest(requestInfo));
diff --git a/java/com/google/gerrit/server/AnonymousUser.java b/java/com/google/gerrit/server/AnonymousUser.java
index c96d61a..91d2d05 100644
--- a/java/com/google/gerrit/server/AnonymousUser.java
+++ b/java/com/google/gerrit/server/AnonymousUser.java
@@ -27,6 +27,12 @@
}
@Override
+ public Object getCacheKey() {
+ // Treat all anonymous users as a single user
+ return "anonymous";
+ }
+
+ @Override
public String toString() {
return "ANONYMOUS";
}
diff --git a/java/com/google/gerrit/server/CurrentUser.java b/java/com/google/gerrit/server/CurrentUser.java
index 1955340..75afc04 100644
--- a/java/com/google/gerrit/server/CurrentUser.java
+++ b/java/com/google/gerrit/server/CurrentUser.java
@@ -90,6 +90,12 @@
*/
public abstract GroupMembership getEffectiveGroups();
+ /**
+ * Returns a unique identifier for this user that is intended to be used as a cache key. Returned
+ * object should to implement {@code equals()} and {@code hashCode()} for effective caching.
+ */
+ public abstract Object getCacheKey();
+
/** Unique name of the user on this server, if one has been assigned. */
public Optional<String> getUserName() {
return Optional.empty();
diff --git a/java/com/google/gerrit/server/IdentifiedUser.java b/java/com/google/gerrit/server/IdentifiedUser.java
index 14d74d0..7cafdc0 100644
--- a/java/com/google/gerrit/server/IdentifiedUser.java
+++ b/java/com/google/gerrit/server/IdentifiedUser.java
@@ -390,6 +390,11 @@
return effectiveGroups;
}
+ @Override
+ public Object getCacheKey() {
+ return getAccountId();
+ }
+
public PersonIdent newRefLogIdent() {
return newRefLogIdent(new Date(), TimeZone.getDefault());
}
diff --git a/java/com/google/gerrit/server/InternalUser.java b/java/com/google/gerrit/server/InternalUser.java
index 821a0c6..381819d 100644
--- a/java/com/google/gerrit/server/InternalUser.java
+++ b/java/com/google/gerrit/server/InternalUser.java
@@ -36,6 +36,11 @@
}
@Override
+ public String getCacheKey() {
+ return "internal";
+ }
+
+ @Override
public boolean isInternalUser() {
return true;
}
diff --git a/java/com/google/gerrit/server/PeerDaemonUser.java b/java/com/google/gerrit/server/PeerDaemonUser.java
index 8a8b67a..b27e05c 100644
--- a/java/com/google/gerrit/server/PeerDaemonUser.java
+++ b/java/com/google/gerrit/server/PeerDaemonUser.java
@@ -40,6 +40,11 @@
return GroupMembership.EMPTY;
}
+ @Override
+ public Object getCacheKey() {
+ return getRemoteAddress();
+ }
+
public SocketAddress getRemoteAddress() {
return peer;
}
diff --git a/java/com/google/gerrit/server/cache/PerThreadCache.java b/java/com/google/gerrit/server/cache/PerThreadCache.java
new file mode 100644
index 0000000..b4f79d1
--- /dev/null
+++ b/java/com/google/gerrit/server/cache/PerThreadCache.java
@@ -0,0 +1,146 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.cache;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.gerrit.common.Nullable;
+import java.util.Map;
+import java.util.function.Supplier;
+
+/**
+ * Caches object instances for a request as {@link ThreadLocal} in the serving thread.
+ *
+ * <p>This class is intended to cache objects that have a high instantiation cost, are specific to
+ * the current request and potentially need to be instantiated multiple times while serving a
+ * request.
+ *
+ * <p>This is different from the key-value storage in {@code CurrentUser}: {@code CurrentUser}
+ * offers a key-value storage by providing thread-safe {@code get} and {@code put} methods. Once the
+ * value is retrieved through {@code get} there is not thread-safety anymore - apart from the
+ * retrieved object guarantees. Depending on the implementation of {@code CurrentUser}, it might be
+ * shared between the request serving thread as well as sub- or background treads.
+ *
+ * <p>In comparison to that, this class guarantees thread safety even on non-thread-safe objects as
+ * its cache is tied to the serving thread only. While allowing to cache non-thread-safe objects, it
+ * has the downside of not sharing any objects with background threads or executors.
+ *
+ * <p>Lastly, this class offers a cache, that requires callers to also provide a {@code Supplier} in
+ * case the object is not present in the cache, while {@code CurrentUser} provides a storage where
+ * just retrieving stored values is a valid operation.
+ *
+ * <p>To prevent OOM errors on requests that would cache a lot of objects, this class enforces an
+ * internal limit after which no new elements are cached. All {@code get} calls are served by
+ * invoking the {@code Supplier} after that.
+ */
+public class PerThreadCache implements AutoCloseable {
+ private static final ThreadLocal<PerThreadCache> CACHE = new ThreadLocal<>();
+ /**
+ * Cache at maximum 25 values per thread. This value was chosen arbitrarily. Some endpoints (like
+ * ListProjects) break the assumption that the data cached in a request is limited. To prevent
+ * this class from accumulating an unbound number of objects, we enforce this limit.
+ */
+ private static final int PER_THREAD_CACHE_SIZE = 25;
+
+ /**
+ * Unique key for key-value mappings stored in PerThreadCache. The key is based on the value's
+ * class and a list of identifiers that in combination uniquely set the object apart form others
+ * of the same class.
+ */
+ public static final class Key<T> {
+ private final Class<T> clazz;
+ private final ImmutableList<Object> identifiers;
+
+ /**
+ * Returns a key based on the value's class and an identifier that uniquely identify the value.
+ * The identifier needs to implement {@code equals()} and {@hashCode()}.
+ */
+ public static <T> Key<T> create(Class<T> clazz, Object identifier) {
+ return new Key<>(clazz, ImmutableList.of(identifier));
+ }
+
+ /**
+ * Returns a key based on the value's class and a set of identifiers that uniquely identify the
+ * value. Identifiers need to implement {@code equals()} and {@hashCode()}.
+ */
+ public static <T> Key<T> create(Class<T> clazz, Object... identifiers) {
+ return new Key<>(clazz, ImmutableList.copyOf(identifiers));
+ }
+
+ private Key(Class<T> clazz, ImmutableList<Object> identifiers) {
+ this.clazz = clazz;
+ this.identifiers = identifiers;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(clazz, identifiers);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Key)) {
+ return false;
+ }
+ Key<?> other = (Key<?>) o;
+ return this.clazz == other.clazz && this.identifiers.equals(other.identifiers);
+ }
+ }
+
+ public static PerThreadCache create() {
+ checkState(CACHE.get() == null, "called create() twice on the same request");
+ PerThreadCache cache = new PerThreadCache();
+ CACHE.set(cache);
+ return cache;
+ }
+
+ @Nullable
+ public static PerThreadCache get() {
+ return CACHE.get();
+ }
+
+ public static <T> T getOrCompute(Key<T> key, Supplier<T> loader) {
+ PerThreadCache cache = get();
+ return cache != null ? cache.get(key, loader) : loader.get();
+ }
+
+ private final Map<Key<?>, Object> cache = Maps.newHashMapWithExpectedSize(PER_THREAD_CACHE_SIZE);
+
+ private PerThreadCache() {}
+
+ /**
+ * Returns an instance of {@code T} that was either loaded from the cache or obtained from the
+ * provided {@link Supplier}.
+ */
+ public <T> T get(Key<T> key, Supplier<T> loader) {
+ @SuppressWarnings("unchecked")
+ T value = (T) cache.get(key);
+ if (value == null) {
+ value = loader.get();
+ if (cache.size() < PER_THREAD_CACHE_SIZE) {
+ cache.put(key, value);
+ }
+ }
+ return value;
+ }
+
+ @Override
+ public void close() {
+ CACHE.remove();
+ }
+}
diff --git a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
index 637a5c7..8d92347 100644
--- a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
+++ b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
@@ -2691,12 +2691,10 @@
private void readChangesForReplace() {
try (TraceTimer traceTimer = newTimer("readChangesForReplace")) {
- Collection<ChangeNotes> allNotes =
- notesFactory.createUsingIndexLookup(
- replaceByChange.values().stream().map(r -> r.ontoChange).collect(toList()));
- for (ChangeNotes notes : allNotes) {
- replaceByChange.get(notes.getChangeId()).notes = notes;
- }
+ replaceByChange.values().stream()
+ .map(r -> r.ontoChange)
+ .map(id -> notesFactory.create(project.getNameKey(), id))
+ .forEach(notes -> replaceByChange.get(notes.getChangeId()).notes = notes);
}
}
diff --git a/java/com/google/gerrit/server/permissions/DefaultPermissionBackend.java b/java/com/google/gerrit/server/permissions/DefaultPermissionBackend.java
index 1b029b1..cf6a184 100644
--- a/java/com/google/gerrit/server/permissions/DefaultPermissionBackend.java
+++ b/java/com/google/gerrit/server/permissions/DefaultPermissionBackend.java
@@ -34,6 +34,7 @@
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.PeerDaemonUser;
import com.google.gerrit.server.account.CapabilityCollection;
+import com.google.gerrit.server.cache.PerThreadCache;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
import com.google.inject.Inject;
@@ -104,7 +105,11 @@
public ForProject project(Project.NameKey project) {
try {
ProjectState state = projectCache.get(project).orElseThrow(illegalState(project));
- return projectControlFactory.create(user, state).asForProject();
+ ProjectControl control =
+ PerThreadCache.getOrCompute(
+ PerThreadCache.Key.create(ProjectControl.class, project, user.getCacheKey()),
+ () -> projectControlFactory.create(user, state));
+ return control.asForProject();
} catch (Exception e) {
Throwable cause = e.getCause() != null ? e.getCause() : e;
return FailedPermissionBackend.project(
diff --git a/java/com/google/gerrit/server/query/change/ChangeQueryBuilder.java b/java/com/google/gerrit/server/query/change/ChangeQueryBuilder.java
index 464ba81..2021b2b 100644
--- a/java/com/google/gerrit/server/query/change/ChangeQueryBuilder.java
+++ b/java/com/google/gerrit/server/query/change/ChangeQueryBuilder.java
@@ -629,7 +629,15 @@
}
if ("submittable".equalsIgnoreCase(value)) {
- return new SubmittablePredicate(SubmitRecord.Status.OK);
+ // SubmittablePredicate will match if *any* of the submit records are OK,
+ // but we need to check that they're *all* OK, so check that none of the
+ // submit records match any of the negative cases. To avoid checking yet
+ // more negative cases for CLOSED and FORCED, instead make sure at least
+ // one submit record is OK.
+ return Predicate.and(
+ new SubmittablePredicate(SubmitRecord.Status.OK),
+ Predicate.not(new SubmittablePredicate(SubmitRecord.Status.NOT_READY)),
+ Predicate.not(new SubmittablePredicate(SubmitRecord.Status.RULE_ERROR)));
}
if ("ignored".equalsIgnoreCase(value)) {
diff --git a/java/com/google/gerrit/server/query/change/SingleGroupUser.java b/java/com/google/gerrit/server/query/change/SingleGroupUser.java
index 7947b6b..c451d46 100644
--- a/java/com/google/gerrit/server/query/change/SingleGroupUser.java
+++ b/java/com/google/gerrit/server/query/change/SingleGroupUser.java
@@ -36,4 +36,9 @@
public GroupMembership getEffectiveGroups() {
return groups;
}
+
+ @Override
+ public Object getCacheKey() {
+ return groups.getKnownGroups();
+ }
}
diff --git a/java/com/google/gerrit/server/restapi/change/QueryChanges.java b/java/com/google/gerrit/server/restapi/change/QueryChanges.java
index 0fec476..3c8157b 100644
--- a/java/com/google/gerrit/server/restapi/change/QueryChanges.java
+++ b/java/com/google/gerrit/server/restapi/change/QueryChanges.java
@@ -27,8 +27,11 @@
import com.google.gerrit.index.query.QueryParseException;
import com.google.gerrit.index.query.QueryRequiresAuthException;
import com.google.gerrit.index.query.QueryResult;
+import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.DynamicOptions;
import com.google.gerrit.server.change.ChangeJson;
+import com.google.gerrit.server.permissions.GlobalPermission;
+import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeQueryBuilder;
@@ -49,10 +52,13 @@
private final ChangeQueryBuilder qb;
private final Provider<ChangeQueryProcessor> queryProcessorProvider;
private final HashMap<String, DynamicOptions.DynamicBean> dynamicBeans = new HashMap<>();
+ private final Provider<CurrentUser> userProvider;
+ private final PermissionBackend permissionBackend;
private EnumSet<ListChangesOption> options;
private Integer limit;
private Integer start;
private Boolean noLimit;
+ private Boolean skipVisibility;
@Option(
name = "--query",
@@ -94,6 +100,15 @@
this.noLimit = on;
}
+ @Option(name = "--skip-visibility", usage = "Skip visibility check, only for administrators")
+ public void skipVisibility(boolean on) throws AuthException, PermissionBackendException {
+ if (on) {
+ CurrentUser user = userProvider.get();
+ permissionBackend.user(user).check(GlobalPermission.ADMINISTRATE_SERVER);
+ }
+ skipVisibility = on;
+ }
+
@Override
public void setDynamicBean(String plugin, DynamicOptions.DynamicBean dynamicBean) {
dynamicBeans.put(plugin, dynamicBean);
@@ -103,10 +118,14 @@
QueryChanges(
ChangeJson.Factory json,
ChangeQueryBuilder qb,
- Provider<ChangeQueryProcessor> queryProcessorProvider) {
+ Provider<ChangeQueryProcessor> queryProcessorProvider,
+ Provider<CurrentUser> userProvider,
+ PermissionBackend permissionBackend) {
this.json = json;
this.qb = qb;
this.queryProcessorProvider = queryProcessorProvider;
+ this.userProvider = userProvider;
+ this.permissionBackend = permissionBackend;
options = EnumSet.noneOf(ListChangesOption.class);
}
@@ -152,6 +171,9 @@
if (noLimit != null) {
queryProcessor.setNoLimit(noLimit);
}
+ if (skipVisibility != null) {
+ queryProcessor.enforceVisibility(!skipVisibility);
+ }
dynamicBeans.forEach((p, b) -> queryProcessor.setDynamicBean(p, b));
if (queries == null || queries.isEmpty()) {
diff --git a/javatests/com/google/gerrit/acceptance/api/change/ChangeSubmitRequirementIT.java b/javatests/com/google/gerrit/acceptance/api/change/ChangeSubmitRequirementIT.java
index fd9af0e..eb5b9b0 100644
--- a/javatests/com/google/gerrit/acceptance/api/change/ChangeSubmitRequirementIT.java
+++ b/javatests/com/google/gerrit/acceptance/api/change/ChangeSubmitRequirementIT.java
@@ -22,6 +22,8 @@
import com.google.gerrit.entities.SubmitRecord;
import com.google.gerrit.entities.SubmitRequirement;
import com.google.gerrit.extensions.annotations.Exports;
+import com.google.gerrit.extensions.api.changes.ChangeApi;
+import com.google.gerrit.extensions.api.changes.ReviewInput;
import com.google.gerrit.extensions.common.ChangeInfo;
import com.google.gerrit.extensions.common.SubmitRequirementInfo;
import com.google.gerrit.extensions.config.FactoryModule;
@@ -33,7 +35,6 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
-import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Test;
public class ChangeSubmitRequirementIT extends AbstractDaemonTest {
@@ -93,20 +94,113 @@
assertThat(result.get(0).requirements).containsExactly(reqInfo);
}
+ @Test
+ public void submittableQueryRuleNotReady() throws Exception {
+ ChangeApi change = newChangeApi();
+
+ // Satisfy the default rule.
+ approveChange(change);
+
+ // The custom rule is NOT_READY.
+ rule.block(true);
+ change.index();
+
+ assertThat(queryIsSubmittable()).isEmpty();
+ }
+
+ @Test
+ public void submittableQueryRuleError() throws Exception {
+ ChangeApi change = newChangeApi();
+
+ // Satisfy the default rule.
+ approveChange(change);
+
+ rule.status(Optional.of(SubmitRecord.Status.RULE_ERROR));
+ change.index();
+
+ assertThat(queryIsSubmittable()).isEmpty();
+ }
+
+ @Test
+ public void submittableQueryDefaultRejected() throws Exception {
+ ChangeApi change = newChangeApi();
+
+ // CodeReview:-2 the change, causing the default rule to fail.
+ rejectChange(change);
+
+ rule.status(Optional.of(SubmitRecord.Status.OK));
+ change.index();
+
+ assertThat(queryIsSubmittable()).isEmpty();
+ }
+
+ @Test
+ public void submittableQueryRuleOk() throws Exception {
+ ChangeApi change = newChangeApi();
+
+ // Satisfy the default rule.
+ approveChange(change);
+
+ rule.status(Optional.of(SubmitRecord.Status.OK));
+ change.index();
+
+ List<ChangeInfo> result = queryIsSubmittable();
+ assertThat(result).hasSize(1);
+ assertThat(result.get(0).changeId).isEqualTo(change.info().changeId);
+ }
+
+ @Test
+ public void submittableQueryRuleNoRecord() throws Exception {
+ ChangeApi change = newChangeApi();
+
+ // Satisfy the default rule.
+ approveChange(change);
+
+ // Our custom rule isn't providing any submit records.
+ rule.status(Optional.empty());
+ change.index();
+
+ // is:submittable should return the change, since it was approved and the custom rule is not
+ // blocking it.
+ List<ChangeInfo> result = queryIsSubmittable();
+ assertThat(result).hasSize(1);
+ assertThat(result.get(0).changeId).isEqualTo(change.info().changeId);
+ }
+
+ private List<ChangeInfo> queryIsSubmittable() throws Exception {
+ return gApi.changes().query("is:submittable project:" + project.get()).get();
+ }
+
+ private ChangeApi newChangeApi() throws Exception {
+ return gApi.changes().id(createChange().getChangeId());
+ }
+
+ private void approveChange(ChangeApi changeApi) throws Exception {
+ changeApi.current().review(ReviewInput.approve());
+ }
+
+ private void rejectChange(ChangeApi changeApi) throws Exception {
+ changeApi.current().review(ReviewInput.reject());
+ }
+
@Singleton
private static class CustomSubmitRule implements SubmitRule {
- private final AtomicBoolean block = new AtomicBoolean(true);
+ private Optional<SubmitRecord.Status> recordStatus = Optional.empty();
public void block(boolean block) {
- this.block.set(block);
+ this.status(block ? Optional.of(SubmitRecord.Status.NOT_READY) : Optional.empty());
+ }
+
+ public void status(Optional<SubmitRecord.Status> status) {
+ this.recordStatus = status;
}
@Override
public Optional<SubmitRecord> evaluate(ChangeData changeData) {
- if (block.get()) {
+ if (this.recordStatus.isPresent()) {
SubmitRecord record = new SubmitRecord();
record.labels = new ArrayList<>();
- record.status = SubmitRecord.Status.NOT_READY;
+ record.status = this.recordStatus.get();
record.requirements = ImmutableList.of(req);
return Optional.of(record);
}
diff --git a/javatests/com/google/gerrit/acceptance/api/change/QueryChangesIT.java b/javatests/com/google/gerrit/acceptance/api/change/QueryChangesIT.java
index 33ec556..14704ad 100644
--- a/javatests/com/google/gerrit/acceptance/api/change/QueryChangesIT.java
+++ b/javatests/com/google/gerrit/acceptance/api/change/QueryChangesIT.java
@@ -25,21 +25,27 @@
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.acceptance.AbstractDaemonTest;
import com.google.gerrit.acceptance.NoHttpd;
+import com.google.gerrit.acceptance.PushOneCommit;
import com.google.gerrit.acceptance.UseClockStep;
import com.google.gerrit.acceptance.config.GerritConfig;
import com.google.gerrit.acceptance.testsuite.account.AccountOperations;
import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.acceptance.testsuite.request.RequestScopeOperations;
+import com.google.gerrit.entities.AccessSection;
import com.google.gerrit.entities.Account;
import com.google.gerrit.entities.Patch;
import com.google.gerrit.entities.Permission;
import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.api.changes.ReviewInput;
import com.google.gerrit.extensions.common.ChangeInfo;
+import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.extensions.restapi.BadRequestException;
import com.google.gerrit.extensions.restapi.TopLevelResource;
+import com.google.gerrit.server.project.ProjectConfig;
import com.google.gerrit.server.restapi.change.QueryChanges;
import com.google.inject.Inject;
import com.google.inject.Provider;
+import java.util.Arrays;
import java.util.List;
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
import org.eclipse.jgit.junit.TestRepository;
@@ -50,6 +56,7 @@
@Inject private AccountOperations accountOperations;
@Inject private ProjectOperations projectOperations;
@Inject private Provider<QueryChanges> queryChangesProvider;
+ @Inject private RequestScopeOperations requestScopeOperations;
@Test
@SuppressWarnings("unchecked")
@@ -283,9 +290,91 @@
assertThat(result.get(1).get(0)._number).isEqualTo(numericId2);
}
+ @Test
+ public void skipVisibility_rejectedForNonAdmin() throws Exception {
+ requestScopeOperations.setApiUser(user.id());
+ final QueryChanges queryChanges = queryChangesProvider.get();
+ String query = "is:open repo:" + project.get();
+ queryChanges.addQuery(query);
+ AuthException thrown =
+ assertThrows(AuthException.class, () -> queryChanges.skipVisibility(true));
+ assertThat(thrown).hasMessageThat().isEqualTo("administrate server not permitted");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void skipVisibility_noReadPermission() throws Exception {
+ createChange().getChangeId();
+ requestScopeOperations.setApiUser(admin.id());
+ QueryChanges queryChanges = queryChangesProvider.get();
+
+ queryChanges.addQuery("is:open repo:" + project.get());
+ List<List<ChangeInfo>> result =
+ (List<List<ChangeInfo>>) queryChanges.apply(TopLevelResource.INSTANCE).value();
+ assertThat(result).hasSize(1);
+
+ try (ProjectConfigUpdate u = updateProject(allProjects)) {
+ ProjectConfig cfg = u.getConfig();
+ removeAllBranchPermissions(cfg, Permission.READ);
+ u.save();
+ }
+
+ queryChanges = queryChangesProvider.get();
+ queryChanges.addQuery("is:open repo:" + project.get());
+ List<List<ChangeInfo>> result2 =
+ (List<List<ChangeInfo>>) queryChanges.apply(TopLevelResource.INSTANCE).value();
+ assertThat(result2).hasSize(0);
+
+ queryChanges = queryChangesProvider.get();
+ queryChanges.addQuery("is:open repo:" + project.get());
+ queryChanges.skipVisibility(true);
+ List<List<ChangeInfo>> result3 =
+ (List<List<ChangeInfo>>) queryChanges.apply(TopLevelResource.INSTANCE).value();
+ assertThat(result3).hasSize(1);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void skipVisibility_privateChange() throws Exception {
+ TestRepository<InMemoryRepository> userRepo = cloneProject(project, user);
+ PushOneCommit.Result result =
+ pushFactory.create(user.newIdent(), userRepo).to("refs/for/master");
+ requestScopeOperations.setApiUser(user.id());
+ gApi.changes().id(result.getChangeId()).setPrivate(true);
+
+ requestScopeOperations.setApiUser(admin.id());
+ QueryChanges queryChanges = queryChangesProvider.get();
+
+ queryChanges.addQuery("is:open repo:" + project.get());
+ List<List<ChangeInfo>> result2 =
+ (List<List<ChangeInfo>>) queryChanges.apply(TopLevelResource.INSTANCE).value();
+ assertThat(result2).hasSize(0);
+
+ queryChanges = queryChangesProvider.get();
+ queryChanges.addQuery("is:open repo:" + project.get());
+ queryChanges.skipVisibility(true);
+ List<List<ChangeInfo>> result3 =
+ (List<List<ChangeInfo>>) queryChanges.apply(TopLevelResource.INSTANCE).value();
+ assertThat(result3).hasSize(1);
+ }
+
private static void assertNoChangeHasMoreChangesSet(List<ChangeInfo> results) {
for (ChangeInfo info : results) {
assertThat(info._moreChanges).isNull();
}
}
+
+ private static void removeAllBranchPermissions(ProjectConfig cfg, String... permissions) {
+ for (AccessSection s : ImmutableList.copyOf(cfg.getAccessSections())) {
+ if (s.getName().startsWith("refs/heads/")
+ || s.getName().startsWith("refs/for/")
+ || s.getName().equals("refs/*")) {
+ cfg.upsertAccessSection(
+ s.getName(),
+ updatedSection -> {
+ Arrays.stream(permissions).forEach(p -> updatedSection.remove(Permission.builder(p)));
+ });
+ }
+ }
+ }
}
diff --git a/javatests/com/google/gerrit/server/cache/PerThreadCacheTest.java b/javatests/com/google/gerrit/server/cache/PerThreadCacheTest.java
new file mode 100644
index 0000000..5d420d3
--- /dev/null
+++ b/javatests/com/google/gerrit/server/cache/PerThreadCacheTest.java
@@ -0,0 +1,103 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.cache;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.testing.GerritJUnit.assertThrows;
+
+import java.util.function.Supplier;
+import org.junit.Test;
+
+public class PerThreadCacheTest {
+
+ @SuppressWarnings("TruthIncompatibleType")
+ @Test
+ public void key_respectsClass() {
+ assertThat(PerThreadCache.Key.create(String.class))
+ .isEqualTo(PerThreadCache.Key.create(String.class));
+ assertThat(PerThreadCache.Key.create(String.class))
+ .isNotEqualTo(
+ /* expected: Key<String>, actual: Key<Integer> */ PerThreadCache.Key.create(
+ Integer.class));
+ }
+
+ @Test
+ public void key_respectsIdentifiers() {
+ assertThat(PerThreadCache.Key.create(String.class, "id1"))
+ .isEqualTo(PerThreadCache.Key.create(String.class, "id1"));
+ assertThat(PerThreadCache.Key.create(String.class, "id1"))
+ .isNotEqualTo(PerThreadCache.Key.create(String.class, "id2"));
+ }
+
+ @Test
+ public void endToEndCache() {
+ try (PerThreadCache ignored = PerThreadCache.create()) {
+ PerThreadCache cache = PerThreadCache.get();
+ PerThreadCache.Key<String> key1 = PerThreadCache.Key.create(String.class);
+
+ String value1 = cache.get(key1, () -> "value1");
+ assertThat(value1).isEqualTo("value1");
+
+ Supplier<String> neverCalled =
+ () -> {
+ throw new IllegalStateException("this method must not be called");
+ };
+ assertThat(cache.get(key1, neverCalled)).isEqualTo("value1");
+ }
+ }
+
+ @Test
+ public void cleanUp() {
+ PerThreadCache.Key<String> key = PerThreadCache.Key.create(String.class);
+ try (PerThreadCache ignored = PerThreadCache.create()) {
+ PerThreadCache cache = PerThreadCache.get();
+ String value1 = cache.get(key, () -> "value1");
+ assertThat(value1).isEqualTo("value1");
+ }
+
+ // Create a second cache and assert that it is not connected to the first one.
+ // This ensures that the cleanup is actually working.
+ try (PerThreadCache ignored = PerThreadCache.create()) {
+ PerThreadCache cache = PerThreadCache.get();
+ String value1 = cache.get(key, () -> "value2");
+ assertThat(value1).isEqualTo("value2");
+ }
+ }
+
+ @Test
+ public void doubleInstantiationFails() {
+ try (PerThreadCache ignored = PerThreadCache.create()) {
+ IllegalStateException thrown =
+ assertThrows(IllegalStateException.class, () -> PerThreadCache.create());
+ assertThat(thrown).hasMessageThat().contains("called create() twice on the same request");
+ }
+ }
+
+ @Test
+ public void enforceMaxSize() {
+ try (PerThreadCache cache = PerThreadCache.create()) {
+ // Fill the cache
+ for (int i = 0; i < 50; i++) {
+ PerThreadCache.Key<String> key = PerThreadCache.Key.create(String.class, i);
+ cache.get(key, () -> "cached value");
+ }
+ // Assert that the value was not persisted
+ PerThreadCache.Key<String> key = PerThreadCache.Key.create(String.class, 1000);
+ cache.get(key, () -> "new value");
+ String value = cache.get(key, () -> "directly served");
+ assertThat(value).isEqualTo("directly served");
+ }
+ }
+}
diff --git a/javatests/com/google/gerrit/server/extensions/webui/UiActionsTest.java b/javatests/com/google/gerrit/server/extensions/webui/UiActionsTest.java
index 1cdca1b..de23ef4 100644
--- a/javatests/com/google/gerrit/server/extensions/webui/UiActionsTest.java
+++ b/javatests/com/google/gerrit/server/extensions/webui/UiActionsTest.java
@@ -101,6 +101,11 @@
}
@Override
+ public Object getCacheKey() {
+ return new Object();
+ }
+
+ @Override
public boolean isIdentifiedUser() {
return true;
}
diff --git a/javatests/com/google/gerrit/server/permissions/RefControlTest.java b/javatests/com/google/gerrit/server/permissions/RefControlTest.java
index da28ad1..87db21f 100644
--- a/javatests/com/google/gerrit/server/permissions/RefControlTest.java
+++ b/javatests/com/google/gerrit/server/permissions/RefControlTest.java
@@ -1259,6 +1259,11 @@
}
@Override
+ public Object getCacheKey() {
+ return new Object();
+ }
+
+ @Override
public Optional<String> getUserName() {
return Optional.ofNullable(username);
}
diff --git a/plugins/replication b/plugins/replication
index 7d21e80..011c577 160000
--- a/plugins/replication
+++ b/plugins/replication
@@ -1 +1 @@
-Subproject commit 7d21e802e5057c2c40201de327a4fa5ef4186ab0
+Subproject commit 011c577ef4ee2149d73db974e0ef2fcd3f66bfcf
diff --git a/polygerrit-ui/app/rollup.config.js b/polygerrit-ui/app/rollup.config.js
index c8e9baa..d93b5ea 100644
--- a/polygerrit-ui/app/rollup.config.js
+++ b/polygerrit-ui/app/rollup.config.js
@@ -69,7 +69,13 @@
output: {
format: 'iife',
compact: true,
- plugins: [terser()]
+ plugins: [
+ terser({
+ output: {
+ comments: false
+ }
+ })
+ ]
},
//Context must be set to window to correctly processing global variables
context: 'window',