Convert fetch-job.sh to fetch-job.py A side effect of this reimplementation is that now one remote repository can be fetched into multiple local repositories, which wasn't possible with the previous shell script based implementation. Change-Id: Id9b6a1eca0662b66c593b9da6cf2974644043be8
diff --git a/container-images/fetch-job/Dockerfile b/container-images/fetch-job/Dockerfile index e089da3..e0d0c30 100644 --- a/container-images/fetch-job/Dockerfile +++ b/container-images/fetch-job/Dockerfile
@@ -1,8 +1,21 @@ ARG TAG=latest FROM base:${TAG} +USER root + +COPY dependencies/* /var/tools/ +COPY requirements.txt /var/tools/ +WORKDIR /var/tools + +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN apk update && \ - apk add --no-cache yq + apk add --no-cache python3 && \ + python3 -m venv $VIRTUAL_ENV && \ + python3 -m ensurepip --upgrade && \ + rm -r /usr/lib/python*/ensurepip && \ + python3 -m pip install --require-hashes -r requirements.txt --no-cache --upgrade && \ + python3 -m pipenv install --python 3.12 --system COPY tools/* /var/tools/ @@ -10,4 +23,4 @@ VOLUME ["/var/gerrit/git"] -ENTRYPOINT ["/var/tools/fetch-job.sh"] +ENTRYPOINT ["python3", "/var/tools/fetch-job.py"]
diff --git a/container-images/fetch-job/README.md b/container-images/fetch-job/README.md index 9ddc620..8f9e156 100644 --- a/container-images/fetch-job/README.md +++ b/container-images/fetch-job/README.md
@@ -8,11 +8,11 @@ ## Content * base image -* `fetch-job.sh`: script executing the fetches +* `fetch-job.py`: script executing the fetches ## Start -* execution of the provided `fetch-job.sh` +* execution of the provided `fetch-job.py` ## Configuration
diff --git a/container-images/fetch-job/dependencies/Pipfile b/container-images/fetch-job/dependencies/Pipfile new file mode 100644 index 0000000..ab826bd --- /dev/null +++ b/container-images/fetch-job/dependencies/Pipfile
@@ -0,0 +1,12 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +pyyaml = "~=6.0.3" + +[requires] +python_version = "3.12"
diff --git a/container-images/fetch-job/requirements.in b/container-images/fetch-job/requirements.in new file mode 100644 index 0000000..a7f62a1 --- /dev/null +++ b/container-images/fetch-job/requirements.in
@@ -0,0 +1,3 @@ +setuptools +wheel +pipenv
diff --git a/container-images/fetch-job/requirements.txt b/container-images/fetch-job/requirements.txt new file mode 100644 index 0000000..079853f --- /dev/null +++ b/container-images/fetch-job/requirements.txt
@@ -0,0 +1,56 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes requirements.in +# +certifi==2026.4.22 \ + --hash=sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a \ + --hash=sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580 + # via pipenv +distlib==0.4.0 \ + --hash=sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16 \ + --hash=sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d + # via virtualenv +filelock==3.29.0 \ + --hash=sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90 \ + --hash=sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258 + # via + # python-discovery + # virtualenv +packaging==26.2 \ + --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ + --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 + # via + # pipenv + # wheel +pipenv==2025.0.4 \ + --hash=sha256:36fc2a7841ccdb2f58a9f787b296c2e15dea3b5b79b84d4071812f28b7e8d7a2 \ + --hash=sha256:e1fbe4cfd25ab179f123d1fbb1fa1cdc0b3ffcdb1f21c775dcaa12ccc356f2bb + # via -r requirements.in +platformdirs==4.4.0 \ + --hash=sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85 \ + --hash=sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf + # via + # python-discovery + # virtualenv +python-discovery==1.3.0 \ + --hash=sha256:441d9ced3dfce36e113beb35ca302c71c7ef06f3c0f9c227a0b9bb3bd49b9e9f \ + --hash=sha256:d098f1e86be5d45fe4d14bf1029294aabbd332f4321179dec85e76cddce834b0 + # via virtualenv +virtualenv==21.3.1 \ + --hash=sha256:c2305bc1fddeec40699b8370d13f8d431b0701f00ce895061ce493aeded4426b \ + --hash=sha256:d1a71cf58f2f9228fff23a1f6ec15d39785c6b32e03658d104974247145edd35 + # via pipenv +wheel==0.47.0 \ + --hash=sha256:212281cab4dff978f6cedd499cd893e1f620791ca6ff7107cf270781e587eced \ + --hash=sha256:cc72bd1009ba0cf63922e28f94d9d83b920aa2bb28f798a31d0691b02fa3c9b3 + # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==82.0.1 \ + --hash=sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9 \ + --hash=sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb + # via + # -r requirements.in + # pipenv
diff --git a/container-images/fetch-job/tools/fetch-job.py b/container-images/fetch-job/tools/fetch-job.py new file mode 100755 index 0000000..ad113f4 --- /dev/null +++ b/container-images/fetch-job/tools/fetch-job.py
@@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import subprocess +import sys +import yaml + +GIT_HOME = "/var/gerrit/git" +DEFAULT_CONFIG_FILE = "/var/gerrit/etc/incoming-replication.config.yaml" +DEFAULT_REFSPEC_TEMPLATE = "+refs/heads/*:refs/heads/{remote_name}/*" + + +def parse_timeout(value): + """Convert a timeout string like '5m', '30s', or '120' to seconds (int).""" + if value is None or value == "null": + return None + value = str(value).strip() + if value.endswith("m"): + return int(value[:-1]) * 60 + if value.endswith("s"): + return int(value[:-1]) + return int(value) + + +def git(git_dir, *args, timeout=None): + cmd = ["git", f"--git-dir={git_dir}"] + list(args) + subprocess.run(cmd, check=True, timeout=timeout) + + +def configure_remote(git_dir, remote_name, remote_url, refspec): + subprocess.run( + [ + "git", + f"--git-dir={git_dir}", + "config", + f"remote.{remote_name}.url", + remote_url, + ], + check=True, + ) + subprocess.run( + [ + "git", + f"--git-dir={git_dir}", + "config", + "unset", + "--all", + f"remote.{remote_name}.fetch", + ], + # exit code 5 means the key did not exist — that is fine on first run + check=False, + ) + subprocess.run( + [ + "git", + f"--git-dir={git_dir}", + "config", + f"remote.{remote_name}.fetch", + refspec, + ], + check=True, + ) + + +def process_remote(remote): + remote_name = remote["name"] + server_url = remote["url"].rstrip("/") + timeout_seconds = parse_timeout(remote.get("timeout")) + + for fetch in remote.get("fetch", []): + remote_repo = fetch["remoteRepo"] + local_repo = fetch.get("localRepo") or remote_repo + refspec = fetch.get("refSpec") or DEFAULT_REFSPEC_TEMPLATE.format( + remote_name=remote_name + ) + + remote_url = f"{server_url}/{remote_repo}" + git_dir = f"{GIT_HOME}/{local_repo}.git" + + configure_remote(git_dir, remote_name, remote_url, refspec) + + print(f"Fetching {remote_repo} from {remote_name} into {git_dir}") + try: + subprocess.run( + ["git", f"--git-dir={git_dir}", "fetch", remote_name], + check=True, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired: + print( + f"ERROR: fetch of {remote_repo} from {remote_name} timed out " + f"after {timeout_seconds}s", + file=sys.stderr, + ) + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser( + description="Fetch from remote Git repositories into local Gerrit repositories." + ) + parser.add_argument( + "--config", + default=DEFAULT_CONFIG_FILE, + metavar="CONFIG_FILE", + help=f"Path to the YAML config file (default: {DEFAULT_CONFIG_FILE})", + ) + args = parser.parse_args() + + try: + with open(args.config) as f: + config = yaml.safe_load(f) + except FileNotFoundError: + print(f"ERROR: config file not found: {args.config}", file=sys.stderr) + sys.exit(1) + + for remote in config.get("remotes", []): + process_remote(remote) + + +if __name__ == "__main__": + main()
diff --git a/container-images/fetch-job/tools/fetch-job.sh b/container-images/fetch-job/tools/fetch-job.sh deleted file mode 100755 index 44c3d23..0000000 --- a/container-images/fetch-job/tools/fetch-job.sh +++ /dev/null
@@ -1,75 +0,0 @@ -#!/bin/ash -# Copyright (C) 2011, 2024 SAP SE -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -usage() { - me=`basename "$0"` - echo >&2 "Usage: $me [--config CONFIG_FILE]" - exit 1 -} - -GIT_HOME=/var/gerrit/git -CONFIG_FILE=/var/gerrit/etc/incoming-replication.config.yaml - -while test $# -gt 0 ; do - case "$1" in - --help) - usage - ;; - - --config) - shift - CONFIG_FILE=$1 - shift - ;; - - *) - break - esac -done - -if ! test -f $CONFIG_FILE; then - echo "No config file has been found." - usage -fi - -NUM_REMOTES=$(yq e '.remotes | length' "$CONFIG_FILE") -for i in $(seq 0 $((NUM_REMOTES-1))); do - REMOTE=$(NUM_REMOTE=$i yq e '.remotes[env(NUM_REMOTE)]' "$CONFIG_FILE") - REMOTE_NAME="$(echo "$REMOTE" | yq e '.name' -)" - REMOTE_SERVER_URL="$(echo "$REMOTE" | yq e '.url' -)" - REMOTE_TIMEOUT="$(echo "$REMOTE" | yq e '.timeout' -)" - - NUM_FETCHES=$(echo "$REMOTE" | yq e '.fetch | length' -) - for j in $(seq 0 $((NUM_FETCHES-1))); do - FETCH=$(echo "$REMOTE" | NUM_FETCH=$j yq e '.fetch[env(NUM_FETCH)]' -) - REMOTE_REPO="$(echo "$FETCH" | yq e '.remoteRepo' -)" - - LOCAL_REPO="$(echo "$FETCH" | yq e '.localRepo' -)" - (test -z "$LOCAL_REPO" || [ "$LOCAL_REPO" = "null" ]) && LOCAL_REPO="$REMOTE_REPO" - - REFSPEC="$(echo "$FETCH" | yq e '.refSpec' -)" - (test -z "$REFSPEC" || [ "$REFSPEC" = "null" ]) && REFSPEC="+refs/heads/*:refs/heads/$REMOTE_NAME/*" - - REMOTE_URL="$REMOTE_SERVER_URL/$REMOTE_REPO" - - GIT_DIR="--git-dir=$GIT_HOME/$LOCAL_REPO.git" - git $GIT_DIR config remote.$REMOTE_NAME.url $REMOTE_URL - git $GIT_DIR config --unset-all remote.$REMOTE_NAME.fetch - git $GIT_DIR config remote.$REMOTE_NAME.fetch $REFSPEC - - echo "Fetching $REMOTE_REPO from $REMOTE_NAME" - timeout $REMOTE_TIMEOUT git $GIT_DIR fetch $REMOTE_NAME - done -done