Support import tag in jiri manifest
Change-Id: I73b0d87300824698d9f96853956d8b4c40a46387
diff --git a/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriManifest.java b/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriManifest.java
index 3c68ba1..3b41f1c 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriManifest.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriManifest.java
@@ -17,6 +17,7 @@
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.commons.lang.StringUtils;
/** Refer https://fuchsia.googlesource.com/jiri/+/HEAD/manifest.md for manifest specification. */
@XmlRootElement(name = "manifest")
@@ -54,7 +55,63 @@
}
static class Import {
- // Don't need all fields, as we don't use it
+ @XmlAttribute(required = true)
+ String manifest;
+
+ @XmlAttribute(required = true)
+ String name;
+
+ @XmlAttribute(required = true)
+ String remote;
+
+ @XmlAttribute
+ String revision;
+
+ @XmlAttribute
+ String remotebranch;
+
+ public Import() {
+ manifest = "";
+ name = "";
+ remote = "";
+ revision = "";
+ remotebranch = "";
+ }
+
+ public void fillDefault() {
+ if (remotebranch.isEmpty()) {
+ remotebranch = "master";
+ }
+ }
+
+ public String Key() {
+ return name + "=" + StringUtils.strip(remote, "/");
+ }
+
+ /** @return the manifest */
+ public String getManifest() {
+ return manifest;
+ }
+
+ /** @return the name */
+ public String getName() {
+ return name;
+ }
+
+ /** @return the remote */
+ public String getRemote() {
+ return remote;
+ }
+
+ /** @return the revision */
+ public String getRevision() {
+ return revision;
+ }
+
+ /** @return the remotebranch */
+ public String getRemotebranch() {
+ return remotebranch;
+ }
}
static class Imports {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriManifestParser.java b/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriManifestParser.java
index 0f80895..55cc503 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriManifestParser.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriManifestParser.java
@@ -14,6 +14,10 @@
package com.googlesource.gerrit.plugins.supermanifest;
+import static com.google.gerrit.reviewdb.client.RefNames.REFS_HEADS;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.googlesource.gerrit.plugins.supermanifest.SuperManifestRefUpdatedListener.GerritRemoteReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.URI;
@@ -28,61 +32,140 @@
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
+import org.apache.commons.lang.StringUtils;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Repository;
class JiriManifestParser {
- public static JiriProjects getProjects(Repository repo, String ref, String manifest)
- throws ConfigInvalidException, IOException {
- Queue<String> q = new LinkedList<>();
- q.add(manifest);
- HashSet<String> processedFiles = new HashSet<>();
- HashMap<String, JiriProjects.Project> projectMap = new HashMap<>();
+ static class ManifestItem {
+ public ManifestItem(
+ String repoKey, String manifest, String ref, String pKey, boolean revisionPinned) {
+ this.repoKey = repoKey;
+ this.manifest = manifest;
+ this.ref = ref;
+ this.revisionPinned = revisionPinned;
+ this.projectKey = pKey;
+ }
- while (q.size() != 0) {
- String file = q.remove();
- if (processedFiles.contains(file)) {
- continue;
- }
- processedFiles.add(file);
- JiriManifest m;
- try {
- m = parseManifest(repo, ref, file);
- } catch (JAXBException | XMLStreamException e) {
- throw new ConfigInvalidException("XML parse error", e);
- }
- if (m.imports.getImports().length != 0) {
- throw new ConfigInvalidException(
- String.format("Manifest %s contains remote imports which are not supported", file));
- }
+ String repoKey;
+ String manifest;
+ String ref;
+ boolean revisionPinned;
- for (JiriProjects.Project project : m.projects.getProjects()) {
- project.fillDefault();
- if (projectMap.containsKey(project.Key())) {
- if (!projectMap.get(project.Key()).equals(project))
- throw new ConfigInvalidException(
- String.format(
- "Duplicate conflicting project %s in manifest %s\n%s\n%s",
- project.Key(),
- file,
- project.toString(),
- projectMap.get(project.Key()).toString()));
- } else {
- projectMap.put(project.Key(), project);
- }
- }
+ // In jiri if import is pinned to a revision and if
+ // we have a corresponding project in the manifest, jiri would
+ // pin that project to same revision. So passing key to match
+ // project to import tag.
+ // For Eg, if you have manifest in manifest2 repo
+ // <manifest><projects><project name="manifest2" .../>
+ // And If you import this from your main manifest
+ // <manifest><imports><import name="manifest2" revision="A"... />
+ // jiri will pin manifest2 project to A as well.
+ String projectKey;
+ }
- URI parentURI;
- try {
- parentURI = new URI(file);
- } catch (URISyntaxException e) {
- throw new ConfigInvalidException("Invalid parent URI", e);
- }
- for (JiriManifest.LocalImport l : m.imports.getLocalImports()) {
- q.add(parentURI.resolve(l.getFile()).getPath());
+ static class RepoMap<K, V extends Repository> extends HashMap<K, V> implements AutoCloseable {
+ @Override
+ public void close() {
+ for (Repository repo : this.values()) {
+ repo.close();
}
}
- return new JiriProjects(projectMap.values().toArray(new JiriProjects.Project[0]));
+ }
+
+ public static JiriProjects getProjects(
+ GerritRemoteReader reader, String repoKey, String ref, String manifest)
+ throws ConfigInvalidException, IOException {
+
+ try (RepoMap<String, Repository> repoMap = new RepoMap<>()) {
+ repoMap.put(repoKey, reader.openRepository(repoKey));
+ Queue<ManifestItem> q = new LinkedList<>();
+ q.add(new ManifestItem(repoKey, manifest, ref, "", false));
+ HashMap<String, HashSet<String>> processedRepoFiles = new HashMap<>();
+ HashMap<String, JiriProjects.Project> projectMap = new HashMap<>();
+
+ while (q.size() != 0) {
+ ManifestItem mi = q.remove();
+ Repository repo = repoMap.get(mi.repoKey);
+ if (repo == null) {
+ repo = reader.openRepository(mi.repoKey);
+ repoMap.put(mi.repoKey, repo);
+ }
+ HashSet<String> processedFiles = processedRepoFiles.get(mi.repoKey);
+ if (processedFiles == null) {
+ processedFiles = new HashSet<String>();
+ processedRepoFiles.put(mi.repoKey, processedFiles);
+ }
+ if (processedFiles.contains(mi.manifest)) {
+ continue;
+ }
+ processedFiles.add(mi.manifest);
+ JiriManifest m;
+ try {
+ m = parseManifest(repo, mi.ref, mi.manifest);
+ } catch (JAXBException | XMLStreamException e) {
+ throw new ConfigInvalidException("XML parse error", e);
+ }
+
+ for (JiriProjects.Project project : m.projects.getProjects()) {
+ project.fillDefault();
+ if (mi.revisionPinned && project.Key().equals(mi.projectKey)) {
+ project.setRevision(mi.ref);
+ }
+ if (projectMap.containsKey(project.Key())) {
+ if (!projectMap.get(project.Key()).equals(project))
+ throw new ConfigInvalidException(
+ String.format(
+ "Duplicate conflicting project %s in manifest %s\n%s\n%s",
+ project.Key(),
+ mi.manifest,
+ project.toString(),
+ projectMap.get(project.Key()).toString()));
+ } else {
+ projectMap.put(project.Key(), project);
+ }
+ }
+
+ URI parentURI;
+ try {
+ parentURI = new URI(mi.manifest);
+ } catch (URISyntaxException e) {
+ throw new ConfigInvalidException("Invalid parent URI", e);
+ }
+ for (JiriManifest.LocalImport l : m.imports.getLocalImports()) {
+ ManifestItem tw =
+ new ManifestItem(
+ mi.repoKey,
+ parentURI.resolve(l.getFile()).getPath(),
+ mi.ref,
+ mi.projectKey,
+ mi.revisionPinned);
+ q.add(tw);
+ }
+
+ for (JiriManifest.Import i : m.imports.getImports()) {
+ i.fillDefault();
+ URI uri;
+ try {
+ uri = new URI(i.getRemote());
+ } catch (URISyntaxException e) {
+ throw new ConfigInvalidException("Invalid URI", e);
+ }
+ String iRepoKey = new Project.NameKey(StringUtils.strip(uri.getPath(), "/")).toString();
+ String iRef = i.getRevision();
+ boolean revisionPinned = true;
+ if (iRef.isEmpty()) {
+ iRef = REFS_HEADS + i.getRemotebranch();
+ revisionPinned = false;
+ }
+
+ ManifestItem tmi =
+ new ManifestItem(iRepoKey, i.getManifest(), iRef, i.Key(), revisionPinned);
+ q.add(tmi);
+ }
+ }
+ return new JiriProjects(projectMap.values().toArray(new JiriProjects.Project[0]));
+ }
}
private static JiriManifest parseManifest(Repository repo, String ref, String file)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriProjects.java b/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriProjects.java
index 271e52e..31913b2 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriProjects.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriProjects.java
@@ -18,6 +18,7 @@
import java.util.Comparator;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
+import org.apache.commons.lang.StringUtils;
class JiriProjects {
@XmlElement(name = "project")
@@ -172,7 +173,12 @@
}
public String Key() {
- return name + "=" + remote;
+ return name + "=" + StringUtils.strip(remote, "/");
+ }
+
+ /** @param revision the revision to set */
+ public void setRevision(String revision) {
+ this.revision = revision;
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriUpdater.java b/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriUpdater.java
index 150fc59..264496f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriUpdater.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/supermanifest/JiriUpdater.java
@@ -249,11 +249,13 @@
@Override
public void update(GerritRemoteReader reader, ConfigEntry c, String srcRef)
throws IOException, GitAPIException, ConfigInvalidException {
- Repository srcRepo = reader.openRepository(c.getSrcRepoKey().toString());
- Repository destRepo = reader.openRepository(c.getDestRepoKey().toString());
- JiriProjects projects = JiriManifestParser.getProjects(srcRepo, srcRef, c.getXmlPath());
- String targetRef = c.getDestBranch().equals("*") ? srcRef : REFS_HEADS + c.getDestBranch();
- updateSubmodules(
- destRepo, targetRef, URI.create(c.getDestRepoKey().toString() + "/"), projects, reader);
+ try (Repository destRepo = reader.openRepository(c.getDestRepoKey().toString())) {
+ JiriProjects projects =
+ JiriManifestParser.getProjects(
+ reader, c.getSrcRepoKey().toString(), srcRef, c.getXmlPath());
+ String targetRef = c.getDestBranch().equals("*") ? srcRef : REFS_HEADS + c.getDestBranch();
+ updateSubmodules(
+ destRepo, targetRef, URI.create(c.getDestRepoKey().toString() + "/"), projects, reader);
+ }
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/supermanifest/JiriSuperManifestIT.java b/src/test/java/com/googlesource/gerrit/plugins/supermanifest/JiriSuperManifestIT.java
index a8b3427..f6f7a55 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/supermanifest/JiriSuperManifestIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/supermanifest/JiriSuperManifestIT.java
@@ -20,6 +20,7 @@
import com.google.gerrit.acceptance.GitUtil;
import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
import com.google.gerrit.acceptance.PushOneCommit;
+import com.google.gerrit.acceptance.PushOneCommit.Result;
import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.extensions.api.projects.BranchApi;
import com.google.gerrit.extensions.restapi.ResourceNotFoundException;
@@ -32,6 +33,7 @@
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.BlobBasedConfig;
import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.revwalk.RevCommit;
import org.junit.Test;
@TestPlugin(
@@ -172,6 +174,275 @@
assertThat(branch.file("project3").getContentType()).isEqualTo("x-git/gitlink; charset=UTF-8");
}
+ @Test
+ public void ImportTagWorks() throws Exception {
+ setupTestRepos("project");
+
+ // Make sure the manifest exists so the configuration loads successfully.
+ NameKey manifest1Key = createProject("manifest1");
+ TestRepository<InMemoryRepository> manifest1Repo = cloneProject(manifest1Key, admin);
+
+ NameKey manifest2Key = createProject("manifest2");
+ TestRepository<InMemoryRepository> manifest2Repo = cloneProject(manifest2Key, admin);
+
+ NameKey superKey = createProject("superproject");
+ cloneProject(superKey, admin);
+
+ pushConfig(
+ "[superproject \""
+ + superKey.get()
+ + ":refs/heads/destbranch\"]\n"
+ + " srcRepo = "
+ + manifest1Key.get()
+ + "\n"
+ + " srcRef = refs/heads/srcbranch\n"
+ + " srcPath = default\n"
+ + " toolType = jiri\n");
+
+ // XML change will trigger commit to superproject.
+ String xml1 =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<manifest>\n<imports>\n"
+ + "<import name=\""
+ + manifest2Key.get()
+ + "\" manifest=\"default\" remote=\""
+ + canonicalWebUrl.get()
+ + manifest2Key.get()
+ + "\" />\n</imports>"
+ + "<projects>\n"
+ + "<project name=\""
+ + testRepoKeys[0].get()
+ + "\" remote=\""
+ + canonicalWebUrl.get()
+ + testRepoKeys[0].get()
+ + "\" path=\"project1\" />\n"
+ + "</projects>\n</manifest>\n";
+
+ String xml2 =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<manifest>\n<projects>\n"
+ + "<project name=\""
+ + manifest2Key.get()
+ + "\" remote=\""
+ + canonicalWebUrl.get()
+ + manifest2Key.get()
+ + "\" path=\"manifest2\" />\n"
+ + "</projects>\n</manifest>\n";
+ pushFactory
+ .create(db, admin.getIdent(), manifest2Repo, "Subject", "default", xml2)
+ .to("refs/heads/master")
+ .assertOkStatus();
+ pushFactory
+ .create(db, admin.getIdent(), manifest1Repo, "Subject", "default", xml1)
+ .to("refs/heads/srcbranch")
+ .assertOkStatus();
+
+ BranchApi branch = gApi.projects().name(superKey.get()).branch("refs/heads/destbranch");
+ assertThat(branch.file("project1").getContentType()).isEqualTo("x-git/gitlink; charset=UTF-8");
+ assertThat(branch.file("manifest2").getContentType()).isEqualTo("x-git/gitlink; charset=UTF-8");
+ }
+
+ @Test
+ public void ImportTagWithRevisionWorks() throws Exception {
+ setupTestRepos("project");
+
+ // Make sure the manifest exists so the configuration loads successfully.
+ NameKey manifest1Key = createProject("manifest1");
+ TestRepository<InMemoryRepository> manifest1Repo = cloneProject(manifest1Key, admin);
+
+ NameKey manifest2Key = createProject("manifest2");
+ TestRepository<InMemoryRepository> manifest2Repo = cloneProject(manifest2Key, admin);
+
+ NameKey superKey = createProject("superproject");
+ cloneProject(superKey, admin);
+
+ pushConfig(
+ "[superproject \""
+ + superKey.get()
+ + ":refs/heads/destbranch\"]\n"
+ + " srcRepo = "
+ + manifest1Key.get()
+ + "\n"
+ + " srcRef = refs/heads/srcbranch\n"
+ + " srcPath = default\n"
+ + " toolType = jiri\n");
+
+ String xml2 =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<manifest>\n<projects>\n"
+ + "<project name=\""
+ + manifest2Key.get()
+ + "\" remote=\""
+ + canonicalWebUrl.get()
+ + manifest2Key.get()
+ + "\" path=\"manifest2\" />\n"
+ + "</projects>\n</manifest>\n";
+ Result c =
+ pushFactory
+ .create(db, admin.getIdent(), manifest2Repo, "Subject", "default", xml2)
+ .to("refs/heads/master");
+ c.assertOkStatus();
+ RevCommit commit = c.getCommit();
+
+ // Add new project, that should not be imported
+ xml2 =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<manifest>\n<projects>\n"
+ + "<project name=\""
+ + manifest2Key.get()
+ + "\" remote=\""
+ + canonicalWebUrl.get()
+ + manifest2Key.get()
+ + "\" path=\"manifest2\" />\n"
+ + "<project name=\""
+ + testRepoKeys[1].get()
+ + "\" remote=\""
+ + canonicalWebUrl.get()
+ + testRepoKeys[1].get()
+ + "\" path=\"project2\" />\n"
+ + "</projects>\n</manifest>\n";
+
+ pushFactory
+ .create(db, admin.getIdent(), manifest2Repo, "Subject", "default", xml2)
+ .to("refs/heads/master")
+ .assertOkStatus();
+
+ // XML change will trigger commit to superproject.
+ String xml1 =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<manifest>\n<imports>\n"
+ + "<import name=\""
+ + manifest2Key.get()
+ + "\" manifest=\"default\" remote=\""
+ + canonicalWebUrl.get()
+ + manifest2Key.get()
+ + "\" revision=\""
+ + commit.name()
+ + "\"/>\n</imports>"
+ + "<projects>\n"
+ + "<project name=\""
+ + testRepoKeys[0].get()
+ + "\" remote=\""
+ + canonicalWebUrl.get()
+ + testRepoKeys[0].get()
+ + "\" path=\"project1\" />\n"
+ + "</projects>\n</manifest>\n";
+
+ pushFactory
+ .create(db, admin.getIdent(), manifest1Repo, "Subject", "default", xml1)
+ .to("refs/heads/srcbranch")
+ .assertOkStatus();
+
+ BranchApi branch = gApi.projects().name(superKey.get()).branch("refs/heads/destbranch");
+ assertThat(branch.file("project1").getContentType()).isEqualTo("x-git/gitlink; charset=UTF-8");
+ assertThat(branch.file("manifest2").getContentType()).isEqualTo("x-git/gitlink; charset=UTF-8");
+ assertThat(branch.file("manifest2").asString()).contains(commit.name());
+ try {
+ branch.file("project2");
+ fail("wanted exception");
+ } catch (ResourceNotFoundException e) {
+ // all fine.
+ }
+ }
+
+ @Test
+ public void ImportTagWithRemoteBranchWorks() throws Exception {
+ setupTestRepos("project");
+
+ // Make sure the manifest exists so the configuration loads successfully.
+ NameKey manifest1Key = createProject("manifest1");
+ TestRepository<InMemoryRepository> manifest1Repo = cloneProject(manifest1Key, admin);
+
+ NameKey manifest2Key = createProject("manifest2");
+ TestRepository<InMemoryRepository> manifest2Repo = cloneProject(manifest2Key, admin);
+
+ NameKey superKey = createProject("superproject");
+ cloneProject(superKey, admin);
+
+ pushConfig(
+ "[superproject \""
+ + superKey.get()
+ + ":refs/heads/destbranch\"]\n"
+ + " srcRepo = "
+ + manifest1Key.get()
+ + "\n"
+ + " srcRef = refs/heads/srcbranch\n"
+ + " srcPath = default\n"
+ + " toolType = jiri\n");
+
+ String xml2 =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<manifest>\n<projects>\n"
+ + "<project name=\""
+ + manifest2Key.get()
+ + "\" remote=\""
+ + canonicalWebUrl.get()
+ + manifest2Key.get()
+ + "\" path=\"manifest2\" />\n"
+ + "</projects>\n</manifest>\n";
+ pushFactory
+ .create(db, admin.getIdent(), manifest2Repo, "Subject", "default", xml2)
+ .to("refs/heads/b1")
+ .assertOkStatus();
+
+ // Add new project, that should not be imported
+ xml2 =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<manifest>\n<projects>\n"
+ + "<project name=\""
+ + manifest2Key.get()
+ + "\" remote=\""
+ + canonicalWebUrl.get()
+ + manifest2Key.get()
+ + "\" path=\"manifest2\" />\n"
+ + "<project name=\""
+ + testRepoKeys[1].get()
+ + "\" remote=\""
+ + canonicalWebUrl.get()
+ + testRepoKeys[1].get()
+ + "\" path=\"project2\" />\n"
+ + "</projects>\n</manifest>\n";
+
+ pushFactory
+ .create(db, admin.getIdent(), manifest2Repo, "Subject", "default", xml2)
+ .to("refs/heads/master")
+ .assertOkStatus();
+
+ // XML change will trigger commit to superproject.
+ String xml1 =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<manifest>\n<imports>\n"
+ + "<import name=\""
+ + manifest2Key.get()
+ + "\" manifest=\"default\" remote=\""
+ + canonicalWebUrl.get()
+ + manifest2Key.get()
+ + "\" remotebranch=\"b1\" />\n</imports>"
+ + "<projects>\n"
+ + "<project name=\""
+ + testRepoKeys[0].get()
+ + "\" remote=\""
+ + canonicalWebUrl.get()
+ + testRepoKeys[0].get()
+ + "\" path=\"project1\" />\n"
+ + "</projects>\n</manifest>\n";
+
+ pushFactory
+ .create(db, admin.getIdent(), manifest1Repo, "Subject", "default", xml1)
+ .to("refs/heads/srcbranch")
+ .assertOkStatus();
+
+ BranchApi branch = gApi.projects().name(superKey.get()).branch("refs/heads/destbranch");
+ assertThat(branch.file("project1").getContentType()).isEqualTo("x-git/gitlink; charset=UTF-8");
+ assertThat(branch.file("manifest2").getContentType()).isEqualTo("x-git/gitlink; charset=UTF-8");
+ try {
+ branch.file("project2");
+ fail("wanted exception");
+ } catch (ResourceNotFoundException e) {
+ // all fine.
+ }
+ }
+
private void outer() throws Exception {
inner();
}
