Auto generate the --task--invalid outputs

By additionally auto-generating the content of the 'invalid' test suites
from the content specified in the task_states.md file, it ensures that
all the test suite results stays in sync, making it much easier to write
new tests, or to alter existing ones.

Change-Id: I372f4fe985a71d54e5e0dbcb25a400104b18ddbe
diff --git a/src/main/resources/Documentation/task_states.md b/src/main/resources/Documentation/task_states.md
index 6d99339..e990856 100644
--- a/src/main/resources/Documentation/task_states.md
+++ b/src/main/resources/Documentation/task_states.md
@@ -1177,7 +1177,8 @@
                      "applicable" : false,
                      "hasPass" : true,
                      "name" : "NA Bad PASS query",
-                     "status" : "FAIL"
+                     "status" : "FAIL"      # Only Test Suite: all
+                     "status" : "INVALID"   # Only Test Suite: !all
                   },
                   {
                      "applicable" : false,
@@ -1189,7 +1190,8 @@
                      "applicable" : false,
                      "hasPass" : true,
                      "name" : "NA Bad INPROGRESS query",
-                     "status" : "FAIL"
+                     "status" : "FAIL"      # Only Test Suite: all
+                     "status" : "INVALID"   # Only Test Suite: !all
                   },
                   {
                      "applicable" : true,
@@ -1447,7 +1449,8 @@
                      "applicable" : false,
                      "hasPass" : true,
                      "name" : "NA Bad PASS query",
-                     "status" : "FAIL"
+                     "status" : "FAIL"      # Only Test Suite: all
+                     "status" : "INVALID"   # Only Test Suite: !all
                   },
                   {
                      "applicable" : false,
@@ -1459,7 +1462,8 @@
                      "applicable" : false,
                      "hasPass" : true,
                      "name" : "NA Bad INPROGRESS query",
-                     "status" : "FAIL"
+                     "status" : "FAIL"      # Only Test Suite: all
+                     "status" : "INVALID"   # Only Test Suite: !all
                   },
                   {
                      "applicable" : true,
diff --git a/test/check_task_statuses.sh b/test/check_task_statuses.sh
index 1cabcf9..419d7c8 100755
--- a/test/check_task_statuses.sh
+++ b/test/check_task_statuses.sh
@@ -29,6 +29,20 @@
     fi
 }
 
+# -------- Pre JSON --------
+#
+# pre_json is a "templated json" used in the test docs to express test results. It looks
+# like json but has some extra comments to express when a certain output should be used.
+# These comments look like: "# Only Test Suite: <suite>"
+#
+
+remove_suite() { # suite < pre_json > json
+    grep -v "# Only Test Suite: $1" | \
+    sed -e's/# Only Test Suite:.*$//; s/ *$//'
+}
+
+remove_not_suite() { remove_suite !"$1" ; } # suite < pre_json > json
+
 # --------
 gssh() { ssh -x -p "$PORT" "$SERVER" gerrit "$@" ; } # cmd [args]...
 
@@ -43,20 +57,31 @@
   local out ; out=$("$@" 2>&1) || { echo "$out" ; exit ; }
 }
 
-replace_change_properties() { # file change_token change_number change_id project branch status topic
+# change_token change_number change_id project branch status topic < templated_txt > change_txt
+replace_change_properties() {
+    sed -e "s/_change$1_number/$2/g" \
+        -e "s/_change$1_id/$3/g" \
+        -e "s/_change$1_project/$4/g" \
+        -e "s/_change$1_branch/$5/g" \
+        -e "s/_change$1_status/$6/g" \
+        -e "s/_change$1_topic/$7/g"
+}
 
-    sed -i -e "s/_change$2_number/$3/g" \
-              -e "s/_change$2_id/$4/g" \
-              -e "s/_change$2_project/$5/g" \
-              -e "s/_change$2_branch/$6/g" \
-              -e "s/_change$2_status/$7/g" \
-              -e "s/_change$2_topic/$8/g" "$1"
+replace_default_changes() { # file
+    local out=$(replace_change_properties "1" "${CHANGES[0]}" < "$1")
+    echo "$out" | replace_change_properties "2" "${CHANGES[1]}" > "$1"
 }
 
 replace_user() { # < text_with_testuser > text_with_$USER
     sed -e"s/testuser/$USER/"
 }
 
+json_pp() {
+    python -c "import sys, json; \
+            print json.dumps(json.loads(sys.stdin.read()), indent=3, \
+            separators=(',', ' : '), sort_keys=True)"
+}
+
 example() { # example_num
     awk '/```/{Q++;E=(Q+1)/2};E=='"$1" < "$DOC_STATES" | grep -v '```' | replace_user
 }
@@ -187,16 +212,9 @@
 mkdir -p "$ALL_TASKS" "$USER_TASKS"
 
 CHANGES=($(gssh query "status:open limit:2" | grep 'number:' | awk '{print $2}'))
-replace_change_properties "$DOC_STATES" "1" "${CHANGES[0]}"
-replace_change_properties "$DOC_STATES" "2" "${CHANGES[1]}"
-replace_change_properties "$MYDIR/preview" "1" "${CHANGES[0]}"
-replace_change_properties "$MYDIR/preview" "2" "${CHANGES[1]}"
-replace_change_properties "$MYDIR/preview.invalid" "1" "${CHANGES[0]}"
-replace_change_properties "$MYDIR/preview.invalid" "2" "${CHANGES[1]}"
-replace_change_properties "$MYDIR/invalid" "1" "${CHANGES[0]}"
-replace_change_properties "$MYDIR/invalid" "2" "${CHANGES[1]}"
-replace_change_properties "$MYDIR/invalid-applicable" "1" "${CHANGES[0]}"
-replace_change_properties "$MYDIR/invalid-applicable" "2" "${CHANGES[1]}"
+replace_default_changes "$DOC_STATES"
+replace_default_changes "$MYDIR/preview"
+replace_default_changes "$MYDIR/preview.invalid"
 
 example 1 |sed -e"s/current-user/$USER/" > "$ROOT_CFG"
 example 2 > "$COMMON_CFG"
@@ -206,23 +224,32 @@
 q_setup update_repo "$ALL" "$REMOTE_ALL" "$REF_ALL"
 q_setup update_repo "$USERS" "$REMOTE_USERS" "$REF_USERS"
 
-example 5 | tail -n +3 | sed -e'/^   \.\.\.,/d; s/^   \.\.\./}/; s/^   ],/   ]/' \
-    > "$EXPECTED".all
-
 change3_id=$(gen_change_id)
 change3_number=$(create_repo_change "$OUT/$PROJECT" "$REMOTE_TEST" "$BRANCH" "$change3_id")
 
-replace_change_properties "$EXPECTED".all "3" \
-    "$change3_number" \
-    "$change3_id" \
-    "$PROJECT" \
-    "refs\/heads\/$BRANCH" \
-    "NEW" \
-    ""
+all_pjson=$(example 5 | tail -n +3 | sed -e'/^   \.\.\.,/d; s/^   \.\.\./}/; s/^   ],/   ]/')
 
-"$MYDIR"/strip_non_applicable.py < "$EXPECTED".all | \
+all_pjson=$(echo "$all_pjson" | \
+    replace_change_properties \
+        "3" \
+        "$change3_number" \
+        "$change3_id" \
+        "$PROJECT" \
+        "refs\/heads\/$BRANCH" \
+        "NEW" \
+        "")
+
+no_all_json=$(echo "$all_pjson" | remove_suite all)
+
+echo "$no_all_json" | "$MYDIR"/strip_non_applicable.py | \
     grep -v "\"applicable\" :" > "$EXPECTED".applicable
 
+echo "$all_pjson" | remove_not_suite all | json_pp > "$EXPECTED".all
+
+echo "$no_all_json" | "$MYDIR"/strip_non_invalid.py > "$EXPECTED".invalid
+
+"$MYDIR"/strip_non_invalid.py < "$EXPECTED".applicable > "$EXPECTED".invalid-applicable
+
 RESULT=0
 query="change:$change3_number status:open"
 test_generated applicable --task--applicable "$query"
@@ -233,7 +260,6 @@
 test_file preview --task--preview "$cnum,1" --task--all "$query"
 test_file preview.invalid --task--preview "$cnum,1" --task--invalid "$query"
 
-test_file invalid --task--invalid "$query"
-test_file invalid-applicable --task--applicable --task--invalid "$query"
-
+test_generated invalid --task--invalid "$query"
+test_generated invalid-applicable --task--applicable --task--invalid "$query"
 exit $RESULT
diff --git a/test/invalid b/test/invalid
deleted file mode 100644
index 5b5bc32..0000000
--- a/test/invalid
+++ /dev/null
@@ -1,559 +0,0 @@
-   "plugins" : [
-      {
-         "name" : "task",
-         "roots" : [
-            {
-               "applicable" : false,
-               "hasPass" : false,
-               "name" : "Root N/A",
-               "status" : "INVALID"
-            },
-            {
-               "applicable" : true,
-               "hasPass" : false,
-               "name" : "Root grouping NA (subtask NA)",
-               "status" : "WAITING",
-               "subTasks" : [
-                  {
-                     "applicable" : false,
-                     "hasPass" : false,
-                     "name" : "Subtask NA",
-                     "status" : "INVALID"
-                  }
-               ]
-            },
-            {
-               "applicable" : true,
-               "hasPass" : false,
-               "name" : "Subtasks External (Missing)",
-               "status" : "WAITING",
-               "subTasks" : [
-                  {
-                     "name" : "UNKNOWN",
-                     "status" : "INVALID"
-                  }
-               ]
-            },
-            {
-               "applicable" : true,
-               "hasPass" : false,
-               "name" : "Subtasks External (User Missing)",
-               "status" : "WAITING",
-               "subTasks" : [
-                  {
-                     "name" : "UNKNOWN",
-                     "status" : "INVALID"
-                  }
-               ]
-            },
-            {
-               "name" : "UNKNOWN",
-               "status" : "INVALID"
-            },
-            {
-               "applicable" : true,
-               "hasPass" : false,
-               "name" : "INVALIDS",
-               "status" : "WAITING",
-               "subTasks" : [
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "No PASS criteria",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : true,
-                     "name" : "WAITING (subtask INVALID)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "applicable" : true,
-                           "hasPass" : false,
-                           "name" : "Subtask INVALID",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "WAITING (subtask duplicate)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "applicable" : true,
-                           "hasPass" : false,
-                           "name" : "Subtask INVALID",
-                           "status" : "INVALID"
-                        },
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : true,
-                     "name" : "WAITING (subtask missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "Grouping WAITING (subtask INVALID)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "applicable" : true,
-                           "hasPass" : false,
-                           "name" : "Subtask INVALID",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "Grouping WAITING (subtask missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "Subtask INVALID",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "Subtask Optional",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : false,
-                     "hasPass" : true,
-                     "name" : "NA Bad PASS query",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : false,
-                     "hasPass" : true,
-                     "name" : "NA Bad FAIL query",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : false,
-                     "hasPass" : true,
-                     "name" : "NA Bad INPROGRESS query",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "Looping",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "name" : "UNKNOWN",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (tasks-factory missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory type missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory type INVALID)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory duplicate)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory changes type missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory changes missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory changes invalid)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (tasks-factory changes loop)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "applicable" : true,
-                           "hasPass" : true,
-                           "name" : "_change1_number",
-                           "status" : "FAIL",
-                           "subTasks" : [
-                              {
-                                 "name" : "UNKNOWN",
-                                 "status" : "INVALID"
-                              }
-                           ]
-                        },
-                        {
-                           "applicable" : true,
-                           "hasPass" : true,
-                           "name" : "_change2_number",
-                           "status" : "FAIL",
-                           "subTasks" : [
-                              {
-                                 "name" : "UNKNOWN",
-                                 "status" : "INVALID"
-                              }
-                           ]
-                        }
-                     ]
-                  }
-               ]
-            },
-            {
-               "applicable" : false,
-               "hasPass" : false,
-               "name" : "NA INVALIDS",
-               "status" : "WAITING",
-               "subTasks" : [
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "No PASS criteria",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : true,
-                     "name" : "WAITING (subtask INVALID)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "applicable" : true,
-                           "hasPass" : false,
-                           "name" : "Subtask INVALID",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "WAITING (subtask duplicate)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "applicable" : true,
-                           "hasPass" : false,
-                           "name" : "Subtask INVALID",
-                           "status" : "INVALID"
-                        },
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : true,
-                     "name" : "WAITING (subtask missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "Grouping WAITING (subtask INVALID)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "applicable" : true,
-                           "hasPass" : false,
-                           "name" : "Subtask INVALID",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "Grouping WAITING (subtask missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "Subtask INVALID",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "Subtask Optional",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : false,
-                     "hasPass" : true,
-                     "name" : "NA Bad PASS query",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : false,
-                     "hasPass" : true,
-                     "name" : "NA Bad FAIL query",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : false,
-                     "hasPass" : true,
-                     "name" : "NA Bad INPROGRESS query",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "Looping",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "name" : "UNKNOWN",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (tasks-factory missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory type missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory type INVALID)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory duplicate)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory changes type missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory changes missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (names-factory changes invalid)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "applicable" : true,
-                     "hasPass" : false,
-                     "name" : "task (tasks-factory changes loop)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "applicable" : true,
-                           "hasPass" : true,
-                           "name" : "_change1_number",
-                           "status" : "FAIL",
-                           "subTasks" : [
-                              {
-                                 "name" : "UNKNOWN",
-                                 "status" : "INVALID"
-                              }
-                           ]
-                        },
-                        {
-                           "applicable" : true,
-                           "hasPass" : true,
-                           "name" : "_change2_number",
-                           "status" : "FAIL",
-                           "subTasks" : [
-                              {
-                                 "name" : "UNKNOWN",
-                                 "status" : "INVALID"
-                              }
-                           ]
-                        }
-                     ]
-                  }
-               ]
-            }
-         ]
-      }
-   ],
diff --git a/test/invalid-applicable b/test/invalid-applicable
deleted file mode 100644
index 369ac47..0000000
--- a/test/invalid-applicable
+++ /dev/null
@@ -1,244 +0,0 @@
-   "plugins" : [
-      {
-         "name" : "task",
-         "roots" : [
-            {
-               "hasPass" : false,
-               "name" : "Subtasks External (Missing)",
-               "status" : "WAITING",
-               "subTasks" : [
-                  {
-                     "name" : "UNKNOWN",
-                     "status" : "INVALID"
-                  }
-               ]
-            },
-            {
-               "hasPass" : false,
-               "name" : "Subtasks External (User Missing)",
-               "status" : "WAITING",
-               "subTasks" : [
-                  {
-                     "name" : "UNKNOWN",
-                     "status" : "INVALID"
-                  }
-               ]
-            },
-            {
-               "name" : "UNKNOWN",
-               "status" : "INVALID"
-            },
-            {
-               "hasPass" : false,
-               "name" : "INVALIDS",
-               "status" : "WAITING",
-               "subTasks" : [
-                  {
-                     "hasPass" : false,
-                     "name" : "No PASS criteria",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "hasPass" : true,
-                     "name" : "WAITING (subtask INVALID)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "hasPass" : false,
-                           "name" : "Subtask INVALID",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "WAITING (subtask duplicate)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "hasPass" : false,
-                           "name" : "Subtask INVALID",
-                           "status" : "INVALID"
-                        },
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : true,
-                     "name" : "WAITING (subtask missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "Grouping WAITING (subtask INVALID)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "hasPass" : false,
-                           "name" : "Subtask INVALID",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "Grouping WAITING (subtask missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "Subtask INVALID",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "Subtask Optional",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "Looping",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "name" : "UNKNOWN",
-                     "status" : "INVALID"
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "task (tasks-factory missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "task (names-factory type missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "task (names-factory type INVALID)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "task (names-factory duplicate)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "task (names-factory changes type missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "task (names-factory changes missing)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "task (names-factory changes invalid)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "name" : "UNKNOWN",
-                           "status" : "INVALID"
-                        }
-                     ]
-                  },
-                  {
-                     "hasPass" : false,
-                     "name" : "task (tasks-factory changes loop)",
-                     "status" : "WAITING",
-                     "subTasks" : [
-                        {
-                           "hasPass" : true,
-                           "name" : "_change1_number",
-                           "status" : "FAIL",
-                           "subTasks" : [
-                              {
-                                 "name" : "UNKNOWN",
-                                 "status" : "INVALID"
-                              }
-                           ]
-                        },
-                        {
-                           "hasPass" : true,
-                           "name" : "_change2_number",
-                           "status" : "FAIL",
-                           "subTasks" : [
-                              {
-                                 "name" : "UNKNOWN",
-                                 "status" : "INVALID"
-                              }
-                           ]
-                        }
-                     ]
-                  }
-               ]
-            }
-         ]
-      }
-   ],
diff --git a/test/strip_non_invalid.py b/test/strip_non_invalid.py
new file mode 100755
index 0000000..dafae15
--- /dev/null
+++ b/test/strip_non_invalid.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import json
+
+STATUS='status'
+SUBTASKS='subTasks'
+
+def keep_invalid(tasks):
+    i=0
+    while i < len(tasks):
+        nexti = i + 1
+
+        task=tasks[i]
+        if SUBTASKS in task.keys():
+            subtasks=task[SUBTASKS]
+            keep_invalid(subtasks)
+            if len(subtasks) == 0:
+                del task[SUBTASKS]
+
+        status=''
+        if STATUS in task.keys():
+            status = task[STATUS]
+        if status != 'INVALID' and not SUBTASKS in task.keys():
+            del tasks[i]
+            nexti = i
+
+        i = nexti
+
+plugins=json.loads(sys.stdin.read())
+roots=plugins['plugins'][0]['roots']
+keep_invalid(roots)
+print json.dumps(plugins, indent=3, separators=(',', ' : '), sort_keys=True)