Create a landing page with batchrun (#12)

* Create a landing page with batchrun
* Modify batchrun behavior to handle this use case

diffir/batchrun.py

@@ -4,6 +4,10 @@
 import os
 from functools import partial
 from pathlib import Path
+from markupsafe import Markup
+import json
+from collections import defaultdict
+from jinja2 import Environment, PackageLoader, select_autoescape
 
 import ir_datasets
 
@@ -15,7 +19,10 @@
 
 def process_runs(fns, config, output):
     task_config, html = diff(fns, config, cli=False, web=True, print_html=False)
-    outdir = output / task_config["dataset"]
+
+    dataset_name = task_config["dataset"].replace("/", ".")
+    measure_name = task_config["measure"] if task_config["measure"] != "qrel" else "qrel." + task_config["metric"]
+    outdir = output / f"{dataset_name}---measure___{measure_name}"
     outdir.mkdir(exist_ok=True, parents=True)
 
     outfn = outdir / ("___".join(os.path.basename(x) for x in fns) + ".html")
@@ -25,6 +32,39 @@ def process_runs(fns, config, output):
     return outdir
 
 
+def regenerate_landing_page(outdir):
+    datasets = []
+    name_dict = {}
+    runfiles = defaultdict(list)
+    for dirname in os.listdir(outdir):
+        if not os.path.isdir(os.path.join(outdir, dirname)):
+            continue
+
+        dataset_name = dirname
+        configs = dataset_name.split("---")
+        display_name = configs[0] + (
+            "" if len(configs) == 1 else " (" + ",".join([c.replace("___", ":") for c in configs[1:]]) + ")"
+        )
+        name_dict[dataset_name] = display_name
+        datasets.append(dataset_name)
+
+        with open(os.path.join(outdir, dataset_name, "runs.txt")) as f:
+            for filename in f:
+                print(filename.strip())
+                runfiles[dataset_name].append(filename.strip())
+
+    env = Environment(loader=PackageLoader("diffir", "templates"), autoescape=select_autoescape(["html", "xml"]))
+
+    landing_template = env.get_template("landing.html")
+    with open(os.path.join(outdir, "index.html"), "wt") as outf:
+        print(
+            landing_template.render(
+                datasets=datasets, name_dict=name_dict, runfiles=runfiles, rawdata=Markup(json.dumps(runfiles))
+            ),
+            file=outf,
+        )
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("directory")
@@ -69,6 +109,8 @@ def main():
         for run in sorted(single_runs):
             print(run.split("/")[-1], file=outf)
 
+    regenerate_landing_page(output)
+
 
 if __name__ == "__main__":
     main()

diffir/run.py

@@ -113,11 +113,11 @@ def create_query_objects(self, run_1, run_2, qids, qid2diff, metric_name, datase
         assert dataset.has_qrels(), "Cannot determine whether the doc is relevant - need qrels"
         qrels = dataset.qrels_dict()
         run1_metrics = defaultdict(lambda: defaultdict(lambda: None))
-        for metrics in iter_calc([P @ 1, P @ 3, P @ 5, P @ 10, nDCG @ 1, nDCG @ 3, nDCG @ 5, nDCG @ 10], qrels, run_1):
+        for metrics in iter_calc([P@1, P@3, P@5, P@10, nDCG@1, nDCG@3, nDCG@5, nDCG@10], qrels, run_1):
             run1_metrics[metrics.query_id][str(metrics.measure)] = metrics.value
         if run_2:
             run2_metrics = defaultdict(lambda: defaultdict(lambda: None))
-            for metrics in iter_calc([P @ 1, P @ 3, P @ 5, P @ 10, nDCG @ 1, nDCG @ 3, nDCG @ 5, nDCG @ 10], qrels, run_2):
+            for metrics in iter_calc([P@1, P@3, P@5, P@10, nDCG@1, nDCG@3, nDCG@5, nDCG@10], qrels, run_2):
                 run2_metrics[metrics.query_id][str(metrics.measure)] = metrics.value
         docstore = dataset.docs_store()
         qids_set = set(qids)  # Sets do O(1) lookups

diffir/templates/landing.html

@@ -144,7 +144,7 @@ <h6 class="text-white">Contact</h6>
 
       <p id="tabs" class="panel-tabs">
         {% for dataset in datasets %}
-          <a id="tab-for-{{dataset}}" class="{% if loop.index < 2 %}is-active {% endif %} dataset-tab" data-dataset={{dataset}}>{{dataset}}</a>
+          <a id="tab-for-{{dataset}}" class="{% if loop.index < 2 %}is-active {% endif %} dataset-tab" data-dataset={{dataset}}>{{name_dict[dataset]}}</a>
         {% endfor %}
       </p>
 
@@ -229,15 +229,14 @@ <h6 class="text-white">Contact</h6>
     $("a.dataset-tab").click(function(e){
       let prev_dataset_name = $("a.dataset-tab.is-active").attr('data-dataset')
       let dataset_name = $(this).attr("data-dataset")
-
       // start of: Clear all effects applied `to runfiles in the previous tab
-      let $prev_selection_left = $("#runfiles-for-" + prev_dataset_name + "-left .runfile")
+      let $prev_selection_left = $("#" + $.escapeSelector("runfiles-for-" + prev_dataset_name + "-left") + " .runfile")
       $prev_selection_left.removeClass("is-active")
       $prev_selection_left.removeClass("disabled")
       $prev_selection_left.removeClass("other-side-disabled")
       $prev_selection_left.removeClass("search-excluded")
 
-      let $prev_selection_right = $("#runfiles-for-" + prev_dataset_name + "-right .runfile")
+      let $prev_selection_right = $("#" + $.escapeSelector("runfiles-for-" + prev_dataset_name + "-right")+" .runfile")
       $prev_selection_right.removeClass("is-active")
       $prev_selection_right.removeClass("disabled")
       $prev_selection_right.removeClass("other-side-disabled")
@@ -253,11 +252,11 @@ <h6 class="text-white">Contact</h6>
 
 
       $(".panel-runfiles").removeClass("is-active")
-      $("#runfiles-for-" + dataset_name + "-left").addClass("is-active")
-      $("#runfiles-for-" + dataset_name + "-right").addClass("is-active")
+      $("#" + $.escapeSelector("runfiles-for-" + dataset_name + "-left")).addClass("is-active")
+      $("#" + $.escapeSelector("runfiles-for-" + dataset_name + "-right")).addClass("is-active")
 
       $(".dataset-tab").removeClass("is-active")
-      $("#tab-for-" + dataset_name).addClass("is-active")
+      $("#"+$.escapeSelector("tab-for-" + dataset_name)).addClass("is-active")
     });
 
     // When a runfile is clicked, select it, and disable clicking on other runfiles on the same side
@@ -338,7 +337,7 @@ <h6 class="text-white">Contact</h6>
     function apply_search_results(dataset, side, results) {
       let search_results = new Set(results);
 
-      $("#runfiles-for-" + dataset +"-" + side + " .runfile").each(function(){
+      $("#"+ $.escapeSelector("runfiles-for-" + dataset +"-" + side) + " .runfile").each(function(){
         const runfile_name = $(this).attr("data-runfile");
 
         if(!search_results.has(runfile_name)){