Validation too!

Author: sgugger

examples/question_answering.ipynb

@@ -1510,6 +1510,11 @@
    "outputs": [],
    "source": [
     "def prepare_validation_features(examples):\n",
+    "    # Some of the questions have lots of whitespace on the left, which is not useful and will make the\n",
+    "    # truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n",
+    "    # left whitespace\n",
+    "    examples[\"question\"] = [q.lstrip() for q in examples[\"question\"]]\n",
+    "\n",
     "    # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n",
     "    # in one example possible giving several features when a context is long, each of those features having a\n",
     "    # context that overlaps a bit the context of the previous feature.\n",