Add input argument piping (#15)

Author: niosus

README.md

@@ -9,7 +9,7 @@ The idea behind the script is the following:
 
 It is expected that the submitted homework will follow the folder structure specified in the `homework.yml` file. 
 
-## Core funcionality ##
+## Core functionality ##
 
 ### Run different tests ###
 For now we support running tests for code written in different languages:

homework_checker/core/schema_manager.py

@@ -33,10 +33,12 @@ def __init__(self: SchemaManager, file_name: Path):
 
         test_schema = {
             Tags.NAME_TAG: str,
-            Optional(Tags.INPUT_TAG): str,
+            Optional(Tags.INPUT_ARGS_TAG): str,
             Optional(Tags.INJECT_FOLDER_TAG): [injection_schema],
             Optional(Tags.RUN_GTESTS_TAG, default=False): bool,
             Optional(Tags.EXPECTED_OUTPUT_TAG): Or(str, float, int),
+            Optional(Tags.INPUT_PIPE_TAG, default=""): str,
+            Optional(Tags.OUTPUT_PIPE_TAG, default=""): str,
             Optional(Tags.TIMEOUT_TAG, default=60): float,
         }
 
@@ -47,9 +49,10 @@ def __init__(self: SchemaManager, file_name: Path):
             Optional(Tags.OUTPUT_TYPE_TAG, default=OutputTags.STRING): Or(
                 OutputTags.STRING, OutputTags.NUMBER
             ),
-            Optional(Tags.COMPILER_FLAGS_TAG, default="-Wall"): str,
+            Optional(
+                Tags.COMPILER_FLAGS_TAG, default="-std=c++17 -Wall -Wpedantic -Wextra"
+            ): str,
             Optional(Tags.BINARY_NAME_TAG, default="main"): str,
-            Optional(Tags.PIPE_TAG, default=""): str,
             Optional(Tags.BUILD_TYPE_TAG, default=BuildTags.CMAKE): Or(
                 BuildTags.CMAKE, BuildTags.SIMPLE
             ),

homework_checker/core/schema_tags.py

@@ -16,11 +16,12 @@ class Tags:
     INJECT_DESTINATION_TAG = "destination"
     INJECT_FOLDER_TAG = "inject_folders"
     INJECT_SOURCE_TAG = "source"
-    INPUT_TAG = "input_args"
+    INPUT_ARGS_TAG = "input_args"
+    INPUT_PIPE_TAG = "input_pipe_args"
     LANGUAGE_TAG = "language"
     NAME_TAG = "name"
+    OUTPUT_PIPE_TAG = "output_pipe_args"
     OUTPUT_TYPE_TAG = "output_type"
-    PIPE_TAG = "pipe_through"
     RUN_GTESTS_TAG = "run_google_tests"
     TASKS_TAG = "tasks"
     TESTS_TAG = "tests"

homework_checker/core/tasks.py

@@ -74,12 +74,11 @@ def __init__(
         self._output_type = task_node[Tags.OUTPUT_TYPE_TAG]
         self._student_task_folder = student_task_folder
         self._binary_name = task_node[Tags.BINARY_NAME_TAG]
-        self._pipe_through = task_node[Tags.PIPE_TAG]
         self._build_timeout = task_node[Tags.BUILD_TIMEOUT_TAG]
+
+        self._test_nodes = []
         if Tags.TESTS_TAG in task_node:
             self._test_nodes = task_node[Tags.TESTS_TAG]
-        else:
-            self._test_nodes = []  # Sometimes we don't have tests.
         self.__test_counter = 0
 
     def __with_number_prefix(self: Task, test_name: str) -> str:
@@ -187,7 +186,7 @@ class CppTask(Task):
 
     CMAKE_BUILD_CMD = "cmake .. && make -j2"
     REMAKE_AND_TEST = "make clean && rm -r * && cmake .. && make -j2 && ctest -VV"
-    BUILD_CMD_SIMPLE = "clang++ -std=c++14 -o {binary} {compiler_flags} {binary}.cpp"
+    BUILD_CMD_SIMPLE = "clang++ {compiler_flags} -o {binary} {binary}.cpp"
 
     def __init__(self: CppTask, task_node: dict, root_folder: Path, job_file: Path):
         """Initialize the C++ Task."""
@@ -247,17 +246,28 @@ def _run_test(self: CppTask, test_node: dict, executable_folder: Path):
                 cwd=executable_folder,
                 timeout=test_node[Tags.TIMEOUT_TAG],
             )
+
+        output_pipe_args = None
+        if Tags.OUTPUT_PIPE_TAG in test_node:
+            output_pipe_args = test_node[Tags.OUTPUT_PIPE_TAG]
+        input_pipe_args = None
+        if Tags.INPUT_PIPE_TAG in test_node:
+            input_pipe_args = test_node[Tags.INPUT_PIPE_TAG]
+
         input_str = ""
-        if Tags.INPUT_TAG in test_node:
-            input_str = test_node[Tags.INPUT_TAG]
+        if Tags.INPUT_ARGS_TAG in test_node:
+            input_str = test_node[Tags.INPUT_ARGS_TAG]
         run_cmd = "./{binary_name} {args}".format(
             binary_name=self._binary_name, args=input_str
         )
-        if self._pipe_through:
-            run_cmd += " " + self._pipe_through
+        if input_pipe_args:
+            run_cmd = input_pipe_args + " | " + run_cmd
+        if output_pipe_args:
+            run_cmd += " | " + output_pipe_args
         run_result = tools.run_command(
             run_cmd, cwd=executable_folder, timeout=test_node[Tags.TIMEOUT_TAG]
         )
+
         if not run_result.succeeded():
             return run_result
         # TODO(igor): do I need explicit error here?
@@ -294,12 +304,21 @@ def _code_style_errors(self: BashTask):
     def _run_test(
         self: BashTask, test_node: dict, executable_folder: Path
     ) -> tools.CmdResult:
+        output_pipe_args = None
+        if Tags.OUTPUT_PIPE_TAG in test_node:
+            output_pipe_args = test_node[Tags.OUTPUT_PIPE_TAG]
+        input_pipe_args = None
+        if Tags.INPUT_PIPE_TAG in test_node:
+            input_pipe_args = test_node[Tags.INPUT_PIPE_TAG]
+
         input_str = ""
-        if Tags.INPUT_TAG in test_node:
-            input_str = test_node[Tags.INPUT_TAG]
+        if Tags.INPUT_ARGS_TAG in test_node:
+            input_str = test_node[Tags.INPUT_ARGS_TAG]
         run_cmd = BashTask.RUN_CMD.format(binary_name=self._binary_name, args=input_str)
-        if self._pipe_through:
-            run_cmd += " " + self._pipe_through
+        if input_pipe_args:
+            run_cmd = input_pipe_args + " | " + run_cmd
+        if output_pipe_args:
+            run_cmd += " | " + output_pipe_args
         run_result = tools.run_command(
             run_cmd, cwd=executable_folder, timeout=test_node[Tags.TIMEOUT_TAG]
         )

homework_checker/core/tests/data/homework/example_job.yml

@@ -9,11 +9,12 @@ homeworks:
         folder: task_1        # Name of the folder containing the Task.
         output_type: string   # We expect a string as an output.
         binary_name: main
-        tests:                # An Task can have multiple tests.
+        compiler_flags: "-std=c++17"
+        tests:                # A Task can have multiple tests.
           - name: String output test
             expected_output: > # this wraps into a long string, no line breaks.
               This is a long test output that we expect to be produced by the
-              code. We will compare the ouput to this EXACTLY.
+              code. We will compare the output to this EXACTLY.
           - name: Input output test
             input_args: Some string
             expected_output: >
@@ -24,11 +25,13 @@ homeworks:
       - name: Build failure task  # This one should not build, no need for tests
         language: cpp
         folder: task_2
+        compiler_flags: "-std=c++17"
         output_type: string
       - name: CMake build arithmetics task
         language: cpp
         folder: task_3
         output_type: number
+        compiler_flags: "-std=c++17"
         binary_name: sum_numbers
         tests:
           - name: Test integer arithmetics
@@ -50,6 +53,18 @@ homeworks:
               test_me.sh
           - name: Test wrong output
             expected_output: Different output that doesn't match generated one
+      - name: Test input piping
+        language: cpp
+        folder: task_5
+        compiler_flags: "-std=c++17"
+        output_type: string
+        tests:
+          - name: Test input piping
+            input_pipe_args: echo hello world
+            expected_output: |
+              Input string:
+              Input another string:
+              hello_world
 
   - name: "Homework where things go wrong"
     folder: "homework_2"
@@ -59,6 +74,7 @@ homeworks:
         language: cpp
         folder: task_1
         build_type: simple
+        compiler_flags: "-std=c++17"
         output_type: number
         tests:
           # Should fail as the binary returns a string
@@ -67,6 +83,7 @@ homeworks:
       - name: While loop task
         language: cpp
         folder: task_2
+        compiler_flags: "-std=c++17"
         build_type: simple
         tests:
           - name: Test timeout  # Should fail because of the timeout.
@@ -76,6 +93,7 @@ homeworks:
         language: cpp
         folder: task_3
         build_type: simple
+        compiler_flags: "-std=c++17"
         output_type: number
         tests:
           - name: Test 1
@@ -86,6 +104,7 @@ homeworks:
     tasks:
       - name: Google Tests
         language: cpp
+        compiler_flags: "-std=c++17"
         folder: cpptests
         tests:
           - name: Just build
@@ -104,14 +123,14 @@ homeworks:
         language: bash
         folder: bashtests
         binary_name: ls_me
-        pipe_through: "| head -n 2"
         tests:
           - name: ls
             inject_folders: # Just multiple folders.
               - source: solutions/fail
                 destination: fail
               - source: solutions/pass
                 destination: pass
+            output_pipe_args: "head -n 2"
             expected_output: |  # This maintains whitespaces.
               fail
               ls_me.sh
@@ -124,6 +143,7 @@ homeworks:
         folder: task_1
         build_type: simple
         output_type: number
+        compiler_flags: "-std=c++17"
         tests:
           - name: Irrelevant
             expected_output: 4

homework_checker/core/tests/data/homework/homework_1/task_1/main.cpp

@@ -4,7 +4,7 @@ int main(int argc, char const *argv[]) {
   if (argc == 1) {
     fprintf(stdout,
             "This is a long test output that we expect to be produced by "
-            "the code. We will compare the ouput to this EXACTLY.\n");
+            "the code. We will compare the output to this EXACTLY.\n");
   } else {
     fprintf(stdout, "%s %s output\n", argv[1], argv[2]);
   }

homework_checker/core/tests/data/homework/homework_1/task_5/CMakeLists.txt

@@ -0,0 +1,4 @@
+cmake_minimum_required(VERSION 3.0.0)
+project(test_cpp_project)
+
+add_executable(main main.cpp)

homework_checker/core/tests/data/homework/homework_1/task_5/main.cpp

@@ -0,0 +1,10 @@
+#include <iostream>
+
+int main() {
+  std::string smth, smth_else;
+  std::cout << "Input string:" << std::endl;
+  std::cin >> smth;
+  std::cout << "Input another string:" << std::endl;
+  std::cin >> smth_else;
+  std::cout << smth << "_" << smth_else;
+}

homework_checker/core/tests/test_task.py

@@ -129,6 +129,30 @@ def test_check_cmake_arithmetics_task(self: "TestTask"):
         self.assertTrue(results["Test integer arithmetics"].succeeded())
         self.assertFalse(results["Test float arithmetics"].succeeded())
 
+    def test_check_input_args_piping(self: "TestTask"):
+        """Check that we can pipe arguments into a binary."""
+
+        homework_name, task = self.__get_homework_name_and_task(
+            homework_index=0, task_index=4
+        )
+        self.assertEqual(homework_name, "Sample homework")
+        self.assertEqual(task.name, "Test input piping")
+        results = task.check()
+        results = TestTask.__sanitize_results(results)
+        expected_number_of_build_outputs = 1
+        expected_number_of_test_outputs = 1
+        expected_number_of_code_style_outputs = 0
+        self.assertEqual(
+            len(results),
+            expected_number_of_build_outputs
+            + expected_number_of_test_outputs
+            + expected_number_of_code_style_outputs,
+            "Wrong results: {}".format(results),
+        )
+        self.assertTrue(results[BUILD_SUCCESS_TAG].succeeded())
+        print(results["Test input piping"].stderr)
+        self.assertTrue(results["Test input piping"].succeeded())
+
     def test_check_bash_task(self: "TestTask"):
         """Check a simple cmake build on arithmetics example."""
 

homework_checker/core/tools.py

@@ -4,6 +4,7 @@
 import re
 from os import environ
 from subprocess import Popen, TimeoutExpired, CalledProcessError, CompletedProcess
+from sys import stdin
 from typing import Union, List, Optional, Mapping, Any, Tuple
 from pathlib import Path
 import tempfile
@@ -219,6 +220,7 @@ def run_command(
             command,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
+            stdin=subprocess.PIPE,
             shell=shell,
             cwd=str(cwd),
             env=env,
@@ -248,7 +250,7 @@ def __run_subprocess(
     str_input: str = None,
     timeout: float = None,
     check: bool = False,
-    **kwargs
+    **kwargs,
 ) -> subprocess.CompletedProcess:
     """Run a command as a subprocess.
 

schema/schema.yml

@@ -10,16 +10,17 @@ homeworks:
         name: String value
         ~[optional]~ binary_name: String value
         ~[optional]~ build_timeout: Float value
-        ~[optional]~ build_type: Any of ['cmake', 'simple']
         ~[optional]~ compiler_flags: String value
+        ~[optional]~ build_type: Any of ['cmake', 'simple']
         ~[optional]~ output_type: Any of ['string', 'number']
-        ~[optional]~ pipe_through: String value
         ~[optional]~ tests:
           - name: String value
             ~[optional]~ expected_output: Any of ['String value', 'Float value', 'Int value']
             ~[optional]~ inject_folders:
               - destination: String value
                 source: String value
             ~[optional]~ input_args: String value
+            ~[optional]~ input_pipe_args: String value
+            ~[optional]~ output_pipe_args: String value
             ~[optional]~ run_google_tests: Boolean value
             ~[optional]~ timeout: Float value

setup.py

@@ -6,7 +6,7 @@
 from setuptools import find_packages
 from setuptools.command.install import install
 
-VERSION_STRING = "1.0.3"
+VERSION_STRING = "1.0.4"
 
 PACKAGE_NAME = "homework_checker"