diff --git a/Dockerfile b/Dockerfile index 3cf3bca..63f1e31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ RUN pip install -r /requirements.txt RUN apt-get update \ && apt-get install jq -y \ && apt-get autoremove -y \ + && apt-get install coreutils -y \ && rm -rf /var/lib/apt/lists/* COPY . /opt/test-runner diff --git a/bin/run.py b/bin/run.py index 4f4f77d..8ec7fb7 100644 --- a/bin/run.py +++ b/bin/run.py @@ -43,18 +43,11 @@ def main(): help="max amount of points the test suite is worth", ) - parser.add_argument( - "timeout", - metavar="time", - type=int, - help="max amount of time the test can run before before terminating", - ) - parser.add_argument("pytest_args", nargs=REMAINDER) args = parser.parse_args() - runner.run(args.input, args.output, args.max_score, args.timeout*60, args.pytest_args) + runner.run(args.input, args.output, args.max_score, args.pytest_args) if __name__ == "__main__": main() diff --git a/bin/run.sh b/bin/run.sh index c0944d8..5a330fe 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -25,14 +25,15 @@ while [ $# -gt 0 ]; do shift done -echo "TIMEOUT is $TIMEOUT" +TIMEOUT=$((TIMEOUT * 60)) +echo "TIMEOUT is $TIMEOUT seconds" echo "MAX_SCORE is $MAX_SCORE" -if [ -n "$SETUP_COMMAND" ]; then - echo "Running setup command: $SETUP_COMMAND" - eval "$SETUP_COMMAND" +timeout "$TIMEOUT" python3 bin/run.py ./ ./autograding_output/ "$MAX_SCORE" +exit_status=$? +if [ $exit_status -eq 124 ]; then + echo "The command took longer than $TIMEOUT seconds to execute. Please increase the timeout to avoid this error." + echo '{"status": "error", "message": "The command timed out"}' > autograding_output/results.json fi -python3 /opt/test-runner/bin/run.py ./ ./autograding_output/ "$MAX_SCORE" "$TIMEOUT" - echo "result=$(jq -c . autograding_output/results.json | jq -sRr @base64)" >> "$GITHUB_OUTPUT" diff --git a/pytest.ini b/pytest.ini index 9ae8221..4fd8681 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] addopts = - --color=no + --color=yes norecursedirs = .git .github example* traceback-styles* cache_dir = @@ -9,3 +9,11 @@ markers = task: A concept exercise task. console_output_style = classic +log_cli = + true +log_level = + NOTSET +log_format = + %(asctime)s %(levelname)s %(message)s +log_date_format = + %Y-%m-%d %H:%M:%S diff --git a/runner/__init__.py b/runner/__init__.py index a335aad..4c9a205 100644 --- a/runner/__init__.py +++ b/runner/__init__.py @@ -198,7 +198,7 @@ def _sanitize_args(args: List[str]) -> List[str]: return clean -def run(indir: Directory, outdir: Directory, max_score: int, timeout_duration: int, args: List[str]) -> None: +def run(indir: Directory, outdir: Directory, max_score: int, args: List[str]) -> None: """ Run the tests for the given exercise and produce a results.json. """ @@ -214,14 +214,7 @@ def run(indir: Directory, outdir: Directory, max_score: int, timeout_duration: i # run the tests and report reporter = ResultsReporter() reporter.results.max_score = max_score - try: - @timeout_decorator.timeout(timeout_duration) - def run_tests(): - pytest.main(_sanitize_args(args or []) + [str(tf) for tf in test_files], plugins=[reporter]) - run_tests() - except TimeoutError: - reporter.results.error("Tests timed out after {} seconds".format(timeout_duration)) - + pytest.main(['-s'] + _sanitize_args(args or []) + [str(tf) for tf in test_files], plugins=[reporter]) # dump the report out_file.write_text(reporter.results.as_json()) # remove cache directories