Pass source code location to oracles

tested/dsl/schema-strict.json

@@ -530,6 +530,13 @@
               "items" : {
                 "$ref" : "#/definitions/yamlValueOrPythonExpression"
               }
+            },
+            "languages": {
+              "type" : "array",
+              "description" : "Which programming languages are supported by this oracle.",
+              "items" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              }
             }
           }
         }
@@ -585,6 +592,13 @@
               "items" : {
                 "$ref" : "#/definitions/yamlValueOrPythonExpression"
               }
+            },
+            "languages": {
+              "type" : "array",
+              "description" : "Which programming languages are supported by this oracle.",
+              "items" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              }
             }
           }
         }

tested/dsl/schema.json

@@ -530,6 +530,13 @@
               "items" : {
                 "$ref" : "#/definitions/yamlValueOrPythonExpression"
               }
+            },
+            "languages": {
+              "type" : "array",
+              "description" : "Which programming languages are supported by this oracle.",
+              "items" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              }
             }
           }
         }
@@ -585,6 +592,13 @@
               "items" : {
                 "$ref" : "#/definitions/yamlValueOrPythonExpression"
               }
+            },
+            "languages": {
+              "type" : "array",
+              "description" : "Which programming languages are supported by this oracle.",
+              "items" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              }
             }
           }
         }

tested/dsl/translate_parser.py

@@ -91,6 +91,12 @@ class ReturnOracle(dict):
 )
 
 
+def _convert_language_dictionary(
+    original: dict[str, str]
+) -> dict[SupportedLanguage, str]:
+    return {SupportedLanguage(k): v for k, v in original.items()}
+
+
 def _ensure_trailing_newline(text: str) -> str:
     if text and not text.endswith("\n"):
         return text + "\n"
@@ -374,11 +380,13 @@ def _convert_custom_check_oracle(stream: dict) -> CustomCheckOracle:
         cv = _convert_yaml_value(v)
         assert isinstance(cv, Value)
         converted_args.append(cv)
+    languages = stream.get("languages")
     return CustomCheckOracle(
         function=EvaluationFunction(
             file=stream["file"], name=stream.get("name", "evaluate")
         ),
         arguments=converted_args,
+        languages=set(languages) if languages else None,
     )
 
 
@@ -503,7 +511,9 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase:
             message = exception.get("message")
             assert isinstance(message, str)
             assert isinstance(exception["types"], dict)
-            types = cast(dict[str, str], exception["types"])
+            types = _convert_language_dictionary(
+                cast(dict[str, str], exception["types"])
+            )
         output.exception = ExceptionOutputChannel(
             exception=ExpectedException(message=message, types=types)
         )

tested/oracles/common.py

@@ -37,16 +37,22 @@ def evaluate_text(configs, channel, actual):
 from tested.languages.utils import convert_stacktrace_to_clickable_feedback
 from tested.parsing import fallback_field, get_converter
 from tested.serialisation import Value
-from tested.testsuite import ExceptionOutputChannel, NormalOutputChannel, OutputChannel
+from tested.testsuite import (
+    ExceptionOutputChannel,
+    NormalOutputChannel,
+    OutputChannel,
+    SupportedLanguage,
+)
 
 
 @define
 class OracleContext:
     expected: Value
     actual: Value
-    execution_directory: str
-    evaluation_directory: str
-    programming_language: str
+    execution_directory: Path
+    evaluation_directory: Path
+    submission_path: Path | None
+    programming_language: SupportedLanguage
     natural_language: str
 
 

tested/oracles/programmed.py

@@ -32,12 +32,18 @@
 
 @define
 class ConvertedOracleContext:
+    """
+    This is the oracle context that is passed to the actual function.
+    It should thus remain backwards compatible.
+    """
+
     expected: Any
     actual: Any
     execution_directory: str
     evaluation_directory: str
     programming_language: str
     natural_language: str
+    submission_path: str | None
 
     @staticmethod
     def from_context(
@@ -46,10 +52,15 @@ def from_context(
         return ConvertedOracleContext(
             expected=eval(generate_statement(bundle, context.expected)),
             actual=eval(generate_statement(bundle, context.actual)),
-            execution_directory=context.execution_directory,
-            evaluation_directory=context.evaluation_directory,
+            execution_directory=str(context.execution_directory.absolute()),
+            evaluation_directory=str(context.evaluation_directory.absolute()),
             programming_language=context.programming_language,
             natural_language=context.natural_language,
+            submission_path=(
+                str(context.submission_path.absolute())
+                if context.submission_path
+                else None
+            ),
         )
 
 
@@ -237,10 +248,13 @@ def evaluate(
     context = OracleContext(
         expected=expected,
         actual=actual,
-        execution_directory=str(config.context_dir.absolute()),
-        evaluation_directory=str(config.bundle.config.resources.absolute()),
-        programming_language=str(config.bundle.config.programming_language),
+        execution_directory=config.context_dir,
+        evaluation_directory=config.bundle.config.resources,
+        programming_language=config.bundle.config.programming_language,
         natural_language=config.bundle.config.natural_language,
+        submission_path=(
+            config.bundle.config.source if channel.oracle.languages else None
+        ),
     )
     result = _evaluate_programmed(config.bundle, channel.oracle, context)
 

tested/testsuite.py

@@ -11,7 +11,7 @@
 from enum import StrEnum, auto, unique
 from os import path
 from pathlib import Path
-from typing import Any, Literal, Union
+from typing import Any, Literal, Protocol, TypeGuard, Union
 
 from attrs import define, field
 
@@ -155,17 +155,28 @@ class CustomCheckOracle:
     independent. The oracle is run through the judge infrastructure to translate
     values between different programming languages.
 
-    Although most programming languages are supported, we recommend using Python,
-    as TESTed can then apply specific optimisations, meaning it will be faster than
-    other languages.
+    Some examples of what is possible with this oracle are sequence alignment
+    checking, or evaluating non-deterministic return values. Another example is
+    rendering the value into another representation, such as providing a message
+    with the rendered SVG image.
 
-    Some examples of intended use of this oracle are sequence alignment checking,
-    or evaluating non-deterministic return values.
+    While nominally language agnostic, the oracle supports a "languages" property,
+    which allows limiting for which programming languages the oracle is supported.
+    If provided, the oracle will be provided with the location of the source code,
+    enabling language-specific static analysis without having to use language-
+    specific oracles.
     """
 
     function: EvaluationFunction
     arguments: list[Value] = field(factory=list)
     type: Literal["programmed", "custom_check"] = "custom_check"
+    languages: set[SupportedLanguage] | None = field(default=None)
+
+    @languages.validator  # type: ignore
+    def validate_languages(self, _, value):
+        """There should be at least one evaluator."""
+        if value and not len(value):
+            raise ValueError("At least one language is required.")
 
 
 @fallback_field(get_converter(), {"evaluators": "functions"})
@@ -310,7 +321,7 @@ class ExpectedException(WithFeatures):
     # - Specify nothing, in which case the type is not checked.
     # - Specify a dictionary mapping programming names to exception names.
     #   These exception names should already be in the right convention.
-    types: dict[str, str] | None = None
+    types: dict[SupportedLanguage, str] | None = None
 
     def __attrs_post_init__(self):
         if self.message is None and self.types is None:
@@ -319,12 +330,12 @@ def __attrs_post_init__(self):
     def get_used_features(self) -> FeatureSet:
         return FeatureSet({Construct.EXCEPTIONS}, types=set(), nested_types=set())
 
-    def get_type(self, language: str) -> str | None:
+    def get_type(self, language: SupportedLanguage) -> str | None:
         if not self.types:
             return None
         return self.types.get(language)
 
-    def readable(self, language: str) -> str:
+    def readable(self, language: SupportedLanguage) -> str:
         type_ = self.get_type(language)
         if self.message and type_:
             return f"{type_}: {self.message}"
@@ -404,6 +415,29 @@ def get_used_features(self) -> FeatureSet:
 ExitOutput = ExitCodeOutputChannel | IgnoredChannel | EmptyChannel
 
 
+def _get_text_channel_languages(
+    output: TextOutput | ValueOutput | ExceptionOutput | FileOutput,
+) -> set[SupportedLanguage] | None:
+
+    class _OracleChannel(Protocol):
+        oracle: Any
+
+    def _is_oracle_channel(value: Any) -> TypeGuard[_OracleChannel]:
+        return hasattr(value, "oracle")
+
+    if not _is_oracle_channel(output):
+        return None
+
+    # For language-specific oracles, it is easy.
+    if isinstance(output.oracle, LanguageSpecificOracle):
+        return set(output.oracle.functions.keys())
+
+    if isinstance(output.oracle, CustomCheckOracle):
+        return output.oracle.languages
+
+    return None
+
+
 @define
 class Output(WithFeatures):
     """The output channels for a testcase."""
@@ -432,29 +466,28 @@ def get_specific_languages(self) -> set[SupportedLanguage] | None:
 
         :return: None if no language-specific stuff is used, a set of supported languages otherwise.
         """
-        languages = None
+        # Check generic oracles.
+        individual_languages = [
+            _get_text_channel_languages(self.stdout),
+            _get_text_channel_languages(self.stderr),
+            _get_text_channel_languages(self.file),
+            _get_text_channel_languages(self.exception),
+            _get_text_channel_languages(self.result),
+        ]
+
+        # Handle special cases
         if isinstance(self.exception, ExceptionOutputChannel):
-            if isinstance(self.exception.oracle, LanguageSpecificOracle):
-                languages = {
-                    SupportedLanguage(x) for x in self.exception.oracle.functions.keys()
-                }
-            elif (
-                self.exception.exception is not None and self.exception.exception.types
-            ):
-                languages = {
-                    SupportedLanguage(x) for x in self.exception.exception.types.keys()
-                }
-        if isinstance(self.result, ValueOutputChannel):
-            if isinstance(self.result.oracle, LanguageSpecificOracle):
-                langs = {
-                    SupportedLanguage(x) for x in self.result.oracle.functions.keys()
-                }
-                if languages is not None:
-                    languages &= langs
-                else:
-                    languages = langs
-
-        return languages
+            if self.exception.exception is not None and self.exception.exception.types:
+                individual_languages.append(set(self.exception.exception.types.keys()))
+
+        # Remove all None elements and merge the rest.
+        without_none = [x for x in individual_languages if x is not None]
+
+        # If we do not have anything, bail now.
+        if len(without_none) == 0:
+            return None
+
+        return set().union(*without_none)
 
 
 @define

tests/exercises/lotto/evaluation/evaluator.py

@@ -1,4 +1,5 @@
 import re
+import ast
 # noinspection PyUnresolvedReferences
 from evaluation_utils import EvaluationResult, Message
 
@@ -48,3 +49,33 @@ def evaluate(context, count, maximum):
     if valid:
         expected = actual
     return EvaluationResult(valid, expected, actual, messages)
+
+
+def check_for_node(search, context, count, maximum):
+    assert context.programming_language == "python", "This exercise only supports Python"
+    # Check if the submission uses a while loop.
+    with open(context.submission_path, "r") as submission_file:
+        submission = submission_file.read()
+
+    # This has no error handling, so it is not ready for production.
+    nodes = ast.walk(ast.parse(submission))
+    has_while = any(isinstance(node, search) for node in nodes)
+    messages = []
+    if not has_while:
+        messages.append("Your code does not use a while loop, which is mandatory.")
+    eval_result = evaluate(context, count, maximum)
+
+    return EvaluationResult(
+        eval_result.result and has_while,
+        eval_result.readable_expected,
+        eval_result.readable_actual,
+        eval_result.messages + messages
+    )
+
+
+def check_for_while(context, count, maximum):
+    return check_for_node(ast.While, context, count, maximum)
+
+
+def check_for_for(context, count, maximum):
+    return check_for_node(ast.For, context, count, maximum)

tests/exercises/lotto/evaluation/one-programmed-analysis-correct.yaml

@@ -0,0 +1,10 @@
+- tab: "Feedback"
+  testcases:
+    - expression: "loterij(18, 172)"
+      return: !oracle
+        oracle: "custom_check"
+        file: "evaluator.py"
+        name: "check_for_while"
+        value: "7 - 37 - 48 - 54 - 70 - 78 - 81 - 90 - 102 - 103 - 113 - 119 - 120 - 137 - 140 - 154 - 157 - 159"
+        arguments: [18, 172]
+        languages: ["python"]

tests/exercises/lotto/evaluation/one-programmed-analysis-without-language.yaml

@@ -0,0 +1,9 @@
+- tab: "Feedback"
+  testcases:
+    - expression: "loterij(18, 172)"
+      return: !oracle
+        oracle: "custom_check"
+        file: "evaluator.py"
+        name: "check_for_while"
+        value: "7 - 37 - 48 - 54 - 70 - 78 - 81 - 90 - 102 - 103 - 113 - 119 - 120 - 137 - 140 - 154 - 157 - 159"
+        arguments: [18, 172]

tests/exercises/lotto/evaluation/one-programmed-analysis-wrong.yaml

@@ -0,0 +1,10 @@
+- tab: "Feedback"
+  testcases:
+    - expression: "loterij(18, 172)"
+      return: !oracle
+        oracle: "custom_check"
+        file: "evaluator.py"
+        name: "check_for_for"
+        value: "7 - 37 - 48 - 54 - 70 - 78 - 81 - 90 - 102 - 103 - 113 - 119 - 120 - 137 - 140 - 154 - 157 - 159"
+        arguments: [18, 172]
+        languages: ["python"]