From 5a67b63095f3e691462dcb980be6dbd93f76b113 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 9 Apr 2025 13:16:41 +0100 Subject: [PATCH 1/2] Can run and parse results from test file (not extension yet) --- protocol/types.nim | 18 +++++++++++- testrunner.nim | 57 ++++++++++++++++++++++++++++++++++++- tests/ttestrunner.nim | 66 +++++++++++++++++++++++++------------------ utils.nim | 7 ++++- 4 files changed, 118 insertions(+), 30 deletions(-) diff --git a/protocol/types.nim b/protocol/types.nim index d954911..a52b4e5 100644 --- a/protocol/types.nim +++ b/protocol/types.nim @@ -1061,4 +1061,20 @@ type entryPoints*: seq[string] #can be patterns? if empty we could do the same as nimble does or just run `nimble test args` ListTestsResult* = object - projectInfo*: TestProjectInfo \ No newline at end of file + projectInfo*: TestProjectInfo + + RunTestResult* = object + name*: string + time*: float + + RunTestSuiteResult* = object + name*: string + tests*: int + failures*: int + errors*: int + skipped*: int + time*: float + testResults*: seq[RunTestResult] + + RunTestProjectResult* = object + suites*: seq[RunTestSuiteResult] \ No newline at end of file diff --git a/testrunner.nim b/testrunner.nim index 29b00f7..ea5db0e 100644 --- a/testrunner.nim +++ b/testrunner.nim @@ -1,4 +1,4 @@ -import std/[os, osproc, strscans, tables, sequtils, enumerate, strutils] +import std/[os, strscans, tables, enumerate, strutils, xmlparser, xmltree] import chronos, chronos/asyncproc import protocol/types import ls @@ -56,3 +56,58 @@ proc listTestsForEntryPoint*( result = extractTestInfo(rawOutput) finally: await shutdownChildProcess(process) + + +proc parseObject(obj: var object, node: XmlNode) = + for field, value in obj.fieldPairs: + when value is string: + getField(obj, field) = node.attr(field) + elif value is int: + getField(obj, field) = parseInt(node.attr(field)) + elif value is float: + getField(obj, field) = parseFloat(node.attr(field)) + +proc parseTestResult*(node: XmlNode): RunTestResult = + parseObject(result, node) + +proc parseTestSuite*(node: XmlNode): RunTestSuiteResult = + parseObject(result, node) + for testCase in node.findAll("testcase"): + result.testResults.add(parseTestResult(testCase)) + +proc parseTestResults*(xmlContent: string): RunTestProjectResult = + let xml = parseXml(xmlContent) + for suiteNode in xml.findAll("testsuite"): + result.suites.add(parseTestSuite(suiteNode)) + +proc runTests*(entryPoints: seq[string], nimPath: string): Future[RunTestProjectResult] {.async.} = + #For now only one entry point is supported + assert entryPoints.len == 1 + let entryPoint = entryPoints[0] + let resultFile = (getTempDir() / "result.xml").absolutePath + if not fileExists(entryPoint): + error "Entry point does not exist", entryPoint = entryPoint + return RunTestProjectResult() + let process = await startProcess( + nimPath, + arguments = @["c", "-r", entryPoints[0], "--xml:" & resultFile], + options = {UsePath}, + stderrHandle = AsyncProcess.Pipe, + stdoutHandle = AsyncProcess.Pipe, + ) + try: + let res = await process.waitForExit(15.seconds) + if res != 0: + error "Failed to run tests", nimPath = nimPath, entryPoint = entryPoints[0], res = res + error "An error occurred while running tests", error = string.fromBytes(process.stderrStream.read().await) + else: + assert fileExists(resultFile) + let xmlContent = readFile(resultFile) + result = parseTestResults(xmlContent) + except Exception as e: + let processOutput = string.fromBytes(process.stdoutStream.read().await) + error "An error occurred while running tests", error = e.msg + error "Output from process", output = processOutput + finally: + await shutdownChildProcess(process) + \ No newline at end of file diff --git a/tests/ttestrunner.nim b/tests/ttestrunner.nim index 92ff187..da0afb4 100644 --- a/tests/ttestrunner.nim +++ b/tests/ttestrunner.nim @@ -2,33 +2,45 @@ import unittest import std/[os, osproc, strscans, tables, sequtils, enumerate, strutils] import testhelpers import testrunner +import chronos +# suite "Test Parser": + # test "should be able to list tests from an entry point": + # #A project can have multiple entry points for the tests, they are specified in the test runner. + # #We first need to install the project, as it uses a custom version of unittest2 (until it get merged). + # let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" + # cd projectDir: + # let (output, _) = execNimble("install", "-l") + # discard execNimble("setup") + # let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/test1.nim") + # let testProjectInfo = extractTestInfo(listTestsOutput) + # check testProjectInfo.suites.len == 1 + # check testProjectInfo.suites["test1.nim"].tests.len == 1 + # check testProjectInfo.suites["test1.nim"].tests[0].name == "can add" + # check testProjectInfo.suites["test1.nim"].tests[0].file == "test1.nim" + # check testProjectInfo.suites["test1.nim"].tests[0].line == 10 -suite "Test Parser": - test "should be able to list tests from an entry point": - #A project can have multiple entry points for the tests, they are specified in the test runner. - #We first need to install the project, as it uses a custom version of unittest2 (until it get merged). - let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" - cd projectDir: - let (output, _) = execNimble("install", "-l") - discard execNimble("setup") - let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/test1.nim") - let testProjectInfo = extractTestInfo(listTestsOutput) - check testProjectInfo.suites.len == 1 - check testProjectInfo.suites["test1.nim"].tests.len == 1 - check testProjectInfo.suites["test1.nim"].tests[0].name == "can add" - check testProjectInfo.suites["test1.nim"].tests[0].file == "test1.nim" - check testProjectInfo.suites["test1.nim"].tests[0].line == 10 + # test "should be able to list tests and suites": + # let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" + # cd projectDir: + # let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/sampletests.nim") + # let testProjectInfo = extractTestInfo(listTestsOutput) + # check testProjectInfo.suites.len == 3 + # check testProjectInfo.suites["Sample Tests"].tests.len == 1 + # check testProjectInfo.suites["Sample Tests"].tests[0].name == "Sample Test" + # check testProjectInfo.suites["Sample Tests"].tests[0].file == "sampletests.nim" + # check testProjectInfo.suites["Sample Tests"].tests[0].line == 4 + # check testProjectInfo.suites["Sample Suite"].tests.len == 3 + # check testProjectInfo.suites["sampletests.nim"].tests.len == 3 - test "should be able to list tests and suites": - let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" - cd projectDir: - let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/sampletests.nim") - let testProjectInfo = extractTestInfo(listTestsOutput) - check testProjectInfo.suites.len == 3 - check testProjectInfo.suites["Sample Tests"].tests.len == 1 - check testProjectInfo.suites["Sample Tests"].tests[0].name == "Sample Test" - check testProjectInfo.suites["Sample Tests"].tests[0].file == "sampletests.nim" - check testProjectInfo.suites["Sample Tests"].tests[0].line == 4 - check testProjectInfo.suites["Sample Suite"].tests.len == 3 - check testProjectInfo.suites["sampletests.nim"].tests.len == 3 +suite "Test Runner": + test "should be able to run tests and retrieve results": + let entryPoint = getCurrentDir() / "tests" / "projects" / "testrunner" / "tests" / "sampletests.nim" + let testProjectResult = waitFor runTests(@[entryPoint], "nim") + check testProjectResult.suites.len == 4 + check testProjectResult.suites[0].name == "Sample Tests" + check testProjectResult.suites[0].tests == 1 + check testProjectResult.suites[0].failures == 0 + check testProjectResult.suites[0].errors == 0 + check testProjectResult.suites[0].skipped == 0 + check testProjectResult.suites[0].time > 0.0 and testProjectResult.suites[0].time < 1.0 diff --git a/utils.nim b/utils.nim index 39e81bc..e90d78e 100644 --- a/utils.nim +++ b/utils.nim @@ -2,6 +2,8 @@ import std/[unicode, uri, strformat, os, strutils, options, json, jsonutils, sug import chronos, chronicles, chronos/asyncproc import "$nim/compiler/pathutils" import json_rpc/private/jrpc_sys +import macros + type FingerTable = seq[tuple[u16pos, offset: int]] @@ -352,4 +354,7 @@ proc shutdownChildProcess*(p: AsyncProcessRef): Future[void] {.async.} = debug "Process forcibly killed with exit code: ", exitCode = forcedExitCode except CatchableError: debug "Could not kill process in time either!" - writeStackTrace() \ No newline at end of file + writeStackTrace() + +macro getField*(obj: object, fld: string): untyped = + result = newDotExpr(obj, newIdentNode(fld.strVal)) \ No newline at end of file From 6e551c1e75ff0dbf9750ad4cf53708cf266b5dad Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 9 Apr 2025 13:17:32 +0100 Subject: [PATCH 2/2] uncomment tests --- tests/ttestrunner.nim | 54 +++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/ttestrunner.nim b/tests/ttestrunner.nim index da0afb4..114294f 100644 --- a/tests/ttestrunner.nim +++ b/tests/ttestrunner.nim @@ -4,34 +4,34 @@ import testhelpers import testrunner import chronos -# suite "Test Parser": - # test "should be able to list tests from an entry point": - # #A project can have multiple entry points for the tests, they are specified in the test runner. - # #We first need to install the project, as it uses a custom version of unittest2 (until it get merged). - # let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" - # cd projectDir: - # let (output, _) = execNimble("install", "-l") - # discard execNimble("setup") - # let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/test1.nim") - # let testProjectInfo = extractTestInfo(listTestsOutput) - # check testProjectInfo.suites.len == 1 - # check testProjectInfo.suites["test1.nim"].tests.len == 1 - # check testProjectInfo.suites["test1.nim"].tests[0].name == "can add" - # check testProjectInfo.suites["test1.nim"].tests[0].file == "test1.nim" - # check testProjectInfo.suites["test1.nim"].tests[0].line == 10 +suite "Test Parser": + test "should be able to list tests from an entry point": + #A project can have multiple entry points for the tests, they are specified in the test runner. + #We first need to install the project, as it uses a custom version of unittest2 (until it get merged). + let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" + cd projectDir: + let (output, _) = execNimble("install", "-l") + discard execNimble("setup") + let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/test1.nim") + let testProjectInfo = extractTestInfo(listTestsOutput) + check testProjectInfo.suites.len == 1 + check testProjectInfo.suites["test1.nim"].tests.len == 1 + check testProjectInfo.suites["test1.nim"].tests[0].name == "can add" + check testProjectInfo.suites["test1.nim"].tests[0].file == "test1.nim" + check testProjectInfo.suites["test1.nim"].tests[0].line == 10 - # test "should be able to list tests and suites": - # let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" - # cd projectDir: - # let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/sampletests.nim") - # let testProjectInfo = extractTestInfo(listTestsOutput) - # check testProjectInfo.suites.len == 3 - # check testProjectInfo.suites["Sample Tests"].tests.len == 1 - # check testProjectInfo.suites["Sample Tests"].tests[0].name == "Sample Test" - # check testProjectInfo.suites["Sample Tests"].tests[0].file == "sampletests.nim" - # check testProjectInfo.suites["Sample Tests"].tests[0].line == 4 - # check testProjectInfo.suites["Sample Suite"].tests.len == 3 - # check testProjectInfo.suites["sampletests.nim"].tests.len == 3 + test "should be able to list tests and suites": + let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" + cd projectDir: + let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/sampletests.nim") + let testProjectInfo = extractTestInfo(listTestsOutput) + check testProjectInfo.suites.len == 3 + check testProjectInfo.suites["Sample Tests"].tests.len == 1 + check testProjectInfo.suites["Sample Tests"].tests[0].name == "Sample Test" + check testProjectInfo.suites["Sample Tests"].tests[0].file == "sampletests.nim" + check testProjectInfo.suites["Sample Tests"].tests[0].line == 4 + check testProjectInfo.suites["Sample Suite"].tests.len == 3 + check testProjectInfo.suites["sampletests.nim"].tests.len == 3 suite "Test Runner": test "should be able to run tests and retrieve results":