Can run and parse results from test file (not extension yet)

protocol/types.nim

@@ -1061,4 +1061,20 @@ type
     entryPoints*: seq[string] #can be patterns? if empty we could do the same as nimble does or just run `nimble test args`
 
   ListTestsResult* = object
-    projectInfo*: TestProjectInfo
+    projectInfo*: TestProjectInfo
+
+  RunTestResult* = object
+    name*: string
+    time*: float
+
+  RunTestSuiteResult* = object
+    name*: string
+    tests*: int
+    failures*: int
+    errors*: int
+    skipped*: int
+    time*: float
+    testResults*: seq[RunTestResult]
+
+  RunTestProjectResult* = object
+    suites*: seq[RunTestSuiteResult]

testrunner.nim

@@ -1,4 +1,4 @@
-import std/[os, osproc, strscans, tables, sequtils, enumerate, strutils]
+import std/[os, strscans, tables, enumerate, strutils, xmlparser, xmltree]
 import chronos, chronos/asyncproc
 import protocol/types
 import ls
@@ -56,3 +56,58 @@ proc listTestsForEntryPoint*(
       result = extractTestInfo(rawOutput)
   finally:
     await shutdownChildProcess(process)
+
+
+proc parseObject(obj: var object, node: XmlNode) = 
+  for field, value in obj.fieldPairs:
+    when value is string:
+      getField(obj, field) = node.attr(field)
+    elif value is int:
+      getField(obj, field) = parseInt(node.attr(field))
+    elif value is float:
+      getField(obj, field) = parseFloat(node.attr(field))
+
+proc parseTestResult*(node: XmlNode): RunTestResult =
+  parseObject(result, node)
+
+proc parseTestSuite*(node: XmlNode): RunTestSuiteResult =
+  parseObject(result, node) 
+  for testCase in node.findAll("testcase"):
+    result.testResults.add(parseTestResult(testCase))
+
+proc parseTestResults*(xmlContent: string): RunTestProjectResult =
+  let xml = parseXml(xmlContent)    
+  for suiteNode in xml.findAll("testsuite"):
+    result.suites.add(parseTestSuite(suiteNode))
+
+proc runTests*(entryPoints: seq[string], nimPath: string): Future[RunTestProjectResult] {.async.} =
+  #For now only one entry point is supported
+  assert entryPoints.len == 1
+  let entryPoint = entryPoints[0]  
+  let resultFile = (getTempDir() / "result.xml").absolutePath
+  if not fileExists(entryPoint):
+    error "Entry point does not exist", entryPoint = entryPoint
+    return RunTestProjectResult()
+  let process = await startProcess(
+    nimPath,
+    arguments = @["c", "-r", entryPoints[0], "--xml:" & resultFile],
+    options = {UsePath},
+    stderrHandle = AsyncProcess.Pipe,
+    stdoutHandle = AsyncProcess.Pipe,
+  )
+  try:
+    let res = await process.waitForExit(15.seconds)
+    if res != 0:
+      error "Failed to run tests", nimPath = nimPath, entryPoint = entryPoints[0], res = res    
+      error "An error occurred while running tests", error = string.fromBytes(process.stderrStream.read().await)
+    else:
+      assert fileExists(resultFile)
+      let xmlContent = readFile(resultFile)
+      result = parseTestResults(xmlContent)
+  except Exception as e:
+    let processOutput = string.fromBytes(process.stdoutStream.read().await)
+    error "An error occurred while running tests", error = e.msg
+    error "Output from process", output = processOutput
+  finally:
+    await shutdownChildProcess(process)
+

tests/ttestrunner.nim

@@ -2,7 +2,7 @@ import unittest
 import std/[os, osproc, strscans, tables, sequtils, enumerate, strutils]
 import testhelpers
 import testrunner
-
+import chronos
 
 suite "Test Parser":
   test "should be able to list tests from an entry point":
@@ -32,3 +32,15 @@ suite "Test Parser":
       check testProjectInfo.suites["Sample Tests"].tests[0].line == 4
       check testProjectInfo.suites["Sample Suite"].tests.len == 3
       check testProjectInfo.suites["sampletests.nim"].tests.len == 3
+
+suite "Test Runner":
+  test "should be able to run tests and retrieve results":
+    let entryPoint = getCurrentDir() / "tests" / "projects" / "testrunner" / "tests" / "sampletests.nim"
+    let testProjectResult = waitFor runTests(@[entryPoint], "nim")
+    check testProjectResult.suites.len == 4
+    check testProjectResult.suites[0].name == "Sample Tests"
+    check testProjectResult.suites[0].tests == 1
+    check testProjectResult.suites[0].failures == 0
+    check testProjectResult.suites[0].errors == 0
+    check testProjectResult.suites[0].skipped == 0
+    check testProjectResult.suites[0].time > 0.0 and testProjectResult.suites[0].time < 1.0

utils.nim

@@ -2,6 +2,8 @@ import std/[unicode, uri, strformat, os, strutils, options, json, jsonutils, sug
 import chronos, chronicles, chronos/asyncproc
 import "$nim/compiler/pathutils"
 import json_rpc/private/jrpc_sys
+import macros
+
 type
   FingerTable = seq[tuple[u16pos, offset: int]]
 
@@ -352,4 +354,7 @@ proc shutdownChildProcess*(p: AsyncProcessRef): Future[void]  {.async.} =
       debug "Process forcibly killed with exit code: ", exitCode = forcedExitCode
     except CatchableError:
       debug "Could not kill process in time either!"
-      writeStackTrace()
+      writeStackTrace()
+
+macro getField*(obj: object, fld: string): untyped =
+  result = newDotExpr(obj, newIdentNode(fld.strVal))