From d32a3a4f613189c9f7dc8a86b2a0e85a39e51495 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Mon, 7 Apr 2025 12:23:53 +0100 Subject: [PATCH 1/6] Test parser based on the output of unittest2 --- tests/all.nim | 3 +- tests/projects/testrunner/config.nims | 4 + tests/projects/testrunner/src/testrunner.nim | 7 ++ .../testrunner/src/testrunner/submodule.nim | 12 +++ tests/projects/testrunner/testrunner.nimble | 12 +++ .../projects/testrunner/tests/sampletests.nim | 24 ++++++ tests/projects/testrunner/tests/test1.nim | 12 +++ tests/testhelpers.nim | 57 +++++++++++++ tests/tprojectsetup.nim | 60 +------------- tests/ttestrunner.nim | 82 +++++++++++++++++++ 10 files changed, 214 insertions(+), 59 deletions(-) create mode 100644 tests/projects/testrunner/config.nims create mode 100644 tests/projects/testrunner/src/testrunner.nim create mode 100644 tests/projects/testrunner/src/testrunner/submodule.nim create mode 100644 tests/projects/testrunner/testrunner.nimble create mode 100644 tests/projects/testrunner/tests/sampletests.nim create mode 100644 tests/projects/testrunner/tests/test1.nim create mode 100644 tests/testhelpers.nim create mode 100644 tests/ttestrunner.nim diff --git a/tests/all.nim b/tests/all.nim index 1bc2cfe..a1f8d62 100644 --- a/tests/all.nim +++ b/tests/all.nim @@ -3,4 +3,5 @@ import tnimlangserver, tprojectsetup, textensions, - tmisc \ No newline at end of file + tmisc, + ttestrunner \ No newline at end of file diff --git a/tests/projects/testrunner/config.nims b/tests/projects/testrunner/config.nims new file mode 100644 index 0000000..8ee48d2 --- /dev/null +++ b/tests/projects/testrunner/config.nims @@ -0,0 +1,4 @@ +# begin Nimble config (version 2) +when withDir(thisDir(), system.fileExists("nimble.paths")): + include "nimble.paths" +# end Nimble config diff --git a/tests/projects/testrunner/src/testrunner.nim b/tests/projects/testrunner/src/testrunner.nim new file mode 100644 index 0000000..b7a2480 --- /dev/null +++ b/tests/projects/testrunner/src/testrunner.nim @@ -0,0 +1,7 @@ +# This is just an example to get you started. A typical library package +# exports the main API in this file. Note that you cannot rename this file +# but you can remove it if you wish. + +proc add*(x, y: int): int = + ## Adds two numbers together. + return x + y diff --git a/tests/projects/testrunner/src/testrunner/submodule.nim b/tests/projects/testrunner/src/testrunner/submodule.nim new file mode 100644 index 0000000..387dcba --- /dev/null +++ b/tests/projects/testrunner/src/testrunner/submodule.nim @@ -0,0 +1,12 @@ +# This is just an example to get you started. Users of your library will +# import this file by writing ``import testrunner/submodule``. Feel free to rename or +# remove this file altogether. You may create additional modules alongside +# this file as required. + +type + Submodule* = object + name*: string + +proc initSubmodule*(): Submodule = + ## Initialises a new ``Submodule`` object. + Submodule(name: "Anonymous") diff --git a/tests/projects/testrunner/testrunner.nimble b/tests/projects/testrunner/testrunner.nimble new file mode 100644 index 0000000..4652d6d --- /dev/null +++ b/tests/projects/testrunner/testrunner.nimble @@ -0,0 +1,12 @@ +# Package + +version = "0.1.0" +author = "jmgomez" +description = "A new awesome nimble package" +license = "MIT" +srcDir = "src" + + +# Dependencies + +requires "nim", "https://github.com/jmgomez/nim-unittest2#list_tests" diff --git a/tests/projects/testrunner/tests/sampletests.nim b/tests/projects/testrunner/tests/sampletests.nim new file mode 100644 index 0000000..9aaeba3 --- /dev/null +++ b/tests/projects/testrunner/tests/sampletests.nim @@ -0,0 +1,24 @@ +import unittest2 + +suite "Sample Tests": + test "Sample Test": + check(1 == 1) + +test "Global test": + check(1 == 1) + +test "Global test 2": + check(1 == 1) + +suite "Sample Suite": + test "Sample Test": + check(1 == 1) + + test "Sample Test 2": + check(1 == 1) + + test "Sample Test 3": + check(1 == 1) + +test "Another global test": + check(1 == 1) diff --git a/tests/projects/testrunner/tests/test1.nim b/tests/projects/testrunner/tests/test1.nim new file mode 100644 index 0000000..95b332e --- /dev/null +++ b/tests/projects/testrunner/tests/test1.nim @@ -0,0 +1,12 @@ +# This is just an example to get you started. You may wish to put all of your +# tests into a single file, or separate them into multiple `test1`, `test2` +# etc. files (better names are recommended, just make sure the name starts with +# the letter 't'). +# +# To run these tests, simply execute `nimble test`. + +import unittest2 + +import testrunner +test "can add": + check add(5, 5) == 10 diff --git a/tests/testhelpers.nim b/tests/testhelpers.nim new file mode 100644 index 0000000..0820916 --- /dev/null +++ b/tests/testhelpers.nim @@ -0,0 +1,57 @@ +import std/[os, osproc, sequtils, strutils, sugar, unittest] + +template cd*(dir: string, body: untyped) = + ## Sets the current dir to ``dir``, executes ``body`` and restores the + ## previous working dir. + let lastDir = getCurrentDir() + setCurrentDir(dir) + block: + defer: + setCurrentDir(lastDir) + body + +template createNewDir*(dir: string) = + removeDir dir + createDir dir + +template cdNewDir*(dir: string, body: untyped) = + createNewDir dir + cd dir: + body + +let + rootDir = getCurrentDir() + nimblePath* = findExe "nimble" + installDir* = rootDir / "tests" / "nimbleDir" + +type ProcessOutput* = tuple[output: string, exitCode: int] + +proc execNimble*(args: varargs[string]): ProcessOutput = + var quotedArgs = @args + if not args.anyIt("--nimbleDir:" in it or "-l" == it or "--local" == it): + quotedArgs.insert("--nimbleDir:" & installDir) + quotedArgs.insert(nimblePath) + quotedArgs = quotedArgs.map((x: string) => x.quoteShell) + + let path {.used.} = getCurrentDir().parentDir() / "src" + + var cmd = + when not defined(windows): + "PATH=" & path & ":$PATH " & quotedArgs.join(" ") + else: + quotedArgs.join(" ") + when defined(macosx): + # TODO: Yeah, this is really specific to my machine but for my own sanity... + cmd = "DYLD_LIBRARY_PATH=/usr/local/opt/openssl@1.1/lib " & cmd + + result = execCmdEx(cmd) + checkpoint(cmd) + checkpoint(result.output) + +proc execNimbleYes*(args: varargs[string]): ProcessOutput = + execNimble(@args & "-y") + +proc createNimbleProject*(projectDir: string) = + cdNewDir projectDir: + let (output, exitCode) = execNimbleYes("init") + check exitCode == 0 \ No newline at end of file diff --git a/tests/tprojectsetup.nim b/tests/tprojectsetup.nim index 51d1797..1e400f5 100644 --- a/tests/tprojectsetup.nim +++ b/tests/tprojectsetup.nim @@ -3,68 +3,12 @@ import ../protocol/[enums, types] import std/ [ - options, unittest, json, os, jsonutils, sequtils, strutils, sugar, strformat, - osproc, + options, unittest, json, os, jsonutils, sequtils, strutils, sugar, strformat ] import json_rpc/[rpcclient] import chronicles import lspsocketclient - -template cd*(dir: string, body: untyped) = - ## Sets the current dir to ``dir``, executes ``body`` and restores the - ## previous working dir. - let lastDir = getCurrentDir() - setCurrentDir(dir) - block: - defer: - setCurrentDir(lastDir) - body - -template createNewDir*(dir: string) = - removeDir dir - createDir dir - -template cdNewDir*(dir: string, body: untyped) = - createNewDir dir - cd dir: - body - -let - rootDir = getCurrentDir() - nimblePath* = findExe "nimble" - installDir* = rootDir / "tests" / "nimbleDir" - -type ProcessOutput* = tuple[output: string, exitCode: int] - -proc execNimble*(args: varargs[string]): ProcessOutput = - var quotedArgs = @args - if not args.anyIt("--nimbleDir:" in it or "-l" == it or "--local" == it): - quotedArgs.insert("--nimbleDir:" & installDir) - quotedArgs.insert(nimblePath) - quotedArgs = quotedArgs.map((x: string) => x.quoteShell) - - let path {.used.} = getCurrentDir().parentDir() / "src" - - var cmd = - when not defined(windows): - "PATH=" & path & ":$PATH " & quotedArgs.join(" ") - else: - quotedArgs.join(" ") - when defined(macosx): - # TODO: Yeah, this is really specific to my machine but for my own sanity... - cmd = "DYLD_LIBRARY_PATH=/usr/local/opt/openssl@1.1/lib " & cmd - - result = execCmdEx(cmd) - checkpoint(cmd) - checkpoint(result.output) - -proc execNimbleYes*(args: varargs[string]): ProcessOutput = - execNimble(@args & "-y") - -proc createNimbleProject(projectDir: string) = - cdNewDir projectDir: - let (output, exitCode) = execNimbleYes("init") - check exitCode == 0 +import testhelpers suite "nimble setup": let cmdParams = CommandLineParams(transport: some socket, port: getNextFreePort()) diff --git a/tests/ttestrunner.nim b/tests/ttestrunner.nim new file mode 100644 index 0000000..81b163c --- /dev/null +++ b/tests/ttestrunner.nim @@ -0,0 +1,82 @@ +import unittest +import std/[os, osproc, strscans, tables, sequtils, enumerate, strutils] +import testhelpers + +type + TestInfo* = object + name*: string + line*: int + file*: string + + TestSuiteInfo* = object + name*: string #The suite name, empty if it's a global test + tests*: seq[TestInfo] + + TestProjectInfo* = object + entryPoints*: seq[string] + suites*: Table[string, TestSuiteInfo] + + + + +proc extractTestInfo*(rawOutput: string): TestProjectInfo = + result.suites = initTable[string, TestSuiteInfo]() + result.suites[""] = TestSuiteInfo(name: "") + let lines = rawOutput.split("\n") + var currentSuite = "" + + for i, line in enumerate(lines): + var name, file, ignore: string + var lineNumber: int + + if scanf(line, "Suite: $*", name): + currentSuite = name.strip() + result.suites[currentSuite] = TestSuiteInfo(name: currentSuite) + echo "Found suite: ", currentSuite + + elif scanf(line, "$*Test: $*", ignore, name): + let insideSuite = line.startsWith("\t") + # Use currentSuite if inside a suite, empty string if not + let suiteName = if insideSuite: currentSuite else: "" + + #File is always next line of a test + if scanf(lines[i+1], "$*File:$*:$i", ignore, file, lineNumber): + var testInfo = TestInfo(name: name.strip(), file: file.strip(), line: lineNumber) + echo "Adding test: ", testInfo.name, " to suite: ", suiteName + result.suites[suiteName].tests.add(testInfo) + + + + + +suite "Test Parser": + test "should be able to list tests from an entry point": + #A project can have multiple entry points for the tests, they are specified in the test runner. + #We first need to install the project, as it uses a custom version of unittest2 (until it get merged). + let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" + cd projectDir: + let (output, _) = execNimble("install", "-l") + let (listTestsOutput, _) = execCmdEx("nim c -d:listTests ./tests/test1.nim") + let testProjectInfo = extractTestInfo(listTestsOutput) + check testProjectInfo.suites.len == 1 + check testProjectInfo.suites[""].tests.len == 1 + check testProjectInfo.suites[""].tests[0].name == "can add" + check testProjectInfo.suites[""].tests[0].file == "test1.nim" + check testProjectInfo.suites[""].tests[0].line == 11 + + test "should be able to list tests and suites": + let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" + cd projectDir: + let (listTestsOutput, _) = execCmdEx("nim c -d:listTests ./tests/sampletests.nim") + echo "*****" + echo listTestsOutput + echo "*****" + let testProjectInfo = extractTestInfo(listTestsOutput) + echo testProjectInfo + check testProjectInfo.suites.len == 3 + check testProjectInfo.suites["Sample Tests"].tests.len == 1 + check testProjectInfo.suites["Sample Tests"].tests[0].name == "Sample Test" + check testProjectInfo.suites["Sample Tests"].tests[0].file == "sampletests.nim" + check testProjectInfo.suites["Sample Tests"].tests[0].line == 4 + check testProjectInfo.suites["Sample Suite"].tests.len == 3 + check testProjectInfo.suites[""].tests.len == 3 From 38e56c040e43a07c9225f6f4910eedda5aa9daca Mon Sep 17 00:00:00 2001 From: jmgomez Date: Tue, 8 Apr 2025 11:06:42 +0100 Subject: [PATCH 2/6] Update tests to use the runtime test listing --- tests/ttestrunner.nim | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/tests/ttestrunner.nim b/tests/ttestrunner.nim index 81b163c..68f9175 100644 --- a/tests/ttestrunner.nim +++ b/tests/ttestrunner.nim @@ -21,18 +21,16 @@ type proc extractTestInfo*(rawOutput: string): TestProjectInfo = result.suites = initTable[string, TestSuiteInfo]() - result.suites[""] = TestSuiteInfo(name: "") let lines = rawOutput.split("\n") var currentSuite = "" for i, line in enumerate(lines): var name, file, ignore: string var lineNumber: int - if scanf(line, "Suite: $*", name): currentSuite = name.strip() result.suites[currentSuite] = TestSuiteInfo(name: currentSuite) - echo "Found suite: ", currentSuite + # echo "Found suite: ", currentSuite elif scanf(line, "$*Test: $*", ignore, name): let insideSuite = line.startsWith("\t") @@ -42,13 +40,11 @@ proc extractTestInfo*(rawOutput: string): TestProjectInfo = #File is always next line of a test if scanf(lines[i+1], "$*File:$*:$i", ignore, file, lineNumber): var testInfo = TestInfo(name: name.strip(), file: file.strip(), line: lineNumber) - echo "Adding test: ", testInfo.name, " to suite: ", suiteName + # echo "Adding test: ", testInfo.name, " to suite: ", suiteName result.suites[suiteName].tests.add(testInfo) - - suite "Test Parser": test "should be able to list tests from an entry point": #A project can have multiple entry points for the tests, they are specified in the test runner. @@ -56,27 +52,24 @@ suite "Test Parser": let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" cd projectDir: let (output, _) = execNimble("install", "-l") - let (listTestsOutput, _) = execCmdEx("nim c -d:listTests ./tests/test1.nim") + discard execNimble("setup") + let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/test1.nim") let testProjectInfo = extractTestInfo(listTestsOutput) check testProjectInfo.suites.len == 1 - check testProjectInfo.suites[""].tests.len == 1 - check testProjectInfo.suites[""].tests[0].name == "can add" - check testProjectInfo.suites[""].tests[0].file == "test1.nim" - check testProjectInfo.suites[""].tests[0].line == 11 + check testProjectInfo.suites["test1.nim"].tests.len == 1 + check testProjectInfo.suites["test1.nim"].tests[0].name == "can add" + check testProjectInfo.suites["test1.nim"].tests[0].file == "test1.nim" + check testProjectInfo.suites["test1.nim"].tests[0].line == 11 test "should be able to list tests and suites": let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" cd projectDir: - let (listTestsOutput, _) = execCmdEx("nim c -d:listTests ./tests/sampletests.nim") - echo "*****" - echo listTestsOutput - echo "*****" + let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/sampletests.nim") let testProjectInfo = extractTestInfo(listTestsOutput) - echo testProjectInfo check testProjectInfo.suites.len == 3 check testProjectInfo.suites["Sample Tests"].tests.len == 1 check testProjectInfo.suites["Sample Tests"].tests[0].name == "Sample Test" check testProjectInfo.suites["Sample Tests"].tests[0].file == "sampletests.nim" check testProjectInfo.suites["Sample Tests"].tests[0].line == 4 check testProjectInfo.suites["Sample Suite"].tests.len == 3 - check testProjectInfo.suites[""].tests.len == 3 + check testProjectInfo.suites["sampletests.nim"].tests.len == 3 From d4a26398d3d64a29d5e5a6fa5358deb0b7d14fbe Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 9 Apr 2025 10:41:53 +0100 Subject: [PATCH 3/6] debug one test --- tests/all.nim | 10 +++++----- tests/ttestrunner.nim | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/all.nim b/tests/all.nim index a1f8d62..94f494d 100644 --- a/tests/all.nim +++ b/tests/all.nim @@ -1,7 +1,7 @@ import - tsuggestapi, - tnimlangserver, - tprojectsetup, - textensions, - tmisc, + # tsuggestapi, + # tnimlangserver, + # tprojectsetup, + # textensions, + # tmisc, ttestrunner \ No newline at end of file diff --git a/tests/ttestrunner.nim b/tests/ttestrunner.nim index 68f9175..59c7b73 100644 --- a/tests/ttestrunner.nim +++ b/tests/ttestrunner.nim @@ -55,6 +55,7 @@ suite "Test Parser": discard execNimble("setup") let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/test1.nim") let testProjectInfo = extractTestInfo(listTestsOutput) + echo testProjectInfo check testProjectInfo.suites.len == 1 check testProjectInfo.suites["test1.nim"].tests.len == 1 check testProjectInfo.suites["test1.nim"].tests[0].name == "can add" From 4274f650466590819446a658b248838f6de1eb02 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 9 Apr 2025 10:52:26 +0100 Subject: [PATCH 4/6] progress --- tests/projects/testrunner/tests/test1.nim | 1 - tests/ttestrunner.nim | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/projects/testrunner/tests/test1.nim b/tests/projects/testrunner/tests/test1.nim index 95b332e..33253a0 100644 --- a/tests/projects/testrunner/tests/test1.nim +++ b/tests/projects/testrunner/tests/test1.nim @@ -7,6 +7,5 @@ import unittest2 -import testrunner test "can add": check add(5, 5) == 10 diff --git a/tests/ttestrunner.nim b/tests/ttestrunner.nim index 59c7b73..0a8b038 100644 --- a/tests/ttestrunner.nim +++ b/tests/ttestrunner.nim @@ -55,6 +55,9 @@ suite "Test Parser": discard execNimble("setup") let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/test1.nim") let testProjectInfo = extractTestInfo(listTestsOutput) + echo "***LIST TESTS OUTPUT***" + echo listTestsOutput + echo "***TEST PROJECT INFO***" echo testProjectInfo check testProjectInfo.suites.len == 1 check testProjectInfo.suites["test1.nim"].tests.len == 1 From 1695bc29507ada9351d84b08506a0393b98a70f1 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 9 Apr 2025 11:17:27 +0100 Subject: [PATCH 5/6] Fixes tests --- tests/all.nim | 10 +++++----- tests/ttestrunner.nim | 8 ++------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/all.nim b/tests/all.nim index 94f494d..a1f8d62 100644 --- a/tests/all.nim +++ b/tests/all.nim @@ -1,7 +1,7 @@ import - # tsuggestapi, - # tnimlangserver, - # tprojectsetup, - # textensions, - # tmisc, + tsuggestapi, + tnimlangserver, + tprojectsetup, + textensions, + tmisc, ttestrunner \ No newline at end of file diff --git a/tests/ttestrunner.nim b/tests/ttestrunner.nim index 0a8b038..2d6e08b 100644 --- a/tests/ttestrunner.nim +++ b/tests/ttestrunner.nim @@ -54,16 +54,12 @@ suite "Test Parser": let (output, _) = execNimble("install", "-l") discard execNimble("setup") let (listTestsOutput, _) = execCmdEx("nim c -d:unittest2ListTests -r ./tests/test1.nim") - let testProjectInfo = extractTestInfo(listTestsOutput) - echo "***LIST TESTS OUTPUT***" - echo listTestsOutput - echo "***TEST PROJECT INFO***" - echo testProjectInfo + let testProjectInfo = extractTestInfo(listTestsOutput) check testProjectInfo.suites.len == 1 check testProjectInfo.suites["test1.nim"].tests.len == 1 check testProjectInfo.suites["test1.nim"].tests[0].name == "can add" check testProjectInfo.suites["test1.nim"].tests[0].file == "test1.nim" - check testProjectInfo.suites["test1.nim"].tests[0].line == 11 + check testProjectInfo.suites["test1.nim"].tests[0].line == 10 test "should be able to list tests and suites": let projectDir = getCurrentDir() / "tests" / "projects" / "testrunner" From d24796f692ebac2cac32c5d5fe18506a993bb6ff Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 9 Apr 2025 11:25:54 +0100 Subject: [PATCH 6/6] fixes test --- tests/projects/testrunner/tests/test1.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/projects/testrunner/tests/test1.nim b/tests/projects/testrunner/tests/test1.nim index 33253a0..2b1c6e8 100644 --- a/tests/projects/testrunner/tests/test1.nim +++ b/tests/projects/testrunner/tests/test1.nim @@ -8,4 +8,4 @@ import unittest2 test "can add": - check add(5, 5) == 10 + check true