Skip to content

Commit 1e2c133

Browse files
committed
Test cases for parsing JUnit XML reports found in the wild
1 parent 3079505 commit 1e2c133

9 files changed

+477
-0
lines changed

ingest_test.go

+112
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"fmt"
99
"testing"
1010

11+
"github.com/stretchr/testify/assert"
1112
"github.com/stretchr/testify/require"
1213
)
1314

@@ -59,3 +60,114 @@ func TestIngest(t *testing.T) {
5960
})
6061
}
6162
}
63+
64+
func TestExamplesInTheWild(t *testing.T) {
65+
tests := []struct {
66+
title string
67+
filename string
68+
origin string
69+
check func(*testing.T, []Suite)
70+
}{
71+
{
72+
title: "catchsoftware example",
73+
filename: "testdata/catchsoftware.xml",
74+
origin: "https://help.catchsoftware.com/display/ET/JUnit+Format",
75+
check: func(t *testing.T, suites []Suite) {
76+
assert.Len(t, suites, 2)
77+
assert.Len(t, suites[0].Tests, 0)
78+
assert.Len(t, suites[1].Tests, 3)
79+
assert.EqualError(t, suites[1].Tests[0].Error, "Assertion failed")
80+
},
81+
},
82+
{
83+
title: "cubic example",
84+
filename: "testdata/cubic.xml",
85+
origin: "https://llg.cubic.org/docs/junit/",
86+
check: func(t *testing.T, suites []Suite) {
87+
assert.Len(t, suites, 1)
88+
assert.Len(t, suites[0].Tests, 1)
89+
assert.Equal(t, "STDOUT text", suites[0].SystemOut)
90+
assert.Equal(t, "STDERR text", suites[0].SystemErr)
91+
assert.Equal(t, "STDOUT text", suites[0].Tests[0].SystemOut)
92+
assert.Equal(t, "STDERR text", suites[0].Tests[0].SystemErr)
93+
},
94+
},
95+
{
96+
title: "go-junit-report example",
97+
filename: "testdata/go-junit-report.xml",
98+
origin: "https://github.com/jstemmer/go-junit-report/blob/master/testdata/06-report.xml",
99+
check: func(t *testing.T, suites []Suite) {
100+
assert.Len(t, suites, 2)
101+
assert.Len(t, suites[0].Tests, 2)
102+
assert.Len(t, suites[1].Tests, 2)
103+
assert.Equal(t, "1.0", suites[0].Properties["go.version"])
104+
assert.Equal(t, "1.0", suites[1].Properties["go.version"])
105+
assert.EqualError(t, suites[1].Tests[0].Error, "file_test.go:11: Error message\nfile_test.go:11: Longer\n\terror\n\tmessage.")
106+
},
107+
},
108+
{
109+
title: "ibm example",
110+
filename: "testdata/ibm.xml",
111+
origin: "https://www.ibm.com/support/knowledgecenter/en/SSQ2R2_14.2.0/com.ibm.rsar.analysis.codereview.cobol.doc/topics/cac_useresults_junit.html",
112+
check: func(t *testing.T, suites []Suite) {
113+
assert.Len(t, suites, 1)
114+
assert.Len(t, suites[0].Tests, 1)
115+
assert.EqualError(t, suites[0].Tests[0].Error, "\nWARNING: Use a program name that matches the source file name\nCategory: COBOL Code Review – Naming Conventions\nFile: /project/PROGRAM.cbl\nLine: 2\n ")
116+
},
117+
},
118+
{
119+
title: "jenkinsci example",
120+
filename: "testdata/jenkinsci.xml",
121+
origin: "https://github.com/jenkinsci/junit-plugin/blob/master/src/test/resources/hudson/tasks/junit/junit-report-1463.xml",
122+
check: func(t *testing.T, suites []Suite) {
123+
assert.Len(t, suites, 1)
124+
assert.Len(t, suites[0].Tests, 6)
125+
assert.Equal(t, "\n", suites[0].Properties["line.separator"])
126+
assert.Equal(t, `\`, suites[0].Properties["file.separator"])
127+
},
128+
},
129+
{
130+
title: "nose2 example",
131+
filename: "testdata/nose2.xml",
132+
origin: "https://nose2.readthedocs.io/en/latest/plugins/junitxml.html",
133+
check: func(t *testing.T, suites []Suite) {
134+
assert.Len(t, suites, 1)
135+
assert.Len(t, suites[0].Tests, 25)
136+
assert.EqualError(t, suites[0].Tests[22].Error, "Traceback (most recent call last):\n File \"nose2/tests/functional/support/scenario/tests_in_package/pkg1/test/test_things.py\", line 13, in test_typeerr\n raise TypeError(\"oops\")\nTypeError: oops\n")
137+
},
138+
},
139+
{
140+
title: "python junit-xml example",
141+
filename: "testdata/python-junit-xml.xml",
142+
origin: "https://pypi.org/project/junit-xml/",
143+
check: func(t *testing.T, suites []Suite) {
144+
assert.Len(t, suites, 1)
145+
assert.Len(t, suites[0].Tests, 1)
146+
assert.Equal(t, "\n I am stdout!\n ", suites[0].Tests[0].SystemOut)
147+
assert.Equal(t, "\n I am stderr!\n ", suites[0].Tests[0].SystemErr)
148+
},
149+
},
150+
{
151+
title: "surefire example",
152+
filename: "testdata/surefire.xml",
153+
origin: "https://gist.github.com/rwbergstrom/6f0193b1a12dca9d358e6043ee6abba4",
154+
check: func(t *testing.T, suites []Suite) {
155+
assert.Len(t, suites, 1)
156+
assert.Len(t, suites[0].Tests, 1)
157+
assert.Equal(t, "\n", suites[0].Properties["line.separator"])
158+
assert.Equal(t, "Hello, World\n", suites[0].Tests[0].SystemOut)
159+
assert.Equal(t, "I'm an error!\n", suites[0].Tests[0].SystemErr)
160+
},
161+
},
162+
}
163+
164+
for index, test := range tests {
165+
name := fmt.Sprintf("#%d - %s", index+1, test.title)
166+
167+
t.Run(name, func(t *testing.T) {
168+
suites, err := IngestFile(test.filename)
169+
require.NoError(t, err)
170+
test.check(t, suites)
171+
})
172+
}
173+
}

testdata/catchsoftware.xml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<testsuites>
3+
<testsuite name="JUnitXmlReporter" errors="0" tests="0" failures="0" time="0" timestamp="2013-05-24T10:23:58" />
4+
<testsuite name="JUnitXmlReporter.constructor" errors="0" skipped="1" tests="3" failures="1" time="0.006" timestamp="2013-05-24T10:23:58">
5+
<properties>
6+
<property name="java.vendor" value="Sun Microsystems Inc." />
7+
<property name="compiler.debug" value="on" />
8+
<property name="project.jdk.classpath" value="jdk.classpath.1.6" />
9+
</properties>
10+
<testcase classname="JUnitXmlReporter.constructor" name="should default path to an empty string" time="0.006">
11+
<failure message="test failure">Assertion failed</failure>
12+
</testcase>
13+
<testcase classname="JUnitXmlReporter.constructor" name="should default consolidate to true" time="0">
14+
<skipped />
15+
</testcase>
16+
<testcase classname="JUnitXmlReporter.constructor" name="should default useDotNotation to true" time="0" />
17+
</testsuite>
18+
</testsuites>

testdata/cubic.xml

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!-- a description of the JUnit XML format and how Jenkins parses it. See also junit.xsd -->
3+
4+
<!-- if only a single testsuite element is present, the testsuites
5+
element can be omitted. All attributes are optional. -->
6+
<testsuites disabled=""
7+
errors=""
8+
failures=""
9+
name=""
10+
tests=""
11+
time=""
12+
>
13+
14+
<!-- testsuite can appear multiple times, if contained in a testsuites element.
15+
It can also be the root element. -->
16+
<testsuite name=""
17+
tests=""
18+
disabled=""
19+
errors=""
20+
failures=""
21+
hostname=""
22+
id=""
23+
package=""
24+
skipped=""
25+
time=""
26+
timestamp=""
27+
>
28+
29+
<!-- Properties (e.g., environment settings) set during test
30+
execution. The properties element can appear 0 or once. -->
31+
<properties>
32+
<!-- property can appear multiple times. The name and value attributres are required. -->
33+
<property name="" value=""/>
34+
</properties>
35+
36+
<!-- testcase can appear multiple times, see /testsuites/testsuite@tests -->
37+
<testcase name=""
38+
assertions=""
39+
classname=""
40+
status=""
41+
time=""
42+
>
43+
44+
<!-- If the test was not executed or failed, you can specify one
45+
the skipped, error or failure elements. -->
46+
47+
<!-- skipped can appear 0 or once. optional -->
48+
<skipped message=""
49+
/>
50+
51+
<!-- Indicates that the test errored. An errored test is one
52+
that had an unanticipated problem. For example an unchecked
53+
throwable or a problem with the implementation of the
54+
test. Contains as a text node relevant data for the error,
55+
for example a stack trace. optional -->
56+
<error message=""
57+
type=""
58+
></error>
59+
60+
<!-- Indicates that the test failed. A failure is a test which
61+
the code has explicitly failed by using the mechanisms for
62+
that purpose. For example via an assertEquals. Contains as
63+
a text node relevant data for the failure, e.g., a stack
64+
trace. optional -->
65+
<failure message=""
66+
type=""
67+
></failure>
68+
69+
<!-- Data that was written to standard out while the test was executed. optional -->
70+
<system-out>STDOUT text</system-out>
71+
72+
<!-- Data that was written to standard error while the test was executed. optional -->
73+
<system-err>STDERR text</system-err>
74+
</testcase>
75+
76+
<!-- Data that was written to standard out while the test suite was executed. optional -->
77+
<system-out>STDOUT text</system-out>
78+
<!-- Data that was written to standard error while the test suite was executed. optional -->
79+
<system-err>STDERR text</system-err>
80+
</testsuite>
81+
</testsuites>

testdata/go-junit-report.xml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<testsuites>
2+
<testsuite tests="2" failures="0" time="0.160" name="package/name1">
3+
<properties>
4+
<property name="go.version" value="1.0"></property>
5+
</properties>
6+
<testcase classname="name1" name="TestOne" time="0.060"></testcase>
7+
<testcase classname="name1" name="TestTwo" time="0.100"></testcase>
8+
</testsuite>
9+
<testsuite tests="2" failures="1" time="0.151" name="package/name2">
10+
<properties>
11+
<property name="go.version" value="1.0"></property>
12+
</properties>
13+
<testcase classname="name2" name="TestOne" time="0.020">
14+
<failure message="Failed" type="">file_test.go:11: Error message&#xA;file_test.go:11: Longer&#xA;&#x9;error&#xA;&#x9;message.</failure>
15+
</testcase>
16+
<testcase classname="name2" name="TestTwo" time="0.130"></testcase>
17+
</testsuite>
18+
</testsuites>

testdata/ibm.xml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<?xml version="1.0" encoding="UTF-8" ?>
2+
<testsuites id="20140612_170519" name="New_configuration (14/06/12 17:05:19)" tests="225" failures="1262" time="0.001">
3+
<testsuite id="codereview.cobol.analysisProvider" name="COBOL Code Review" tests="45" failures="17" time="0.001">
4+
<testcase id="codereview.cobol.rules.ProgramIdRule" name="Use a program name that matches the source file name" time="0.001">
5+
<failure message="PROGRAM.cbl:2 Use a program name that matches the source file name" type="WARNING">
6+
WARNING: Use a program name that matches the source file name
7+
Category: COBOL Code Review – Naming Conventions
8+
File: /project/PROGRAM.cbl
9+
Line: 2
10+
</failure>
11+
</testcase>
12+
</testsuite>
13+
</testsuites>

testdata/jenkinsci.xml

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<!--
3+
The MIT License
4+
5+
Copyright (c) 2004-2009, Sun Microsystems, Inc., Kohsuke Kawaguchi, Erik Ramfelt
6+
7+
Permission is hereby granted, free of charge, to any person obtaining a copy
8+
of this software and associated documentation files (the "Software"), to deal
9+
in the Software without restriction, including without limitation the rights
10+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
copies of the Software, and to permit persons to whom the Software is
12+
furnished to do so, subject to the following conditions:
13+
14+
The above copyright notice and this permission notice shall be included in
15+
all copies or substantial portions of the Software.
16+
17+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23+
THE SOFTWARE.
24+
-->
25+
26+
<testsuites>
27+
<testsuite name="WLI-FI-Tests-Fake" tests="6" failures="0" errors="0" time="1426" package="IT-Interface-WLI-FI" id="1">
28+
<properties>
29+
<property name="java.runtime.name" value="Java(TM) SE Runtime Environment"/>
30+
<property name="sun.boot.library.path" value="C:\Program Files\Java\jdk1.6.0_02\jre\bin"/>
31+
<property name="java.vm.version" value="1.6.0_02-b06"/>
32+
<property name="java.vm.vendor" value="Sun Microsystems Inc."/>
33+
<property name="java.vendor.url" value="http://java.sun.com/"/>
34+
<property name="path.separator" value=";"/>
35+
<property name="java.vm.name" value="Java HotSpot(TM) Client VM"/>
36+
<property name="file.encoding.pkg" value="sun.io"/>
37+
<property name="sun.java.launcher" value="SUN_STANDARD"/>
38+
<property name="user.country" value="NO"/>
39+
<property name="sun.os.patch.level" value="Service Pack 2"/>
40+
<property name="java.vm.specification.name" value="Java Virtual Machine Specification"/>
41+
<property name="user.dir" value="C:\CI\hudson\jobs\Interface_WLI-FI-Fake\workspace"/>
42+
<property name="java.runtime.version" value="1.6.0_02-b06"/>
43+
<property name="java.awt.graphicsenv" value="sun.awt.Win32GraphicsEnvironment"/>
44+
<property name="java.endorsed.dirs" value="C:\Program Files\Java\jdk1.6.0_02\jre\lib\endorsed"/>
45+
<property name="os.arch" value="x86"/>
46+
<property name="java.io.tmpdir" value="C:\Documents and Settings\e5274\Local Settings\Temp\"/>
47+
<property name="line.separator" value="
48+
"/>
49+
<property name="java.vm.specification.vendor" value="Sun Microsystems Inc."/>
50+
<property name="user.variant" value=""/>
51+
<property name="os.name" value="Windows XP"/>
52+
<property name="sun.jnu.encoding" value="Cp1252"/>
53+
<property name="java.library.path" value="C:\Program Files\Java\jdk1.6.0_02\bin;.;C:\WINDOWS\Sun\Java\bin;C:\WINDOWS\system32;C:\WINDOWS;C:\Program Files\Java\jdk1.6.0_02\bin;C:\PROGRA~1\COMPUW~1\FILE-A~1\Dme;C:\PROGRA~1\COMPUW~1\FILE-A~1;C:\WINDOWS\system32;C:\WINDOWS;C:\WINDOWS\System32\Wbem;C:\oracle\ora92\jre\1.4.2\bin\client\;C:\oracle\ora92\jre\1.4.2\bin;c:\oracle\ora92\bin;C:\Program Files\Oracle\jre\1.1.8\bin;C:\Program Files\IBM\Personal Communications\;C:\Program Files\IBM\Trace Facility\;C:\Groovy\groovy-1.1-RC1\bin;C:\Program Files\Subversion\bin;C:\Program Files\GnuWin32\bin;C:\Program Files\putty;C:\Program Files\Common Files\Compuware"/>
54+
<property name="java.specification.name" value="Java Platform API Specification"/>
55+
<property name="java.class.version" value="50.0"/>
56+
<property name="sun.management.compiler" value="HotSpot Client Compiler"/>
57+
<property name="os.version" value="5.1"/>
58+
<property name="user.home" value="C:\Documents and Settings\e5274"/>
59+
<property name="user.timezone" value="Europe/Berlin"/>
60+
<property name="java.awt.printerjob" value="sun.awt.windows.WPrinterJob"/>
61+
<property name="file.encoding" value="Cp1252"/>
62+
<property name="java.specification.version" value="1.6"/>
63+
<property name="java.class.path" value="C:\eviware\soapUI-Pro-1.7.6\bin\soapui-pro-1.7.6.jar;C:\eviware\soapUI-Pro-1.7.6\lib\activation-1.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\javamail-1.4.jar;C:\eviware\soapUI-Pro-1.7.6\lib\wsdl4j-1.6.2-fixed.jar;C:\eviware\soapUI-Pro-1.7.6\lib\junit-3.8.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\log4j-1.2.14.jar;C:\eviware\soapUI-Pro-1.7.6\lib\looks-2.1.2.jar;C:\eviware\soapUI-Pro-1.7.6\lib\forms-1.0.7.jar;C:\eviware\soapUI-Pro-1.7.6\lib\jcalendar-1.3.2.jar;C:\eviware\soapUI-Pro-1.7.6\lib\commons-logging-1.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\not-yet-commons-ssl-0.3.8.jar;C:\eviware\soapUI-Pro-1.7.6\lib\commons-cli-1.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\commons-beanutils-1.7.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\commons-httpclient-3.0.1-soapui.jar;C:\eviware\soapUI-Pro-1.7.6\lib\swingx-SNAPSHOT.jar;C:\eviware\soapUI-Pro-1.7.6\lib\l2fprod-common-fontchooser-0.2-dev.jar;C:\eviware\soapUI-Pro-1.7.6\lib\commons-codec-1.3.jar;C:\eviware\soapUI-Pro-1.7.6\lib\groovy-all-1.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\jetty-6.1.5.jar;C:\eviware\soapUI-Pro-1.7.6\lib\jetty-util-6.1.5.jar;C:\eviware\soapUI-Pro-1.7.6\lib\servlet-api-2.5-6.1.5.jar;C:\eviware\soapUI-Pro-1.7.6\lib\jxl-2.6.5.jar;C:\eviware\soapUI-Pro-1.7.6\lib\idw-1.5.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\ant-1.6.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xbean-2.3.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xbean_xpath-2.3.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xmlpublic-2.3.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\jsr173_1.0_api-xmlbeans-2.3.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\soapui-xmlbeans-1.7.6.jar;C:\eviware\soapUI-Pro-1.7.6\lib\license4j-1.3.jar;C:\eviware\soapUI-Pro-1.7.6\lib\soapui-1.7.6.jar;C:\eviware\soapUI-Pro-1.7.6\lib\soap-xmlbeans-1.2.jar;C:\eviware\soapUI-Pro-1.7.6\lib\j2ee-xmlbeans-1.4.jar;C:\eviware\soapUI-Pro-1.7.6\lib\ext-xmlbeans-1.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\saxon-8.8.jar;C:\eviware\soapUI-Pro-1.7.6\lib\saxon-dom-8.8.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xmlunit-1.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xmlsec-1.2.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xalan-2.6.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\dtd-xercesImpl-2.9.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\wss4j-1.5.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\bcprov-jdk15-133.jar"/>
64+
<property name="user.name" value="e5274"/>
65+
<property name="java.vm.specification.version" value="1.0"/>
66+
<property name="java.home" value="C:\Program Files\Java\jdk1.6.0_02\jre"/>
67+
<property name="sun.arch.data.model" value="32"/>
68+
<property name="user.language" value="no"/>
69+
<property name="java.specification.vendor" value="Sun Microsystems Inc."/>
70+
<property name="awt.toolkit" value="sun.awt.windows.WToolkit"/>
71+
<property name="java.vm.info" value="mixed mode"/>
72+
<property name="java.version" value="1.6.0_02"/>
73+
<property name="java.ext.dirs" value="C:\Program Files\Java\jdk1.6.0_02\jre\lib\ext;C:\WINDOWS\Sun\Java\lib\ext"/>
74+
<property name="sun.boot.class.path" value="C:\Program Files\Java\jdk1.6.0_02\jre\lib\resources.jar;C:\Program Files\Java\jdk1.6.0_02\jre\lib\rt.jar;C:\Program Files\Java\jdk1.6.0_02\jre\lib\sunrsasign.jar;C:\Program Files\Java\jdk1.6.0_02\jre\lib\jsse.jar;C:\Program Files\Java\jdk1.6.0_02\jre\lib\jce.jar;C:\Program Files\Java\jdk1.6.0_02\jre\lib\charsets.jar;C:\Program Files\Java\jdk1.6.0_02\jre\classes"/>
75+
<property name="java.vendor" value="Sun Microsystems Inc."/>
76+
<property name="file.separator" value="\"/>
77+
<property name="java.vendor.url.bug" value="http://java.sun.com/cgi-bin/bugreport.cgi"/>
78+
<property name="sun.io.unicode.encoding" value="UnicodeLittle"/>
79+
<property name="sun.cpu.endian" value="little"/>
80+
<property name="sun.desktop" value="windows"/>
81+
<property name="sun.cpu.isalist" value=""/>
82+
</properties>
83+
<testcase name="IF_importTradeConfirmationToDwh" time="198.0"/>
84+
<testcase name="IF_getAmartaDisbursements" time="119.0"/>
85+
<testcase name="IF_importGLReconDataToDwh" time="423.0"/>
86+
<testcase name="IF_importTradeInstructionsToDwh" time="278.0"/>
87+
<testcase name="IF_getDeviationTradeInstructions" time="408.0"/>
88+
<testcase name="IF_getDwhGLData" time="0.0"/>
89+
</testsuite>
90+
</testsuites>

0 commit comments

Comments
 (0)