/hg/icedtea-web: Added KnownToFail annotation
jvanek at icedtea.classpath.org
jvanek at icedtea.classpath.org
Tue Jun 5 07:32:21 PDT 2012
changeset 30d29b2698f3 in /hg/icedtea-web
details: http://icedtea.classpath.org/hg/icedtea-web?cmd=changeset;node=30d29b2698f3
author: Jiri Vanek <jvanek at redhat.com>
date: Tue Jun 05 16:35:50 2012 +0200
Added KnownToFail annotation
diffstat:
ChangeLog | 19 +
tests/jnlp_tests/signed/CacheReproducer/testcases/CacheReproducerTest.java | 2 +
tests/junit-runner/JunitLikeXmlOutputListener.java | 170 +++++++--
tests/junit-runner/LessVerboseTextListener.java | 66 +++
tests/netx/jnlp_testsengine/net/sourceforge/jnlp/annotations/KnownToFail.java | 24 +
tests/netx/unit/net/sourceforge/jnlp/JNLPMatcherTest.java | 3 +
tests/netx/unit/net/sourceforge/jnlp/ParserCornerCases.java | 4 +
tests/netx/unit/net/sourceforge/jnlp/ParserMalformedXml.java | 4 +
tests/report-styles/jreport.xsl | 62 +++-
9 files changed, 305 insertions(+), 49 deletions(-)
diffs (truncated from 704 to 500 lines):
diff -r c87cba6608c6 -r 30d29b2698f3 ChangeLog
--- a/ChangeLog Tue Jun 05 16:14:42 2012 +0200
+++ b/ChangeLog Tue Jun 05 16:35:50 2012 +0200
@@ -1,3 +1,22 @@
+2012-05-14 Jiri Vanek <jvanek at redhat.com>
+
+ * tests/netx/jnlp_testsengine/net/sourceforge/jnlp/annotations/KnownToFail.java
+ New file. Annotation for marking failing tests.
+ * tests/report-styles/jreport.xsl:
+ * tests/junit-runner/LessVerboseTextListener.java:
+ * tests/junit-runner/JunitLikeXmlOutputListener.java: Added counting
+ and printing of @KnownToFail annotations if presented.
+ * tests/jnlp_tests/simple/Spaces can be everywhere/testcases/SpacesCanBeEverywhereTests.java:
+ (SpacesCanBeEverywhereRemoteTests1) (SpacesCanBeEverywhereRemoteTests2)
+ (SpacesCanBeEverywhereRemoteTests3)
+ * tests/netx/unit/net/sourceforge/jnlp/JNLPMatcherTest.java: (testTemplateCDATA)
+ (testApplicationCDATA)
+ * tests/netx/unit/net/sourceforge/jnlp/ParserCornerCases.java:
+ (testCDataFirstChild) (testCDataSecondChild) (testCommentInAttributes)
+ * tests/netx/unit/net/sourceforge/jnlp/ParserMalformedXml.java:
+ (testMalformedArguments) (testTagNotClosed) (testUnquotedAttributes)
+ marked as KnownToFail
+
2012-06-05 Jiri Vanek <jvanek at redhat.com>
isDateInRange renamed to isDateInRange_internallForIcedTeaWebTesting
diff -r c87cba6608c6 -r 30d29b2698f3 tests/jnlp_tests/signed/CacheReproducer/testcases/CacheReproducerTest.java
--- a/tests/jnlp_tests/signed/CacheReproducer/testcases/CacheReproducerTest.java Tue Jun 05 16:14:42 2012 +0200
+++ b/tests/jnlp_tests/signed/CacheReproducer/testcases/CacheReproducerTest.java Tue Jun 05 16:35:50 2012 +0200
@@ -48,6 +48,7 @@
import java.util.regex.Pattern;
import net.sourceforge.jnlp.ServerAccess;
import net.sourceforge.jnlp.ServerAccess.ProcessResult;
+import net.sourceforge.jnlp.annotations.KnownToFail;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -108,6 +109,7 @@
};
@Test
+ @KnownToFail
public void startParallelInstancesUponBrokenCache() throws Exception {
clearAndEvaluateCache();
evaluateSimpleTest1OkCache(runSimpleTest1());
diff -r c87cba6608c6 -r 30d29b2698f3 tests/junit-runner/JunitLikeXmlOutputListener.java
--- a/tests/junit-runner/JunitLikeXmlOutputListener.java Tue Jun 05 16:14:42 2012 +0200
+++ b/tests/junit-runner/JunitLikeXmlOutputListener.java Tue Jun 05 16:35:50 2012 +0200
@@ -20,7 +20,9 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import net.sourceforge.jnlp.annotations.Bug;
+import net.sourceforge.jnlp.annotations.KnownToFail;
import org.junit.internal.JUnitSystem;
@@ -28,6 +30,7 @@
import org.junit.runner.Result;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunListener;
+
/**
* This class listens for events in junit testsuite and wrote output to xml.
* Xml tryes to follow ant-tests schema, and is enriched for by-class statistics
@@ -45,6 +48,7 @@
private static final String TEST_ELEMENT = "testcase";
private static final String BUGS = "bugs";
private static final String BUG = "bug";
+ private static final String K2F = "known-to-fail";
private static final String TEST_NAME_ATTRIBUTE = "name";
private static final String TEST_TIME_ATTRIBUTE = "time";
private static final String TEST_ERROR_ELEMENT = "error";
@@ -65,15 +69,24 @@
private static final String SUMMARY_IGNORED_ELEMENT = "ignored";
private long testStart;
- private class ClassCounter {
+ private int failedK2F=0;
+ private int passedK2F=0;
+ private int ignoredK2F=0;
+
+ private class ClassStat {
Class c;
int total;
int failed;
int passed;
+ int ignored;
long time = 0;
+ int totalK2F=0;
+ int failedK2F=0;
+ int passedK2F=0;
+ int ignoredK2F=0;
}
- Map<String, ClassCounter> classStats = new HashMap<String, ClassCounter>();
+ Map<String, ClassStat> classStats = new HashMap<String, ClassStat>();
public JunitLikeXmlOutputListener(JUnitSystem system, File f) {
try {
@@ -99,9 +112,11 @@
attString.append(" ");
Set<Entry<String, String>> entries = atts.entrySet();
for (Entry<String, String> entry : entries) {
- String k=entry.getKey();
- String v= entry.getValue();
- if (v==null)v="null";
+ String k = entry.getKey();
+ String v = entry.getValue();
+ if (v == null) {
+ v = "null";
+ }
attString.append(k).append("=\"").append(attributize(v)).append("\"");
attString.append(" ");
}
@@ -137,7 +152,7 @@
@Override
public void testStarted(Description description) throws Exception {
testFailed = null;
- testStart = System.nanoTime() / 1000l / 1000l;
+ testStart = System.nanoTime();
}
@Override
@@ -146,20 +161,54 @@
}
@Override
+ public void testIgnored(Description description) throws Exception {
+ testDone(description, 0, 0, true);
+ }
+
+ @Override
public void testFinished(org.junit.runner.Description description) throws Exception {
- long testTime = System.nanoTime()/1000l/1000l - testStart;
+ long testTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - testStart);
double testTimeSeconds = ((double) testTime) / 1000d;
+ testDone(description, testTime, testTimeSeconds, false);
+ }
- Map<String, String> testcaseAtts = new HashMap<String, String>(3);
+ private void testDone(Description description, long testTime, double testTimeSeconds, boolean ignored) throws Exception {
+ Class testClass = null;
+ Method testMethod = null;
+ try {
+ testClass = description.getTestClass();
+ String qs = description.getMethodName();
+ //handling @Browser'bugsIds marking of used browser
+ if (qs.contains(" - ")) {
+ qs = qs.replaceAll(" - .*", "");
+ }
+ testMethod = testClass.getMethod(qs);
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ Map<String, String> testcaseAtts = new HashMap<String, String>(4);
NumberFormat formatter = new DecimalFormat("#0.0000");
String stringedTime = formatter.format(testTimeSeconds);
stringedTime.replace(",", ".");
testcaseAtts.put(TEST_TIME_ATTRIBUTE, stringedTime);
testcaseAtts.put(TEST_CLASS_ATTRIBUTE, description.getClassName());
testcaseAtts.put(TEST_NAME_ATTRIBUTE, description.getMethodName());
-
+ KnownToFail k2f=null;
+ try {
+ if (testClass != null && testMethod != null) {
+ k2f = testMethod.getAnnotation(KnownToFail.class);
+ if (k2f != null) {
+ testcaseAtts.put(K2F, Boolean.TRUE.toString());
+ }
+ }
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
openElement(TEST_ELEMENT, testcaseAtts);
if (testFailed != null) {
+ if (k2f != null) {
+ failedK2F++;
+ }
Map<String, String> errorAtts = new HashMap<String, String>(3);
errorAtts.put(ERROR_MESSAGE_ATTRIBUTE, testFailed.getMessage());
@@ -171,25 +220,32 @@
}
writeElement(TEST_ERROR_ELEMENT, testFailed.getTrace(), errorAtts);
+ } else {
+ if (k2f != null) {
+ if (ignored) {
+ ignoredK2F++;
+ } else {
+ passedK2F++;
+
+ }
+ }
}
try {
- Class q = description.getTestClass();
- String qs=description.getMethodName();
- if (qs.contains(" - ")) qs=qs.replaceAll(" - .*", "");
- Method qm = q.getMethod(qs);
- Bug b = qm.getAnnotation(Bug.class);
- if (b != null) {
- openElement(BUGS);
- String[] s = b.id();
- for (String string : s) {
- String ss[]=createBug(string);
- Map<String, String> visibleNameAtt=new HashMap<String, String>(1);
- visibleNameAtt.put("visibleName", ss[0]);
- openElement(BUG,visibleNameAtt);
- writer.write(ss[1]);
+ if (testClass != null && testMethod != null) {
+ Bug bug = testMethod.getAnnotation(Bug.class);
+ if (bug != null) {
+ openElement(BUGS);
+ String[] bugsIds = bug.id();
+ for (String bugId : bugsIds) {
+ String idAndUrl[] = createBug(bugId);
+ Map<String, String> visibleNameAtt = new HashMap<String, String>(1);
+ visibleNameAtt.put("visibleName", idAndUrl[0]);
+ openElement(BUG, visibleNameAtt);
+ writer.write(idAndUrl[1]);
closeElement(BUG);
+ }
+ closeElement(BUGS);
}
- closeElement(BUGS);
}
} catch (Exception ex) {
ex.printStackTrace();
@@ -197,19 +253,34 @@
closeElement(TEST_ELEMENT);
writer.flush();
- ClassCounter cc = classStats.get(description.getClassName());
- if (cc == null) {
- cc = new ClassCounter();
- cc.c=description.getTestClass();
- classStats.put(description.getClassName(), cc);
+ ClassStat classStat = classStats.get(description.getClassName());
+ if (classStat == null) {
+ classStat = new ClassStat();
+ classStat.c = description.getTestClass();
+ classStats.put(description.getClassName(), classStat);
}
- cc.total++;
- cc.time += testTime;
+ classStat.total++;
+ if (k2f != null) {
+ classStat.totalK2F++;
+ }
+ classStat.time += testTime;
if (testFailed == null) {
- cc.passed++;
+ if (ignored) {
+ classStat.ignored++;
+ if (k2f != null) {
+ classStat.ignoredK2F++;
+ }
+ } else {
+ classStat.passed++;
+ if (k2f != null) {
+ classStat.passedK2F++;
+ }
+ }
} else {
-
- cc.failed++;
+ classStat.failed++;
+ if (k2f != null) {
+ classStat.failedK2F++;
+ }
}
}
@@ -223,24 +294,24 @@
int passed = result.getRunCount() - result.getFailureCount() - result.getIgnoreCount();
int failed = result.getFailureCount();
int ignored = result.getIgnoreCount();
- writeElement(SUMMARY_TOTAL_ELEMENT, String.valueOf(result.getRunCount()));
- writeElement(SUMMARY_FAILED_ELEMENT, String.valueOf(failed));
- writeElement(SUMMARY_IGNORED_ELEMENT, String.valueOf(ignored));
- writeElement(SUMMARY_PASSED_ELEMENT, String.valueOf(passed));
+ writeElement(SUMMARY_TOTAL_ELEMENT, String.valueOf(result.getRunCount()),createKnownToFailSumamryAttribute(failedK2F+passedK2F+ignoredK2F));
+ writeElement(SUMMARY_FAILED_ELEMENT, String.valueOf(failed),createKnownToFailSumamryAttribute(failedK2F));
+ writeElement(SUMMARY_IGNORED_ELEMENT, String.valueOf(ignored),createKnownToFailSumamryAttribute(ignoredK2F));
+ writeElement(SUMMARY_PASSED_ELEMENT, String.valueOf(passed),createKnownToFailSumamryAttribute(passedK2F));
closeElement(SUMMARY_ELEMENT);
openElement(CLASSES_ELEMENT);
- Set<Entry<String, ClassCounter>> e = classStats.entrySet();
- for (Entry<String, ClassCounter> entry : e) {
+ Set<Entry<String, ClassStat>> e = classStats.entrySet();
+ for (Entry<String, ClassStat> entry : e) {
Map<String, String> testcaseAtts = new HashMap<String, String>(3);
testcaseAtts.put(TEST_NAME_ATTRIBUTE, entry.getKey());
testcaseAtts.put(TEST_TIME_ATTRIBUTE, String.valueOf(entry.getValue().time));
openElement(TEST_CLASS_ELEMENT, testcaseAtts);
- writeElement(SUMMARY_PASSED_ELEMENT, String.valueOf(entry.getValue().passed));
- writeElement(SUMMARY_FAILED_ELEMENT, String.valueOf(entry.getValue().failed));
- writeElement(SUMMARY_IGNORED_ELEMENT, String.valueOf(entry.getValue().total - entry.getValue().failed - entry.getValue().passed));
- writeElement(SUMMARY_TOTAL_ELEMENT, String.valueOf(entry.getValue().total));
+ writeElement(SUMMARY_PASSED_ELEMENT, String.valueOf(entry.getValue().passed),createKnownToFailSumamryAttribute(entry.getValue().passedK2F));
+ writeElement(SUMMARY_FAILED_ELEMENT, String.valueOf(entry.getValue().failed),createKnownToFailSumamryAttribute(entry.getValue().failedK2F));
+ writeElement(SUMMARY_IGNORED_ELEMENT, String.valueOf(entry.getValue().ignored),createKnownToFailSumamryAttribute(entry.getValue().ignoredK2F));
+ writeElement(SUMMARY_TOTAL_ELEMENT, String.valueOf(entry.getValue().total),createKnownToFailSumamryAttribute(entry.getValue().totalK2F));
try {
Bug b = null;
if (entry.getValue().c != null) {
@@ -250,10 +321,10 @@
openElement(BUGS);
String[] s = b.id();
for (String string : s) {
- String ss[]=createBug(string);
- Map<String, String> visibleNameAtt=new HashMap<String, String>(1);
+ String ss[] = createBug(string);
+ Map<String, String> visibleNameAtt = new HashMap<String, String>(1);
visibleNameAtt.put("visibleName", ss[0]);
- openElement(BUG,visibleNameAtt);
+ openElement(BUG, visibleNameAtt);
writer.write(ss[1]);
closeElement(BUG);
}
@@ -273,6 +344,11 @@
}
+ public Map<String, String> createKnownToFailSumamryAttribute(int count) {
+ Map<String, String> atts = new HashMap<String, String>(1);
+ atts.put(K2F, String.valueOf(count));
+ return atts;
+ }
/**
* When declare for suite class or for Test-marked method,
diff -r c87cba6608c6 -r 30d29b2698f3 tests/junit-runner/LessVerboseTextListener.java
--- a/tests/junit-runner/LessVerboseTextListener.java Tue Jun 05 16:14:42 2012 +0200
+++ b/tests/junit-runner/LessVerboseTextListener.java Tue Jun 05 16:35:50 2012 +0200
@@ -6,6 +6,8 @@
* http://www.eclipse.org/legal/cpl-v10.html
*/
import java.io.PrintStream;
+import java.lang.reflect.Method;
+import net.sourceforge.jnlp.annotations.KnownToFail;
import org.junit.internal.JUnitSystem;
import org.junit.runner.Description;
@@ -17,6 +19,10 @@
private PrintStream writer;
private boolean testFailed = false;
+ private int totalK2F=0;
+ private int failedK2F=0;
+ private int passedK2F=0;
+ private int ignoredK2F=0;
public LessVerboseTextListener(JUnitSystem system) {
writer= system.out();
@@ -28,15 +34,24 @@
}
@Override
+ public void testIgnored(Description description) throws Exception {
+ writer.println("Ignored: " + description.getClassName() + "." + description.getMethodName());
+ printK2F(writer, null, description);
+ }
+
+
+ @Override
public void testFailure(Failure failure) {
testFailed = true;
writer.println("FAILED: " + failure.getTestHeader() + " " + failure.getMessage());
+ printK2F(writer,true,failure.getDescription());
}
@Override
public void testFinished(org.junit.runner.Description description) throws Exception {
if (!testFailed) {
writer.println("Passed: " + description.getClassName() + "." + description.getMethodName());
+ printK2F(writer,false,description);
}
}
@@ -45,7 +60,58 @@
int passed = result.getRunCount() - result.getFailureCount() - result.getIgnoreCount();
int failed = result.getFailureCount();
int ignored = result.getIgnoreCount();
+ writer.println("Total tests run: "+result.getRunCount()+"; From those : " + totalK2F + " known to fail");
+ writer.println("Test known to fail: passed: " + passedK2F + "; failed: " + failedK2F + "; ignored: " + ignoredK2F);
writer.println("Test results: passed: " + passed + "; failed: " + failed + "; ignored: " + ignored);
+
+ }
+
+ private void printK2F(PrintStream writer, Boolean failed, Description description) {
+ try {
+ KnownToFail k2f = getK2F(description);
+ if (k2f != null) {
+ totalK2F++;
+ if (failed != null) {
+ if (failed) {
+ failedK2F++;
+ } else {
+ passedK2F++;
+ }
+ } else {
+ ignoredK2F++;
+ }
+ if (failed != null && !failed) {
+ writer.println(" - WARNING This test is known to fail, but have passed!");
+ } else {
+ writer.println(" - This test is known to fail");
+ }
+ }
+
+
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ public static KnownToFail getK2F(Description description) {
+ try {
+ Class q = description.getTestClass();
+ if (q != null) {
+ String qs = description.getMethodName();
+ if (qs.contains(" - ")) {
+ qs = qs.replaceAll(" - .*", "");
+ }
+ Method qm = q.getMethod(qs);
+ if (qm != null) {
+ KnownToFail k2f = qm.getAnnotation(KnownToFail.class);
+ return k2f;
+
+ }
+ }
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ return null;
}
}
diff -r c87cba6608c6 -r 30d29b2698f3 tests/netx/jnlp_testsengine/net/sourceforge/jnlp/annotations/KnownToFail.java
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/netx/jnlp_testsengine/net/sourceforge/jnlp/annotations/KnownToFail.java Tue Jun 05 16:35:50 2012 +0200
@@ -0,0 +1,24 @@
+package net.sourceforge.jnlp.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * <p>
+ * This annotation marks a test as a known failure (as opposed to a
+ * regression). A test that is a known failure will not hold of a release,
+ * nor should developers hold off a fix if they run the unit tests and a
+ * test marked as a known failure fails.
+ * </p><p>
+ * This annotation is meant for adding tests for bugs before the fix is
+ * implemented.
+ * </p>
+ */
+
+ at Target({ElementType.METHOD,ElementType.TYPE})
+ at Retention(RetentionPolicy.RUNTIME)
+public @interface KnownToFail {
+
+}
diff -r c87cba6608c6 -r 30d29b2698f3 tests/netx/unit/net/sourceforge/jnlp/JNLPMatcherTest.java
--- a/tests/netx/unit/net/sourceforge/jnlp/JNLPMatcherTest.java Tue Jun 05 16:14:42 2012 +0200
+++ b/tests/netx/unit/net/sourceforge/jnlp/JNLPMatcherTest.java Tue Jun 05 16:35:50 2012 +0200
@@ -42,6 +42,7 @@
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.Random;
+import net.sourceforge.jnlp.annotations.KnownToFail;
import org.junit.Assert;
import org.junit.Test;
@@ -79,6 +80,7 @@
}
@Test
+ @KnownToFail
public void testTemplateCDATA() throws JNLPMatcherException, IOException {
InputStreamReader launchReader = this.getLaunchReader();
@@ -240,6 +242,7 @@
}
@Test
+ @KnownToFail
public void testApplicationCDATA() throws JNLPMatcherException, IOException {
InputStreamReader launchReader = this.getLaunchReader();
diff -r c87cba6608c6 -r 30d29b2698f3 tests/netx/unit/net/sourceforge/jnlp/ParserCornerCases.java
More information about the distro-pkg-dev
mailing list