diff --git a/prism/src/parser/ast/Property.java b/prism/src/parser/ast/Property.java
index 25ee43cf..3a6eb13d 100644
--- a/prism/src/parser/ast/Property.java
+++ b/prism/src/parser/ast/Property.java
@@ -175,6 +175,7 @@ public class Property extends ASTElement
 	 * (As required for {@link #checkAgainstExpectedResult(Object)} and {@link #checkAgainstExpectedResult(Object, String)}) 
 	 * @param strExpected Expected result
 	 * @param result The actual result
+	 * @return Whether or not the check was performed
 	 */
 	private boolean checkAgainstExpectedResultString(String strExpected, Object result) throws PrismException
 	{
@@ -183,6 +184,23 @@ public class Property extends ASTElement
 			return false;
 		}
 		
+		// Check for exceptions
+		if (result instanceof Exception) {
+			String errMsg = ((Exception) result).getMessage();
+			if (strExpected.startsWith("Error")) {
+				if (strExpected.startsWith("Error:")) {
+					String words[] = strExpected.substring(6).split(",");
+					for (String word : words) {
+						if (!errMsg.contains(word)) {
+							throw new PrismException("Error message should contain \"" + word + "\"");
+						}
+					}
+				}
+				return true;
+			}
+			throw new PrismException("Unexpected error: " + errMsg);
+		}
+		
 		// Check expected/actual result
 		Type type = expr.getType();
 		
diff --git a/prism/src/prism/PrismCL.java b/prism/src/prism/PrismCL.java
index 271ba850..b99db26b 100644
--- a/prism/src/prism/PrismCL.java
+++ b/prism/src/prism/PrismCL.java
@@ -293,7 +293,7 @@ public class PrismCL implements PrismModelListener
 							}
 						} catch (PrismException e) {
 							// in case of error, report it, store exception as the result and proceed
-							error(e.getMessage());
+							error(e.getMessage(), true);
 							res = new Result(e);
 						}
 
@@ -1720,9 +1720,18 @@ public class PrismCL implements PrismModelListener
 	 * Report a (non-fatal) error to the log.
 	 */
 	private void error(String s)
+	{
+		error(s, false);
+	}
+
+	/**
+	 * Report a (non-fatal) error to the log.
+	 * Optionally, requested that we do not exit, even if test mode is enabled
+	 */
+	private void error(String s, boolean dontExit)
 	{
 		// If (and only if) we are in "test" (and not "testall") mode, treat any error as fatal
-		if (test && testExitsOnFail) {
+		if (test && testExitsOnFail && !dontExit) {
 			errorAndExit(s);
 		}
 		// Normal case: just display error message, but don't exit