dd_matrix.cc: fix recently introduced error handling

+ some whitespace cleanup


git-svn-id: https://www.prismmodelchecker.org/svn/prism/prism/trunk@10493 bbc10eb1-c90d-0410-af57-cb519fbb1720

prism/src/dd/dd_matrix.cc

@@ -29,6 +29,7 @@
 #include <stdlib.h>
 #include <math.h>
 #include <util.h>
+#include <limits>  // for NaN value
 #include <cudd.h>
 #include <cuddInt.h>
 #include "dd_matrix.h"
@@ -263,7 +264,10 @@ long x
 	int *inputs;
 	double val;
 
-	if (dd == NULL) return NULL;
+	if (dd == NULL) {
+		DD_SetErrorFlag();
+		return std::numeric_limits<double>::quiet_NaN();
+	}
 
 	// create array to store 0's & 1's used to query DD
 	inputs = new int[Cudd_ReadSize(ddman)];
@@ -278,7 +282,7 @@ long x
 	node = Cudd_Eval(ddman, dd, inputs);
 	if (node == NULL) {
 		DD_SetErrorFlag();
-		return 0;
+		return std::numeric_limits<double>::quiet_NaN();
 	}
 	val = Cudd_V(node);