From 4b77dd2c510498c1589b50c07696017ee4d5dad4 Mon Sep 17 00:00:00 2001 From: Dave Parker Date: Sat, 28 Mar 2020 13:36:33 +0000 Subject: [PATCH] prism-auto: Add basic filtering of benchmarks using models.csv metadata. New switch --filter-models X which restricts the list of models used from a directory to those that match the filter X. Currently, this can refer to the number of states and/or the model type. Examples: * prism-auto . --filter-models "states>100 and states<10000" * prism-auto . --filter-models "model_type=='DTMC'" * prism-auto . --filter-models "'MC' in model_type" The model metadata is by default read in from a models.csv file (as found, for example, in the PRISM benchmark suite). The name of the file used can be changed with --models-info FILE. --- prism/etc/scripts/prism-auto | 41 +++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/prism/etc/scripts/prism-auto b/prism/etc/scripts/prism-auto index 0f073548..18156910 100755 --- a/prism/etc/scripts/prism-auto +++ b/prism/etc/scripts/prism-auto @@ -11,7 +11,7 @@ # Run "prism-auto -h" for details of further options. -import os,sys,re,subprocess,signal,tempfile,functools,logging,time,platform +import os,sys,re,subprocess,signal,tempfile,functools,logging,time,platform,csv from pipes import quote from optparse import OptionParser from threading import Timer @@ -143,8 +143,45 @@ def getModelsInDir(dir): if os.path.isfile(os.path.join(dir, file)) and isPrismModelFile(file): modelFiles.append((os.path.join(dir, file), [])) #print "Model files in " + dir + ": " + ' '.join( map(lambda pair: pair[0], modelFiles )) + + # If requested, filter the list of models + if options.filterModels: + modelFiles = filterModels(modelFiles, options.filterModels, dir) + return modelFiles +# Restrict a list of models to those satisfying a filter expressed as a string +# Examples: "states>100 and states<10000" or "model_type=='DTMC'" +# Model list is given as pairs of files and argument lists (see getModelsInDir() above) +# Model data is read in from a file models.csv (or something else if specified +# using options.modelsInfoFilename) + +def filterModels(modelFiles, filterString, dir): + modelFilesNew = [] + # Filter is false if the model metadata cannot be found + if os.path.isfile(os.path.join(dir, options.modelsInfoFilename)): + with open(os.path.join(dir, options.modelsInfoFilename), newline='') as csvfile: + # Build a list of models matching the filter + modelFilesFiltered = [] + reader = csv.DictReader(csvfile) + for row in reader: + # Evaluate the filter using eval() + # Only provide access to the model type and number of states for now + # If any fields are missing, the filter evaluates to false + filterVarsAvailable = ['model_type', 'states'] + filterVarsMissing = [key for key in filterVarsAvailable if key in filterString and not row[key]] + if not filterVarsMissing: + if eval(filterString, {"__builtins__": {}}, {'model_type': row['model_type'], 'states': int(row['states'])}): + model = os.path.join(dir, row['model_file']) + if 'model_consts' in row: + model += ' -const ' + row['model_consts'] + modelFilesFiltered.append(model) + # Restrict models list to ones in the list of filter matches + for modelFile in modelFiles: + if (modelFile[0] + ' ' + ' '.join(modelFile[1])).strip() in modelFilesFiltered: + modelFilesNew.append(modelFile) + return modelFilesNew; + # Get a list of all files in the directory that satisfy the given predicate def getFilesInDir(dir, pred): @@ -1031,6 +1068,8 @@ parser.add_option("--no-renaming", action="store_true", dest="noRenaming", defau parser.add_option("--debug", action="store_true", dest="debug", default=False, help="Enable debug mode: display debugging info") parser.add_option("--echo-full", action="store_true", dest="echoFull", default=False, help="An expanded version of -e/--echo") parser.add_option("--models-filename", dest="modelsFilename", metavar="X", default="models", help="Read in list of models/parameters for a directory from file X, if present [default=models]") +parser.add_option("--models-info", dest="modelsInfoFilename", metavar="X", default="models.csv", help="Read model details from CSV file X, if present [default=models.csv]") +parser.add_option("--filter-models", dest="filterModels", metavar="N", default="", help="Filter benchmark models...") parser.add_option("--log-subdirs", action="store_true", dest="logSubdirs" ,default=False, help="Organise PRISM output logs in subdirectories per benchmark argument") parser.add_option("--no-export-tests", action="store_true", dest="noExportTests", default=False, help="Don't check exported files when in test mode") parser.add_option("--skip-export-runs", action="store_true", dest="skipExportRuns", default=False, help="Skip all runs having exports")