Skip to content
Snippets Groups Projects
Commit 3bc9d084 authored by Aymeric Varasse's avatar Aymeric Varasse :innocent:
Browse files

[aimos] Create config file from improved predicate

parent 7f90b052
No related branches found
No related tags found
No related merge requests found
...@@ -4,5 +4,5 @@ theory ACASXU_P5 ...@@ -4,5 +4,5 @@ theory ACASXU_P5
use caisar.DatasetClassification use caisar.DatasetClassification
use caisar.DatasetClassificationProps use caisar.DatasetClassificationProps
goal G: meta_robust model dataset (1.0:t) goal G: meta_robust model dataset (1.0:t) ("reluplex_rotation":string) (0:int) (0:int) (0:int)
end end
...@@ -25,16 +25,71 @@ open Base ...@@ -25,16 +25,71 @@ open Base
let aimos_file = Re__Core.(compile (str "%{aimos_file}")) let aimos_file = Re__Core.(compile (str "%{aimos_file}"))
let build_command config_prover aimos_filename = let write_config (inputs_path : string) (models_path : string)
(perturbation : string) (amplitude : Dataset.amplitude) =
let config_file = Caml.Filename.temp_file "aimos-" ".yml" in
let config_path = Fpath.v config_file in
let ampli = Dataset.string_of_amplitude amplitude in
let transformations : Yaml.value =
match ampli with
| None -> `A [ `O [ ("name", `String perturbation) ] ]
| Some a ->
`A [ `O [ ("name", `String perturbation); ("fn_range", `String a) ] ]
in
let options : Yaml.value =
`O
[
("plot", `Bool false);
("inputs_path", `String inputs_path);
("transformations", transformations);
("custom_t_path", `String "config/custom_transformations.py");
]
in
let models : Yaml.value =
`A
[
`O
[
( "defaults",
`O
[
("models_path", `String models_path);
("out_mode", `String "classif_min");
] );
];
]
in
let full_config : Yaml.value =
`O [ ("options", options); ("models", models) ]
in
Yaml_unix.to_file_exn config_path full_config;
config_file
let build_command config_prover
(predicate : (Language.nn_shape, string) Dataset.predicate) =
let dataset = predicate.dataset in
let inputs_path, models_path =
(Unix.realpath dataset, Unix.realpath predicate.model.filename)
in
let perturbation, (amplitude : Dataset.amplitude) =
match predicate.property with
| Dataset.MetaRobust (_, p, start, stop, step) ->
(p, { start = Some start; stop; step = Some step })
| _ -> failwith "Unsupported property"
in
let aimos_config =
write_config inputs_path models_path perturbation amplitude
in
let command = Whyconf.get_complete_command ~with_steps:false config_prover in let command = Whyconf.get_complete_command ~with_steps:false config_prover in
Re__Core.replace_string aimos_file ~by:aimos_filename command Re__Core.replace_string aimos_file ~by:aimos_config command
let re_aimos_output = Re__Pcre.regexp "((,\\s)(\\d+\\.\\d+))" let re_aimos_output = Re__Pcre.regexp "((,\\s)(\\d+\\.\\d+))"
let build_answer predicate_kind prover_result = let build_answer predicate_kind prover_result =
let threshold = let threshold =
match predicate_kind with match predicate_kind with
| Dataset.MetaRobust f -> Float.of_string (Dataset.string_of_threshold f) | Dataset.MetaRobust (f, _, _, _, _) ->
Float.of_string (Dataset.string_of_threshold f)
| _ -> failwith "Unsupported property" | _ -> failwith "Unsupported property"
in in
match prover_result.Call_provers.pr_answer with match prover_result.Call_provers.pr_answer with
...@@ -61,8 +116,8 @@ let build_answer predicate_kind prover_result = ...@@ -61,8 +116,8 @@ let build_answer predicate_kind prover_result =
assert false assert false
let call_prover limit config config_prover let call_prover limit config config_prover
(predicate : (Language.nn_shape, string) Dataset.predicate) aimos_config = (predicate : (Language.nn_shape, string) Dataset.predicate) =
let command = build_command config_prover aimos_config in let command = build_command config_prover predicate in
let prover_call = let prover_call =
let res_parser = let res_parser =
{ {
......
...@@ -27,5 +27,4 @@ val call_prover : ...@@ -27,5 +27,4 @@ val call_prover :
Whyconf.main -> Whyconf.main ->
Whyconf.config_prover -> Whyconf.config_prover ->
(Language.nn_shape, string) Dataset.predicate -> (Language.nn_shape, string) Dataset.predicate ->
string ->
Call_provers.prover_answer Call_provers.prover_answer
...@@ -51,13 +51,36 @@ let term_of_eps env eps = ...@@ -51,13 +51,36 @@ let term_of_eps env eps =
type threshold = float [@@deriving yojson, show] type threshold = float [@@deriving yojson, show]
type amplitude = {
start : int option;
stop : int;
step : int option;
}
[@@deriving yojson]
let string_of_threshold threshold = Float.to_string threshold let string_of_threshold threshold = Float.to_string threshold
let string_of_amplitude amplitude =
let start = match amplitude.start with Some a -> a | None -> 0 in
if start = amplitude.stop
then None
else
let start_str =
Option.value_map amplitude.start ~default:"" ~f:Int.to_string
in
let step_str =
Option.value_map amplitude.step ~default:"" ~f:Int.to_string
in
Some
(Fmt.str "range(%s, %s, %s)" start_str
(Int.to_string amplitude.stop)
step_str)
type property = type property =
| Correct | Correct
| Robust of eps | Robust of eps
| CondRobust of eps | CondRobust of eps
| MetaRobust of threshold | MetaRobust of threshold * string * int * int * int
[@@deriving yojson, show] [@@deriving yojson, show]
type ('a, 'b) predicate = { type ('a, 'b) predicate = {
...@@ -82,6 +105,7 @@ let failwith_unsupported_term t = ...@@ -82,6 +105,7 @@ let failwith_unsupported_term t =
let failwith_unsupported_ls ls = let failwith_unsupported_ls ls =
failwith (Fmt.str "Unsupported logic symbol %a" Pretty.print_ls ls) failwith (Fmt.str "Unsupported logic symbol %a" Pretty.print_ls ls)
(* TODO: use amplitude type instead of 3 ints *)
let interpret_predicate env ~on_model ~on_dataset task = let interpret_predicate env ~on_model ~on_dataset task =
let task = Trans.apply Introduction.introduce_premises task in let task = Trans.apply Introduction.introduce_premises task in
let term = Task.task_goal_fmla task in let term = Task.task_goal_fmla task in
...@@ -101,14 +125,29 @@ let interpret_predicate env ~on_model ~on_dataset task = ...@@ -101,14 +125,29 @@ let interpret_predicate env ~on_model ~on_dataset task =
| [ { t_node = Tconst (Constant.ConstReal e); _ } ] -> | [ { t_node = Tconst (Constant.ConstReal e); _ } ] ->
let robust_predicate = find_predicate_ls env "robust" in let robust_predicate = find_predicate_ls env "robust" in
let cond_robust_predicate = find_predicate_ls env "cond_robust" in let cond_robust_predicate = find_predicate_ls env "cond_robust" in
let meta_robust_predicate = find_predicate_ls env "meta_robust" in
let f = float_of_real_constant e in let f = float_of_real_constant e in
if Term.ls_equal ls robust_predicate if Term.ls_equal ls robust_predicate
then Robust f then Robust f
else if Term.ls_equal ls cond_robust_predicate else if Term.ls_equal ls cond_robust_predicate
then CondRobust f then CondRobust f
else if Term.ls_equal ls meta_robust_predicate else failwith_unsupported_ls ls
then MetaRobust f | [
{ t_node = Tconst (Constant.ConstReal e); _ };
{ t_node = Tconst (Constant.ConstStr p); _ };
{ t_node = Tconst (Constant.ConstInt start); _ };
{ t_node = Tconst (Constant.ConstInt stop); _ };
{ t_node = Tconst (Constant.ConstInt step); _ };
] ->
let meta_robust_predicate = find_predicate_ls env "meta_robust" in
let f = float_of_real_constant e in
if Term.ls_equal ls meta_robust_predicate
then
MetaRobust
( f,
p,
Number.to_small_integer start,
Number.to_small_integer stop,
Number.to_small_integer step )
else failwith_unsupported_ls ls else failwith_unsupported_ls ls
| _ -> failwith_unsupported_term term | _ -> failwith_unsupported_term term
in in
......
...@@ -30,13 +30,21 @@ val term_of_eps : Env.env -> eps -> Term.term ...@@ -30,13 +30,21 @@ val term_of_eps : Env.env -> eps -> Term.term
type threshold [@@deriving yojson, show] type threshold [@@deriving yojson, show]
type amplitude = {
start : int option;
stop : int;
step : int option;
}
[@@deriving yojson]
val string_of_threshold : threshold -> string val string_of_threshold : threshold -> string
val string_of_amplitude : amplitude -> string option
type property = private type property = private
| Correct | Correct
| Robust of eps | Robust of eps
| CondRobust of eps | CondRobust of eps
| MetaRobust of threshold | MetaRobust of threshold * string * int * int * int
[@@deriving yojson, show] [@@deriving yojson, show]
type ('model, 'dataset) predicate = private { type ('model, 'dataset) predicate = private {
......
...@@ -123,7 +123,7 @@ let answer_saver limit config env config_prover dataset task = ...@@ -123,7 +123,7 @@ let answer_saver limit config env config_prover dataset task =
let additional_info = Fmt.str "(%d/%d)" answer.nb_proved answer.nb_total in let additional_info = Fmt.str "(%d/%d)" answer.nb_proved answer.nb_total in
(prover_answer, Generic (Some additional_info)) (prover_answer, Generic (Some additional_info))
let answer_aimos limit config env config_prover dataset task aimos_config = let answer_aimos limit config env config_prover dataset task =
let predicate = let predicate =
let on_model ls = let on_model ls =
Option.value_or_thunk Option.value_or_thunk
...@@ -145,17 +145,7 @@ let answer_aimos limit config env config_prover dataset task aimos_config = ...@@ -145,17 +145,7 @@ let answer_aimos limit config env config_prover dataset task aimos_config =
in in
Dataset.interpret_predicate env ~on_model ~on_dataset task Dataset.interpret_predicate env ~on_model ~on_dataset task
in in
let aimos_filename = let answer = AIMOS.call_prover limit config config_prover predicate in
match aimos_config with
| None ->
List.map Dirs.Sites.config ~f:(fun dir ->
Caml.Filename.concat dir "aimos_config.yml")
|> List.find_exn ~f:Caml.Sys.file_exists
| Some s -> s
in
let answer =
AIMOS.call_prover limit config config_prover predicate aimos_filename
in
let additional_info = Generic None in let additional_info = Generic None in
(answer, additional_info) (answer, additional_info)
...@@ -262,9 +252,7 @@ let call_prover ?dataset ~limit config env prover config_prover driver task = ...@@ -262,9 +252,7 @@ let call_prover ?dataset ~limit config env prover config_prover driver task =
let prover_answer, additional_info = let prover_answer, additional_info =
match prover with match prover with
| Prover.Saver -> answer_saver limit config env config_prover dataset task | Prover.Saver -> answer_saver limit config env config_prover dataset task
| Aimos -> | Aimos -> answer_aimos limit config env config_prover dataset task
(* TODO: add real config file *)
answer_aimos limit config env config_prover dataset task None
| (Marabou | Pyrat | Nnenum) when Option.is_some dataset -> | (Marabou | Pyrat | Nnenum) when Option.is_some dataset ->
let dataset = Unix.realpath (Option.value_exn dataset) in let dataset = Unix.realpath (Option.value_exn dataset) in
answer_dataset limit config env prover config_prover driver dataset task answer_dataset limit config env prover config_prover driver dataset task
......
...@@ -24,11 +24,14 @@ theory DatasetClassification ...@@ -24,11 +24,14 @@ theory DatasetClassification
use ieee_float.Float64 use ieee_float.Float64
use int.Int use int.Int
use array.Array use array.Array
use option.Option
type features = array t type features = array t
type label_ = int type label_ = int
type record = (features, label_) type record = (features, label_)
type dataset = array record type dataset = array record
type ampli = { start: option int; stop: int; step: option int; }
type amplitude = option ampli
constant dataset: dataset constant dataset: dataset
...@@ -69,7 +72,8 @@ theory DatasetClassificationProps ...@@ -69,7 +72,8 @@ theory DatasetClassificationProps
predicate cond_robust (m: model) (d: dataset) (eps: t) = predicate cond_robust (m: model) (d: dataset) (eps: t) =
correct m d /\ robust m d eps correct m d /\ robust m d eps
predicate meta_robust (m: model) (d: dataset) (threshold: t) predicate meta_robust (m: model) (d: dataset) (threshold: t) (perturbation: string) (start: int) (stop: int) (step: int)
(* predicate meta_robust (m: model) (d: dataset) (threshold: t) (perturbation: string) (ampli: amplitude) *)
end end
theory NN theory NN
......
...@@ -19,7 +19,7 @@ Test verify ...@@ -19,7 +19,7 @@ Test verify
> use caisar.DatasetClassification > use caisar.DatasetClassification
> use caisar.DatasetClassificationProps > use caisar.DatasetClassificationProps
> >
> goal G: meta_robust model dataset (1.0:t) > goal G: meta_robust model dataset (1.0:t) ("reluplex_rotation":string) (0:int) (0:int) (0:int)
> end > end
> EOF > EOF
[caisar] Goal G: Valid [caisar] Goal G: Valid
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment