From 2bcb5a58b7365d0477c34694e7ef8bbf6c700123 Mon Sep 17 00:00:00 2001
From: Andre Maroneze <andre.maroneze@cea.fr>
Date: Thu, 3 Feb 2022 22:41:03 +0100
Subject: [PATCH] [tests] fc_script: allow scripts to detect test mode
Some scripts use features which cannot or should not be tested
(e.g. reliance on an optional external tool); defining PTESTS_TESTING=1
allows these scripts to detect such circumstances and change (slightly)
their behavior to avoid non-reproducible oracles.
---
share/analysis-scripts/build.py | 2 ++
share/analysis-scripts/build_callgraph.py | 2 ++
share/analysis-scripts/estimate_difficulty.py | 2 ++
tests/fc_script/build-callgraph.i | 2 +-
tests/fc_script/list_functions.i | 2 +-
tests/fc_script/main.c | 14 +++++++-------
tests/fc_script/make-wrapper.c | 2 +-
tests/fc_script/recursions.i | 2 +-
8 files changed, 17 insertions(+), 11 deletions(-)
diff --git a/share/analysis-scripts/build.py b/share/analysis-scripts/build.py
index 58fa55f9b53..88a0762d0d2 100755
--- a/share/analysis-scripts/build.py
+++ b/share/analysis-scripts/build.py
@@ -94,6 +94,8 @@ if not framac_bin:
sys.exit("error: FRAMAC_BIN not in environment (set by frama-c-script)")
framac_bin = Path(framac_bin)
+under_test = os.getenv("PTESTS_TESTING")
+
# Prepare blug-related variables and functions ################################
blug = os.getenv('BLUG')
diff --git a/share/analysis-scripts/build_callgraph.py b/share/analysis-scripts/build_callgraph.py
index 8600390f536..6ab8ab2ab6b 100755
--- a/share/analysis-scripts/build_callgraph.py
+++ b/share/analysis-scripts/build_callgraph.py
@@ -31,6 +31,8 @@ import sys
import function_finder
+under_test = os.getenv("PTESTS_TESTING")
+
arg = ""
if len(sys.argv) < 2:
print(f"usage: {sys.argv[0]} file...")
diff --git a/share/analysis-scripts/estimate_difficulty.py b/share/analysis-scripts/estimate_difficulty.py
index 5926d52a0c9..e7549fefd30 100755
--- a/share/analysis-scripts/estimate_difficulty.py
+++ b/share/analysis-scripts/estimate_difficulty.py
@@ -54,6 +54,8 @@ if not header_dirs:
header_dirs = []
files = args["files"]
+under_test = os.getenv("PTESTS_TESTING")
+
# gather information from several sources
def extract_keys(l):
diff --git a/tests/fc_script/build-callgraph.i b/tests/fc_script/build-callgraph.i
index 7ec5a976aeb..be8f7b96630 100644
--- a/tests/fc_script/build-callgraph.i
+++ b/tests/fc_script/build-callgraph.i
@@ -1,6 +1,6 @@
/* run.config
NOFRAMAC: testing frama-c-script, not frama-c itself
- EXECNOW: LOG build-callgraph.res LOG build-callgraph.err %{bin:frama-c-script} heuristic-print-callgraph @PTEST_DIR@/@PTEST_NAME@.i > @PTEST_RESULT@/build-callgraph.res 2> @PTEST_RESULT@/build-callgraph.err
+ EXECNOW: LOG build-callgraph.res LOG build-callgraph.err PTESTS_TESTING=1 %{bin:frama-c-script} heuristic-print-callgraph @PTEST_DIR@/@PTEST_NAME@.i > @PTEST_RESULT@/build-callgraph.res 2> @PTEST_RESULT@/build-callgraph.err
*/
#include <stdio.h>
diff --git a/tests/fc_script/list_functions.i b/tests/fc_script/list_functions.i
index 448f97a3090..225f70d0a90 100644
--- a/tests/fc_script/list_functions.i
+++ b/tests/fc_script/list_functions.i
@@ -15,5 +15,5 @@
DEPS: @PTEST_DEPS@ @PTEST_DIR@/build-callgraph.i
DEPS: @PTEST_DEPS@ @PTEST_DIR@/recursions.i
- EXECNOW: LOG heuristic_list_functions.res LOG heuristic_list_functions.err %{bin:frama-c-script} heuristic-list-functions true true @PTEST_DEPS@ > @PTEST_RESULT@/heuristic_list_functions.res 2> @PTEST_RESULT@/heuristic_list_functions.err
+ EXECNOW: LOG heuristic_list_functions.res LOG heuristic_list_functions.err PTESTS_TESTING=1 %{bin:frama-c-script} heuristic-list-functions true true @PTEST_DEPS@ > @PTEST_RESULT@/heuristic_list_functions.res 2> @PTEST_RESULT@/heuristic_list_functions.err
*/
diff --git a/tests/fc_script/main.c b/tests/fc_script/main.c
index d6526f2b234..9a89c3f67b2 100644
--- a/tests/fc_script/main.c
+++ b/tests/fc_script/main.c
@@ -1,16 +1,16 @@
/* run.config
NOFRAMAC: testing frama-c-script, not frama-c itself
DEPS: for-find-fun2.c for-find-fun.c main.c main2.c main3.c
- EXECNOW: LOG GNUmakefile LOG make_template.res LOG make_template.err PTESTS_TESTING= %{bin:frama-c-script} -C @PTEST_DIR@ make-template $(basename @PTEST_RESULT@) < %{dep:@PTEST_DIR@/make_template.input} > @PTEST_RESULT@/make_template.res 2> @PTEST_RESULT@/make_template.err
+ EXECNOW: LOG GNUmakefile LOG make_template.res LOG make_template.err PTESTS_TESTING=1 %{bin:frama-c-script} -C @PTEST_DIR@ make-template $(basename @PTEST_RESULT@) < %{dep:@PTEST_DIR@/make_template.input} > @PTEST_RESULT@/make_template.res 2> @PTEST_RESULT@/make_template.err
DEPS: main2.c main3.c main.c
- EXECNOW: LOG list_files.res LOG list_files.err %{bin:frama-c-script} list-files %{dep:@PTEST_DIR@/list_files.json} > @PTEST_RESULT@/list_files.res 2> @PTEST_RESULT@/list_files.err
+ EXECNOW: LOG list_files.res LOG list_files.err PTESTS_TESTING=1 %{bin:frama-c-script} list-files %{dep:@PTEST_DIR@/list_files.json} > @PTEST_RESULT@/list_files.res 2> @PTEST_RESULT@/list_files.err
DEPS: for-find-fun2.c for-find-fun.c for-list-functions.c main2.c main3.c main.c make-wrapper2.c make-wrapper3.c make-wrapper.c
- EXECNOW: LOG find_fun1.res LOG find_fun1.err %{bin:frama-c-script} find-fun main2 @PTEST_DIR@ > @PTEST_RESULT@/find_fun1.res 2> @PTEST_RESULT@/find_fun1.err
- EXECNOW: LOG find_fun2.res LOG find_fun2.err %{bin:frama-c-script} find-fun main3 @PTEST_DIR@ > @PTEST_RESULT@/find_fun2.res 2> @PTEST_RESULT@/find_fun2.err
- EXECNOW: LOG find_fun3.res LOG find_fun3.err %{bin:frama-c-script} find-fun false_positive @PTEST_DIR@ > @PTEST_RESULT@/find_fun3.res 2> @PTEST_RESULT@/find_fun3.err
+ EXECNOW: LOG find_fun1.res LOG find_fun1.err PTESTS_TESTING=1 %{bin:frama-c-script} find-fun main2 @PTEST_DIR@ > @PTEST_RESULT@/find_fun1.res 2> @PTEST_RESULT@/find_fun1.err
+ EXECNOW: LOG find_fun2.res LOG find_fun2.err PTESTS_TESTING=1 %{bin:frama-c-script} find-fun main3 @PTEST_DIR@ > @PTEST_RESULT@/find_fun2.res 2> @PTEST_RESULT@/find_fun2.err
+ EXECNOW: LOG find_fun3.res LOG find_fun3.err PTESTS_TESTING=1 %{bin:frama-c-script} find-fun false_positive @PTEST_DIR@ > @PTEST_RESULT@/find_fun3.res 2> @PTEST_RESULT@/find_fun3.err
DEPS:
- EXECNOW: LOG list_functions.res LOG list_functions.err %{bin:frama-c-script} list-functions %{dep:@PTEST_DIR@/for-find-fun2.c} %{dep:@PTEST_DIR@/for-list-functions.c} > @PTEST_RESULT@/list_functions.res 2> @PTEST_RESULT@/list_functions.err
- EXECNOW: LOG list_functions2.res LOG list_functions2.err LOG list_functions2.json %{bin:frama-c-script} list-functions %{dep:@PTEST_DIR@/for-find-fun2.c} %{dep:@PTEST_DIR@/for-list-functions.c -list-functions-declarations} -list-functions-output @PTEST_RESULT@/list_functions2.json -list-functions-debug 1 > @PTEST_RESULT@/list_functions2.res 2> @PTEST_RESULT@/list_functions2.err
+ EXECNOW: LOG list_functions.res LOG list_functions.err PTESTS_TESTING=1 %{bin:frama-c-script} list-functions %{dep:@PTEST_DIR@/for-find-fun2.c} %{dep:@PTEST_DIR@/for-list-functions.c} > @PTEST_RESULT@/list_functions.res 2> @PTEST_RESULT@/list_functions.err
+ EXECNOW: LOG list_functions2.res LOG list_functions2.err LOG list_functions2.json PTESTS_TESTING=1 %{bin:frama-c-script} list-functions %{dep:@PTEST_DIR@/for-find-fun2.c} %{dep:@PTEST_DIR@/for-list-functions.c -list-functions-declarations} -list-functions-output @PTEST_RESULT@/list_functions2.json -list-functions-debug 1 > @PTEST_RESULT@/list_functions2.res 2> @PTEST_RESULT@/list_functions2.err
*/
diff --git a/tests/fc_script/make-wrapper.c b/tests/fc_script/make-wrapper.c
index 89700bc4aa9..0f7d0709e6e 100644
--- a/tests/fc_script/make-wrapper.c
+++ b/tests/fc_script/make-wrapper.c
@@ -2,7 +2,7 @@
MACRO: RM_TMP_DIR rm -rf make-for-make-wrapper.parse make-for-make-wrapper.eva
NOFRAMAC: testing frama-c-script
COMMENT: in case of errors, remove the 'grep' part to get the full output
- EXECNOW: LOG make-wrapper.res LOG make-wrapper.err (cd @PTEST_DIR@ && touch make-wrapper2.c && touch make-wrapper3.c && @RM_TMP_DIR@ && FRAMAC=%{bin:frama-c} %{bin:frama-c-script} make-wrapper --make-dir . -f make-for-make-wrapper.mk | grep -A999999 "make-wrapper recommendations" && @RM_TMP_DIR@) > @PTEST_RESULT@/make-wrapper.res 2> @PTEST_RESULT@/make-wrapper.err
+ EXECNOW: LOG make-wrapper.res LOG make-wrapper.err (cd @PTEST_DIR@ && touch make-wrapper2.c && touch make-wrapper3.c && @RM_TMP_DIR@ && FRAMAC=%{bin:frama-c} PTESTS_TESTING=1 %{bin:frama-c-script} make-wrapper --make-dir . -f make-for-make-wrapper.mk | grep -A999999 "make-wrapper recommendations" && @RM_TMP_DIR@) > @PTEST_RESULT@/make-wrapper.res 2> @PTEST_RESULT@/make-wrapper.err
*/
int defined(int a);
diff --git a/tests/fc_script/recursions.i b/tests/fc_script/recursions.i
index 7d7cbfe0daa..1d1f993cf17 100644
--- a/tests/fc_script/recursions.i
+++ b/tests/fc_script/recursions.i
@@ -1,6 +1,6 @@
/* run.config
NOFRAMAC: testing frama-c-script, not frama-c itself
- EXECNOW: LOG recursions.res LOG recursions.err %{bin:frama-c-script} heuristic-detect-recursion @PTEST_FILE@ > @PTEST_RESULT@/recursions.res 2> @PTEST_RESULT@/recursions.err
+ EXECNOW: LOG recursions.res LOG recursions.err PTESTS_TESTING=1 %{bin:frama-c-script} heuristic-detect-recursion @PTEST_FILE@ > @PTEST_RESULT@/recursions.res 2> @PTEST_RESULT@/recursions.err
*/
volatile int v;
--
GitLab