diff --git a/share/analysis-scripts/benchmark_database.py b/share/analysis-scripts/benchmark_database.py index ccce93970efa9ceac1bb1a9772795c9207c3256c..f73e1b08d5a037b1b7a1d742bd884bacc9501b36 100644 --- a/share/analysis-scripts/benchmark_database.py +++ b/share/analysis-scripts/benchmark_database.py @@ -35,7 +35,7 @@ def dict_factory(cursor, row): class Database: - inserted_targets = {} + inserted_targets : dict[str, bool] = {} def __init__(self, benchmark_tag, benchmark_comment, gitdir, analyzer_rev, reference_rev): self.benchmark_tag = benchmark_tag diff --git a/share/analysis-scripts/build.py b/share/analysis-scripts/build.py index f392d8a1913cfc57e2bc4e02b2df66945598f5e4..629c50a897daae8f4468dcdd302dcb14f1e53c54 100755 --- a/share/analysis-scripts/build.py +++ b/share/analysis-scripts/build.py @@ -40,6 +40,8 @@ import subprocess import function_finder import source_filter +from typing import Callable, TypeVar + script_dir = os.path.dirname(sys.argv[0]) # Command-line parsing ######################################################## @@ -114,21 +116,21 @@ dot_framac_dir = Path(".frama-c") # Check required environment variables and commands in the PATH ############### -framac_bin = os.getenv("FRAMAC_BIN") -if not framac_bin: +framac_bin_str = os.getenv("FRAMAC_BIN") +if not framac_bin_str: sys.exit("error: FRAMAC_BIN not in environment (set by frama-c-script)") -framac_bin = Path(framac_bin) +framac_bin = Path(framac_bin_str) under_test = os.getenv("PTESTS_TESTING") # Prepare blug-related variables and functions ################################ -blug = os.getenv("BLUG") -if not blug: - blug = shutil.which("blug") - if not blug: +blug_str = os.getenv("BLUG") +if not blug_str: + blug_str = shutil.which("blug") + if not blug_str: sys.exit("error: path to 'blug' binary must be in PATH or variable BLUG") -blug = Path(blug) +blug = Path(blug_str) blug_dir = blug.resolve().parent # to import blug_jbdb sys.path.insert(0, blug_dir.as_posix()) @@ -139,20 +141,20 @@ from blug_jbdb import prettify # Auxiliary functions ######################################################### -def call_and_get_output(command_and_args): +def call_and_get_output(command: Path, args: list[str]) -> str: try: - return subprocess.check_output(command_and_args, stderr=subprocess.STDOUT).decode() + return subprocess.check_output([str(command)] + args, stderr=subprocess.STDOUT).decode() except subprocess.CalledProcessError as e: - sys.exit(f"error running command: {command_and_args}\n{e}") + sys.exit(f"error running command: {command} {args}\n{e}") -def ask_if_overwrite(path): +def ask_if_overwrite(path: Path) -> None: yn = input(f"warning: {path} already exists. Overwrite? [y/N] ") if yn == "" or not (yn[0] == "Y" or yn[0] == "y"): sys.exit("Exiting without overwriting.") -def insert_lines_after(lines, line_pattern, new_lines): +def insert_lines_after(lines: list[str], line_pattern: str, new_lines: list[str]) -> None: re_line = re.compile(line_pattern) for i, line in enumerate(lines): if re_line.search(line): @@ -163,7 +165,7 @@ def insert_lines_after(lines, line_pattern, new_lines): # delete the first occurrence of [line_pattern] -def delete_line(lines, line_pattern): +def delete_line(lines: list[str], line_pattern: str) -> None: re_line = re.compile(line_pattern) for i, line in enumerate(lines): if re_line.search(line): @@ -172,7 +174,7 @@ def delete_line(lines, line_pattern): sys.exit(f"error: no lines found matching pattern: {line_pattern}") -def replace_line(lines, line_pattern, value, all_occurrences=False): +def replace_line(lines: list[str], line_pattern: str, value: str, all_occurrences=False) -> None: replaced = False re_line = re.compile(line_pattern) for i, line in enumerate(lines): @@ -187,28 +189,28 @@ def replace_line(lines, line_pattern, value, all_occurrences=False): # replaces '/' and '.' with '_' so that a valid target name is created -def make_target_name(target): - return prettify(target).replace("/", "_").replace(".", "_") +def make_target_name(target: Path) -> str: + return prettify(str(target)).replace("/", "_").replace(".", "_") # sources are pretty-printed relatively to the .frama-c directory, where the # GNUmakefile will reside -def rel_prefix(path): - return path if os.path.isabs(path) else os.path.relpath(path, start=dot_framac_dir) +def rel_prefix(path: Path) -> str: + return str(path) if os.path.isabs(path) else os.path.relpath(path, start=dot_framac_dir) -def pretty_sources(sources): +def pretty_sources(sources: list[Path]) -> list[str]: return [f" {rel_prefix(source)} \\" for source in sources] -def lines_of_file(path): +def lines_of_file(path: Path) -> list[str]: return path.read_text().splitlines() fc_stubs_copied = False -def copy_fc_stubs(): +def copy_fc_stubs() -> Path: global fc_stubs_copied dest = dot_framac_dir / "fc_stubs.c" if not fc_stubs_copied: @@ -230,7 +232,7 @@ def copy_fc_stubs(): # Returns pairs (line_number, has_args) for each likely definition of # [funcname] in [filename]. # [has_args] is used to distinguish between main(void) and main(int, char**). -def find_definitions(funcname, filename): +def find_definitions(funcname: str, filename: str) -> list[tuple[str, bool]]: file_content = source_filter.open_and_filter(filename, not under_test) file_lines = file_content.splitlines(keepends=True) newlines = function_finder.compute_newline_offsets(file_lines) @@ -251,20 +253,23 @@ def find_definitions(funcname, filename): return res -def list_partition(f, l): +T = TypeVar("T") + + +def list_partition(f: Callable[[T], bool], l: list[T]) -> tuple[list[T], list[T]]: """Equivalent to OCaml's List.partition: returns 2 lists with the elements of l, partitioned according to predicate f.""" l1 = [] l2 = [] for e in l: - if f(l): + if f(e): l1.append(e) else: l2.append(e) return l1, l2 -def pp_list(l): +def pp_list(l: list[str]) -> list[str]: """Applies prettify to a list of sources/targets and sorts the result.""" return sorted([prettify(e) for e in l]) @@ -342,7 +347,7 @@ if unknown_sources: # We also need to check if the main function uses a 'main(void)'-style # signature, to patch fc_stubs.c. -main_definitions = {} +main_definitions: dict[Path, list[tuple[Path, str, bool]]] = {} for target, sources in sources_map.items(): main_definitions[target] = [] for source in sources: @@ -369,7 +374,7 @@ if not dot_framac_dir.is_dir(): logging.debug("creating %s", dot_framac_dir) dot_framac_dir.mkdir(parents=True, exist_ok=False) -fc_config = json.loads(call_and_get_output([framac_bin / "frama-c", "-print-config-json"])) +fc_config = json.loads(call_and_get_output(framac_bin / "frama-c", ["-print-config-json"])) lib_dir = Path(fc_config["lib_dir"]) # copy fc_stubs if at least one main function has arguments diff --git a/share/analysis-scripts/build_callgraph.py b/share/analysis-scripts/build_callgraph.py index 8edc32655e804bacfbb7dd8ca1ccdc1faee21bc0..c8aff818a7d035f88071ed138bb1866b40b89831 100755 --- a/share/analysis-scripts/build_callgraph.py +++ b/share/analysis-scripts/build_callgraph.py @@ -44,12 +44,12 @@ class Callgraph: """ # maps each caller to the list of its callees - succs = {} + succs: dict[str, list[str]] = {} # maps (caller, callee) to the list of call locations - edges = {} + edges: dict[tuple[str, str], list[tuple[str, int]]] = {} - def add_edge(self, caller, callee, loc): + def add_edge(self, caller, callee, loc) -> None: if (caller, callee) in self.edges: # edge already exists self.edges[(caller, callee)].append(loc) @@ -90,7 +90,7 @@ def compute(files): return cg -def print_edge(cg, caller, called, padding="", end="\n"): +def print_edge(cg: Callgraph, caller, called, padding="", end="\n") -> None: locs = cg.edges[(caller, called)] for (filename, line) in locs: print( @@ -99,13 +99,13 @@ def print_edge(cg, caller, called, padding="", end="\n"): ) -def print_cg(cg): +def print_cg(cg: Callgraph) -> None: for (caller, called) in cg.edges: print_edge(cg, caller, called) # note: out _must_ exist (the caller must create it if needed) -def print_cg_dot(cg, out=sys.stdout): +def print_cg_dot(cg: Callgraph, out=sys.stdout) -> None: print("digraph callgraph {", file=out) for (caller, called) in cg.edges: print(f" {caller} -> {called};", file=out) @@ -120,7 +120,7 @@ def print_cg_dot(cg, out=sys.stdout): # The difference between visited and just_visited is that the latter refers # to the current bfs; nodes visited in previous bfs already had their cycles # reported, so we do not report them multiple times. -def cycle_bfs(cg, visited, just_visited, n): +def cycle_bfs(cg, visited, just_visited, n) -> list[tuple[str, str]]: if debug: print(f"cycle_bfs: visited = {visited}, just_visited = {just_visited}, n = {n}") just_visited.add(n) @@ -145,13 +145,13 @@ def cycle_bfs(cg, visited, just_visited, n): return [] -def compute_recursive_cycles(cg, acc): +def compute_recursive_cycles(cg, acc) -> None: to_visit = set(cg.nodes()) if not to_visit: # empty graph -> no recursion return - visited = set() + visited: set[str] = set() while to_visit: - just_visited = set() + just_visited: set[str] = set() n = sorted(list(to_visit))[0] cycle = cycle_bfs(cg, visited, just_visited, n) visited = visited.union(just_visited) @@ -162,14 +162,14 @@ def compute_recursive_cycles(cg, acc): to_visit -= visited -def detect_recursion(cg): +def detect_recursion(cg) -> None: to_visit = set(cg.nodes()) if not to_visit: # empty graph -> no recursion - return False - visited = set() + return + visited: set[str] = set() has_cycle = False while to_visit: - just_visited = set() + just_visited: set[str] = set() n = sorted(list(to_visit))[0] cycle = cycle_bfs(cg, visited, just_visited, n) visited = visited.union(just_visited) diff --git a/share/analysis-scripts/estimate_difficulty.py b/share/analysis-scripts/estimate_difficulty.py index c8dcf2155da073f69ca173d02f251eaf3e19223b..b935675690262e09b70f242ff35d8afb7ee32de0 100755 --- a/share/analysis-scripts/estimate_difficulty.py +++ b/share/analysis-scripts/estimate_difficulty.py @@ -172,7 +172,7 @@ with open(framac_share / "compliance" / "posix_identifiers.json", encoding="utf- posix_identifiers = all_data["data"] posix_headers = all_data["headers"] -recursive_cycles = [] +recursive_cycles: list[tuple[tuple[str, int], list[tuple[str, str]]]] = [] reported_recursive_pairs = set() build_callgraph.compute_recursive_cycles(cg, recursive_cycles) for (cycle_start_loc, cycle) in recursive_cycles: @@ -191,8 +191,8 @@ for (cycle_start_loc, cycle) in recursive_cycles: pretty_cycle += f" -> {y}" print(f"[recursion] found recursive cycle near {filename}:{line}: {pretty_cycle}") -callees = [callee for (_, callee) in list(cg.edges.keys())] -callees = set(callees) +callees_list = [callee for (_, callee) in list(cg.edges.keys())] +callees = set(callees_list) used_headers = set() print(f"Estimating difficulty for {len(callees)} function calls...") warnings = 0 diff --git a/share/analysis-scripts/list_files.py b/share/analysis-scripts/list_files.py index 691ef6ac04e66ffe535defeb241a95ed2beb9183..71a4544ea7ae86968450b03a7ebb07ca79f9b95c 100755 --- a/share/analysis-scripts/list_files.py +++ b/share/analysis-scripts/list_files.py @@ -51,11 +51,8 @@ fcmake_pwd = Path(pwd) / ".frama-c" # pwd as seen by the Frama-C makefile with open(arg) as data: jcdb = json.load(data) jcdb_dir = arg.parent -includes = set() -defines = set() -files = set() # set of pairs of (file, file_for_fcmake) +files: set[tuple[str, str]] = set() # set of pairs of (file, file_for_fcmake) for entry in jcdb: - arg_includes = [] # before normalization if not "file" in entry: # ignore entries without a filename continue @@ -81,9 +78,9 @@ print("") files_defining_main = set() re_main = re.compile(r"(int|void)\s+main\s*\([^)]*\)\s*\{") -for (filename, file_for_fcmake) in files: - assert os.path.exists(filename), "file does not exist: %s" % filename - with open(filename, "r") as content_file: +for (fname, file_for_fcmake) in files: + assert os.path.exists(fname), "file does not exist: %s" % fname + with open(fname, "r") as content_file: content = content_file.read() res = re.search(re_main, content) if res is not None: diff --git a/share/analysis-scripts/make_wrapper.py b/share/analysis-scripts/make_wrapper.py index eee32358838c1a1a25af908def2d3d1a904b2c72..c9e758261bf105df938dd164527f27c9702ac7c2 100755 --- a/share/analysis-scripts/make_wrapper.py +++ b/share/analysis-scripts/make_wrapper.py @@ -27,6 +27,7 @@ GNUmakefile template): it parses the output and suggests useful commands whenever it can, by calling frama-c-script itself.""" import argparse +import collections import os import re import subprocess @@ -79,27 +80,38 @@ output_lines = [] cmd_list = [make_cmd, "-C", make_dir] + args with subprocess.Popen(cmd_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: while True: - line = proc.stdout.readline() - if line: - sys.stdout.buffer.write(line) + assert proc.stdout is not None # for mypy + line_bytes = proc.stdout.readline() + if line_bytes: + sys.stdout.buffer.write(line_bytes) sys.stdout.flush() - output_lines.append(line.decode("utf-8")) + output_lines.append(line_bytes.decode("utf-8")) else: break -re_missing_spec = re.compile("Neither code nor (specification|[^ ]+ assigns clause) for function ([^,]+),") +re_missing_spec = re.compile( + "Neither code nor (specification|[^ ]+ assigns clause) for function ([^,]+)," +) re_recursive_call_start = re.compile("detected recursive call") re_recursive_call_stack_start = re.compile(r"^\s+stack:") re_recursive_call_stack_end = re.compile(r"^\[") -tips = [] + +class Tip: + def __init__(self, message: str, action: collections.abc.Callable): + self.message = message + self.action = action + + +tips: list[Tip] = [] lines = iter(output_lines) for line in lines: match = re_recursive_call_start.search(line) if match: - def _action(): + def _action(callstack): + print(callstack) print( "Consider patching, stubbing or adding an ACSL " + "specification to the recursive call, " @@ -123,11 +135,7 @@ for line in lines: line = next(lines) match = re_recursive_call_stack_end.search(line) # note: this ending line can also match re_missing_spec - tip = { - "message": "Found recursive call at:\n" + "".join(msg_lines), - "action": _action, - } - tips.append(tip) + tips.append(Tip("Found recursive call at:", partial(_action, "".join(msg_lines)))) break except StopIteration: print("** Error: did not match expected regex before EOF") @@ -174,16 +182,13 @@ for line in lines: print("Could not find any files defining " + fname + ".") print("Find the sources defining it and add them, " + "or provide a stub.") - tip = { - "message": "Found function with missing " - + spec_kind - + ": " - + fname - + "\n" - + " Looking for files defining it...", - "action": partial(_action, fname), - } - tips.append(tip) + tips.append( + Tip( + f"Found function with missing {spec_kind}: {fname}\n" + + " Looking for files defining it...", + partial(_action, fname), + ) + ) if tips != []: print("") @@ -196,6 +201,6 @@ if tips != []: if counter > 1: print("") print("*** recommendation #" + str(counter) + " ***") - print(str(counter) + ". " + tip["message"]) + print(str(counter) + ". " + str(tip.message)) counter += 1 - tip["action"]() + tip.action() diff --git a/share/analysis-scripts/normalize_jcdb.py b/share/analysis-scripts/normalize_jcdb.py index 64bf31d6aed119023c217a821387c792b8e26c52..4b221b65f56c94305ab348605bdcfc86dc00fae4 100755 --- a/share/analysis-scripts/normalize_jcdb.py +++ b/share/analysis-scripts/normalize_jcdb.py @@ -44,7 +44,6 @@ if not arg.exists(): with open(arg) as data: jcdb_json = json.load(data) jcdb_dir = arg.parent -out_json = {} replacements = set() diff --git a/share/analysis-scripts/source_filter.py b/share/analysis-scripts/source_filter.py index c255e89c28a0906f14964152f4fbb18ad914db00..6d5ce6f9a293a087f302d2a957acd85631372d16 100644 --- a/share/analysis-scripts/source_filter.py +++ b/share/analysis-scripts/source_filter.py @@ -42,32 +42,33 @@ from pathlib import Path import shutil import subprocess import sys +from typing import Optional # warnings about missing commands are disabled during testing emit_warns = os.getenv("PTESTS_TESTING") is None -# Cache for get_command -cached_commands = {} +# Cache for get_command. +cached_commands: dict[str, Optional[Path]] = {} -def resource_path(relative_path): +def resource_path(relative_path: str) -> str: """Get absolute path to resource; only used by the pyinstaller standalone distribution""" base_path = getattr(sys, "_MEIPASS", os.path.dirname(os.path.abspath(__file__))) return os.path.join(base_path, relative_path) -def get_command(command, env_var_name): +def get_command(command: str, env_var_name: str) -> Optional[Path]: """Returns a Path to the command; priority goes to the environment variable, then in the PATH, then in the resource directory (for a pyinstaller binary).""" if command in cached_commands: return cached_commands[command] - p = os.getenv(env_var_name) - if p: - p = Path(p) + cmd = os.getenv(env_var_name) + if cmd: + p = Path(cmd) else: - p = shutil.which(command) - if p: - p = Path(p) + cmd = shutil.which(command) + if cmd: + p = Path(cmd) else: p = Path(resource_path(command)) if not p.exists(): @@ -81,7 +82,7 @@ def get_command(command, env_var_name): return p -def run_and_check(command_and_args, input_data): +def run_and_check(command_and_args: list[str], input_data: str) -> str: try: return subprocess.check_output( command_and_args, @@ -94,25 +95,25 @@ def run_and_check(command_and_args, input_data): sys.exit(f"error running command: {command_and_args}\n{e}") -def filter_with_scc(input_data): +def filter_with_scc(input_data: str) -> str: scc = get_command("scc", "SCC") if scc: - return run_and_check([scc, "-k"], input_data) + return run_and_check([str(scc), "-k"], input_data) else: return input_data -def filter_with_astyle(input_data): +def filter_with_astyle(input_data: str) -> str: astyle = get_command("astyle", "ASTYLE") if astyle: return run_and_check( - [astyle, "--keep-one-line-blocks", "--keep-one-line-statements"], input_data + [str(astyle), "--keep-one-line-blocks", "--keep-one-line-statements"], input_data ) else: return input_data -def open_and_filter(filename, apply_filters): +def open_and_filter(filename: str, apply_filters: bool) -> str: # we ignore encoding errors and use ASCII to avoid issues when # opening files with different encodings (UTF-8, ISO-8859, etc) with open(filename, "r", encoding="ascii", errors="ignore") as f: diff --git a/share/analysis-scripts/summary.py b/share/analysis-scripts/summary.py index b2ee5d102f1b4319ecc279c67e73e45d60fe4c38..ccd83ec44e0d436453e190da5d45b5e16f187ee1 100755 --- a/share/analysis-scripts/summary.py +++ b/share/analysis-scripts/summary.py @@ -31,6 +31,7 @@ import signal import argparse import uuid import csv +import typing from pathlib import Path import frama_c_results @@ -239,6 +240,7 @@ try: framac = os.path.abspath(args.repository_path) gitdir = framac + benchmark_tag: typing.Optional[str] if args.benchmark: benchmark_tag = str(uuid.uuid1()) print("Running benchmarks with benchmark tag", benchmark_tag, "...")