From f14d226f3526634836f56a1db91ece6387838957 Mon Sep 17 00:00:00 2001
From: Andre Maroneze <andre.maroneze@cea.fr>
Date: Mon, 2 May 2022 19:38:54 +0200
Subject: [PATCH] [analysis-scripts] add type hints, remove unused variables,
 minor fixes

---
 share/analysis-scripts/build.py               | 30 +++----
 share/analysis-scripts/build_callgraph.py     |  5 +-
 share/analysis-scripts/estimate_difficulty.py | 78 +++++++++----------
 share/analysis-scripts/external_tool.py       | 37 ++++-----
 share/analysis-scripts/list_files.py          | 13 ++--
 share/analysis-scripts/make_wrapper.py        | 24 +++---
 share/analysis-scripts/normalize_jcdb.py      |  1 -
 share/analysis-scripts/source_filter.py       | 14 ++--
 8 files changed, 101 insertions(+), 101 deletions(-)

diff --git a/share/analysis-scripts/build.py b/share/analysis-scripts/build.py
index 2ce2ed44e06..75e0bf5538e 100755
--- a/share/analysis-scripts/build.py
+++ b/share/analysis-scripts/build.py
@@ -34,6 +34,7 @@ import re
 import shutil
 import sys
 import subprocess
+import typing
 
 import function_finder
 import source_filter
@@ -112,21 +113,20 @@ dot_framac_dir = Path(".frama-c")
 
 # Check required environment variables and commands in the PATH ###############
 
-framac_bin = os.getenv("FRAMAC_BIN")
-if not framac_bin:
-    sys.exit("error: FRAMAC_BIN not in environment (set by frama-c-script)")
-framac_bin = Path(framac_bin)
+framac_bin = Path(
+    os.getenv("FRAMAC_BIN")
+    or sys.exit("error: FRAMAC_BIN not in environment (set by frama-c-script)")
+)
 
 under_test = os.getenv("PTESTS_TESTING")
 
 # Prepare blug-related variables and functions ################################
 
-blug = os.getenv("BLUG")
-if not blug:
-    blug = shutil.which("blug")
-    if not blug:
-        sys.exit("error: path to 'blug' binary must be in PATH or variable BLUG")
-blug = Path(blug)
+blug = Path(
+    os.getenv("BLUG")
+    or shutil.which("blug")
+    or sys.exit("error: path to 'blug' binary must be in PATH or variable BLUG")
+)
 blug_dir = blug.resolve().parent
 # to import blug_jbdb
 sys.path.insert(0, blug_dir.as_posix())
@@ -191,11 +191,13 @@ def make_target_name(target):
 
 # sources are pretty-printed relatively to the .frama-c directory, where the
 # GNUmakefile will reside
-def rel_prefix(path):
+def rel_prefix(path, max_rel_parents=1):
     """Return a relative path to the .frama-c directory if path is relative, or if the relativized
-    path will contain at most a single '..'. Otherwise, return an absolute path."""
+    path will contain at most max_rel_parents (in the form of '..').
+    Otherwise, return an absolute path."""
     rel = os.path.relpath(path, start=dot_framac_dir)
-    if rel.startswith("../.."):
+    max_parent_prefix = "../" * (max_rel_parents + 1)
+    if rel.startswith(max_parent_prefix):
         return os.path.abspath(path)
     else:
         return rel
@@ -346,7 +348,7 @@ if unknown_sources:
 # We also need to check if the main function uses a 'main(void)'-style
 # signature, to patch fc_stubs.c.
 
-main_definitions = {}
+main_definitions: dict[Path, list[typing.Tuple[Path, str, str]]] = {}
 for target, sources in sources_map.items():
     main_definitions[target] = []
     for source in sources:
diff --git a/share/analysis-scripts/build_callgraph.py b/share/analysis-scripts/build_callgraph.py
index 8edc32655e8..f9a6d60a4dd 100755
--- a/share/analysis-scripts/build_callgraph.py
+++ b/share/analysis-scripts/build_callgraph.py
@@ -26,6 +26,7 @@ for a given function name, via heuristic syntactic matching."""
 
 import os
 import sys
+import typing
 
 import function_finder
 import source_filter
@@ -44,10 +45,10 @@ class Callgraph:
     """
 
     # maps each caller to the list of its callees
-    succs = {}
+    succs: dict[str, list[str]] = {}
 
     # maps (caller, callee) to the list of call locations
-    edges = {}
+    edges: dict[typing.Tuple[str, str], list[typing.Tuple[str, int]]] = {}
 
     def add_edge(self, caller, callee, loc):
         if (caller, callee) in self.edges:
diff --git a/share/analysis-scripts/estimate_difficulty.py b/share/analysis-scripts/estimate_difficulty.py
index 0ef08c6acb0..7e669711204 100755
--- a/share/analysis-scripts/estimate_difficulty.py
+++ b/share/analysis-scripts/estimate_difficulty.py
@@ -35,6 +35,7 @@ import re
 import subprocess
 import sys
 import tempfile
+import typing
 
 import build_callgraph
 import external_tool
@@ -86,30 +87,27 @@ fclog.init(debug, verbose)
 ### Auxiliary functions #######################################################
 
 
-def get_dir(path):
-    """Similar to dirname, but returns the path itself if it refers to a directory"""
-    if path.is_dir():
-        return path
-    else:
-        return path.parent
-
-
-def collect_files_and_local_dirs(paths):
+def collect_files_and_local_dirs(
+    paths: typing.Iterable[Path],
+) -> typing.Tuple[set[Path], set[Path]]:
     """Returns the list of directories (and their subdirectories) containing
     the specified paths. Note that this also includes subdirectories which do not
     themselves contain any .c files, but which may contain .h files."""
-    dirs = set()
-    files = set()
+    dirs: set[Path] = set()
+    files: set[Path] = set()
     for p in paths:
         if p.is_dir():
-            files = files.union(glob.glob(f"{p}/**/*.[chi]", recursive=True))
+            files = files.union([Path(p) for p in glob.glob(f"{p}/**/*.[chi]", recursive=True)])
             dirs.add(p)
         else:
             files.add(p)
             dirs.add(p.parent)
-    local_dirs = {s[0] for d in dirs for s in os.walk(d)}
+    local_dirs = {Path(s[0]) for d in dirs for s in os.walk(d)}
     if not files:
-        sys.exit("error: no source files (.c/.i) found in provided paths: " + " ".join(paths))
+        sys.exit(
+            "error: no source files (.c/.i) found in provided paths: "
+            + " ".join([str(p) for p in paths])
+        )
     return files, local_dirs
 
 
@@ -117,7 +115,9 @@ def extract_keys(l):
     return [list(key.keys())[0] for key in l]
 
 
-def get_framac_libc_function_statuses(framac, framac_share):
+def get_framac_libc_function_statuses(
+    framac: typing.Optional[Path], framac_share: Path
+) -> typing.Tuple[list[str], list[str]]:
     if framac:
         (_handler, metrics_tmpfile) = tempfile.mkstemp(prefix="fc_script_est_diff", suffix=".json")
         logging.debug("metrics_tmpfile: %s", metrics_tmpfile)
@@ -150,7 +150,7 @@ def get_framac_libc_function_statuses(framac, framac_share):
     return (defined, spec_only)
 
 
-def grep_includes_in_file(filename):
+def grep_includes_in_file(filename: Path):
     re_include = re.compile(r'\s*#\s*include\s*("|<)([^">]+)("|>)')
     file_content = source_filter.open_and_filter(filename, not under_test)
     i = 0
@@ -163,9 +163,9 @@ def grep_includes_in_file(filename):
             yield (i, kind, header)
 
 
-def get_includes(files):
-    quote_includes = {}
-    chevron_includes = {}
+def get_includes(files: typing.Iterable[Path]):
+    quote_includes: dict[Path, list[typing.Tuple[Path, int]]] = {}
+    chevron_includes: dict[Path, list[typing.Tuple[Path, int]]] = {}
     for filename in files:
         for line, kind, header in grep_includes_in_file(filename):
             if kind == "<":
@@ -180,30 +180,27 @@ def get_includes(files):
     return chevron_includes, quote_includes
 
 
-def is_local_header(local_dirs, header):
+def is_local_header(local_dirs: typing.Iterable[Path], header: Path):
     for d in local_dirs:
-        path = Path(d)
-        if Path(path / header).exists():
+        if (d / header).exists():
             return True
     return False
 
 
-def grep_keywords(keywords, filename):
+def grep_keywords(keywords: list[str], filename: Path) -> dict[str, int]:
     with open(filename, "r") as f:
-        found = {}
+        found: dict[str, int] = {}
         for line in f:
-            if any(x in line for x in keywords):
-                # found one or more keywords; count them
-                for kw in keywords:
-                    if kw in line:
-                        if kw in found:
-                            found[kw] += 1
-                        else:
-                            found[kw] = 1
+            for kw in keywords:
+                if kw in line:
+                    if kw in found:
+                        found[kw] += 1
+                    else:
+                        found[kw] = 1
         return found
 
 
-def pretty_keyword_count(found):
+def pretty_keyword_count(found: dict[str, int]) -> str:
     res = ""
     for kw, count in sorted(found.items()):
         if res:
@@ -248,7 +245,9 @@ if not no_cloc:
     cloc = external_tool.get_command("cloc", "CLOC")
     if cloc:
         data = external_tool.run_and_check(
-            [cloc, "--hide-rate", "--progress-rate=0", "--csv"] + list(files), ""
+            [str(cloc), "--hide-rate", "--progress-rate=0", "--csv"]
+            + list(str(f) for f in files),
+            "",
         )
         data = data.splitlines()
         [nfiles, _sum, nblank, ncomment, ncode] = data[-1].split(",")
@@ -279,7 +278,7 @@ with open(framac_share / "compliance" / "posix_identifiers.json", encoding="utf-
     posix_identifiers = all_data["data"]
     posix_headers = all_data["headers"]
 
-recursive_cycles = []
+recursive_cycles: list[typing.Tuple[typing.Tuple[str, int], list[typing.Tuple[str, str]]]] = []
 reported_recursive_pairs = set()
 build_callgraph.compute_recursive_cycles(cg, recursive_cycles)
 for (cycle_start_loc, cycle) in recursive_cycles:
@@ -305,8 +304,7 @@ for (cycle_start_loc, cycle) in recursive_cycles:
     )
     score["recursion"] += 1
 
-callees = [callee for (_, callee) in list(cg.edges.keys())]
-callees = set(callees)
+callees = set(callee for (_, callee) in list(cg.edges.keys()))
 used_headers = set()
 logging.info("Estimating difficulty for %d function calls...", len(callees))
 warnings = 0
@@ -439,10 +437,10 @@ c11_unsupported = [
 
 logging.info("Checking presence of unsupported C11 features...")
 
-for f in files:
-    found = grep_keywords(c11_unsupported, f)
+for fi in files:
+    found = grep_keywords(c11_unsupported, fi)
     if found:
-        logging.warning("unsupported keyword(s) in %s:%s", f, pretty_keyword_count(found))
+        logging.warning("unsupported keyword(s) in %s:%s", fi, pretty_keyword_count(found))
     score["keywords"] += len(found)
 
 # assembly code
diff --git a/share/analysis-scripts/external_tool.py b/share/analysis-scripts/external_tool.py
index d5e4dfd5aee..b660423228d 100644
--- a/share/analysis-scripts/external_tool.py
+++ b/share/analysis-scripts/external_tool.py
@@ -29,46 +29,41 @@ from pathlib import Path
 import shutil
 import subprocess
 import sys
+import typing
 
 # warnings about missing commands are disabled during testing
 emit_warns = os.getenv("PTESTS_TESTING") is None
 
 # Cache for get_command
-cached_commands = {}
+cached_commands: dict[str, typing.Union[Path, None]] = {}
 
 
-def resource_path(relative_path):
+def resource_path(relative_path: str) -> str:
     """Get absolute path to resource; only used by the pyinstaller standalone distribution"""
     base_path = getattr(sys, "_MEIPASS", os.path.dirname(os.path.abspath(__file__)))
     return os.path.join(base_path, relative_path)
 
 
-def get_command(command, env_var_name):
+def get_command(command: str, env_var_name: str) -> typing.Union[Path, None]:
     """Returns a Path to the command; priority goes to the environment variable,
     then in the PATH, then in the resource directory (for a pyinstaller binary)."""
     if command in cached_commands:
         return cached_commands[command]
-    p = os.getenv(env_var_name)
-    if p:
-        p = Path(p)
+    p = Path(os.getenv(env_var_name) or shutil.which(command) or resource_path(command))
+    if p.exists():
+        cached_commands[command] = p
+        return p
     else:
-        p = shutil.which(command)
-        if p:
-            p = Path(p)
-        else:
-            p = Path(resource_path(command))
-            if not p.exists():
-                if emit_warns:
-                    print(
-                        f"info: optional external command '{command}' not found in PATH;",
-                        f"consider installing it or setting environment variable {env_var_name}",
-                    )
-                p = None
-    cached_commands[command] = p
-    return p
+        if emit_warns:
+            print(
+                f"info: optional external command '{command}' not found in PATH;",
+                f"consider installing it or setting environment variable {env_var_name}",
+            )
+        cached_commands[command] = None
+        return None
 
 
-def run_and_check(command_and_args, input_data):
+def run_and_check(command_and_args: list[str], input_data: str):
     try:
         return subprocess.check_output(
             command_and_args,
diff --git a/share/analysis-scripts/list_files.py b/share/analysis-scripts/list_files.py
index 691ef6ac04e..c147503889b 100755
--- a/share/analysis-scripts/list_files.py
+++ b/share/analysis-scripts/list_files.py
@@ -42,7 +42,7 @@ if not arg.exists():
     sys.exit(f"error: file '{arg}' not found")
 
 # check if arg has a known extension
-def is_known_c_extension(ext):
+def is_known_c_extension(ext: str) -> bool:
     return ext in (".c", ".i", ".ci", ".h")
 
 
@@ -51,11 +51,10 @@ fcmake_pwd = Path(pwd) / ".frama-c"  # pwd as seen by the Frama-C makefile
 with open(arg) as data:
     jcdb = json.load(data)
 jcdb_dir = arg.parent
-includes = set()
-defines = set()
+includes: set[Path] = set()
+defines: set[Path] = set()
 files = set()  # set of pairs of (file, file_for_fcmake)
 for entry in jcdb:
-    arg_includes = []  # before normalization
     if not "file" in entry:
         # ignore entries without a filename
         continue
@@ -81,9 +80,9 @@ print("")
 
 files_defining_main = set()
 re_main = re.compile(r"(int|void)\s+main\s*\([^)]*\)\s*\{")
-for (filename, file_for_fcmake) in files:
-    assert os.path.exists(filename), "file does not exist: %s" % filename
-    with open(filename, "r") as content_file:
+for (f, file_for_fcmake) in files:
+    assert os.path.exists(f), "file does not exist: %s" % f
+    with open(f, "r") as content_file:
         content = content_file.read()
         res = re.search(re_main, content)
         if res is not None:
diff --git a/share/analysis-scripts/make_wrapper.py b/share/analysis-scripts/make_wrapper.py
index 7025db38a3d..6f353d8f67c 100755
--- a/share/analysis-scripts/make_wrapper.py
+++ b/share/analysis-scripts/make_wrapper.py
@@ -31,6 +31,8 @@ import os
 import re
 import subprocess
 import sys
+import typing
+
 from functools import partial
 
 # Check if GNU make is available and has the minimal required version
@@ -78,12 +80,14 @@ framac_script = f"{framac_bin}/frama-c-script"
 output_lines = []
 cmd_list = [make_cmd, "-C", make_dir] + args
 with subprocess.Popen(cmd_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc:
+    if not proc.stdout:
+        sys.exit(f"error capturing stdout for cmd: {cmd_list}")
     while True:
-        line = proc.stdout.readline()
-        if line:
-            sys.stdout.buffer.write(line)
+        line_bytes = proc.stdout.readline()
+        if line_bytes:
+            sys.stdout.buffer.write(line_bytes)
             sys.stdout.flush()
-            output_lines.append(line.decode("utf-8"))
+            output_lines.append(line_bytes.decode("utf-8"))
         else:
             break
 
@@ -99,7 +103,7 @@ for line in lines:
     match = re_recursive_call_start.search(line)
     if match:
 
-        def _action():
+        def action_msg():
             print(
                 "Consider patching, stubbing or adding an ACSL "
                 + "specification to the recursive call, "
@@ -125,7 +129,7 @@ for line in lines:
                 # note: this ending line can also match re_missing_spec
                 tip = {
                     "message": "Found recursive call at:\n" + "".join(msg_lines),
-                    "action": _action,
+                    "action": action_msg,
                 }
                 tips.append(tip)
                 break
@@ -136,7 +140,7 @@ for line in lines:
     if match:
         fname = match.group(1)
 
-        def _action(fname):
+        def action_find_fun(fname):
             out = subprocess.Popen(
                 [framac_script, "find-fun", "-C", make_dir, fname],
                 stdout=subprocess.PIPE,
@@ -178,7 +182,7 @@ for line in lines:
             + fname
             + "\n"
             + "   Looking for files defining it...",
-            "action": partial(_action, fname),
+            "action": partial(action_find_fun, fname),
         }
         tips.append(tip)
 
@@ -193,6 +197,6 @@ if tips != []:
         if counter > 1:
             print("")
             print("*** recommendation #" + str(counter) + " ***")
-        print(str(counter) + ". " + tip["message"])
+        print(str(counter) + ". " + typing.cast(str, tip["message"]))
         counter += 1
-        tip["action"]()
+        typing.cast(typing.Callable, tip["action"])()
diff --git a/share/analysis-scripts/normalize_jcdb.py b/share/analysis-scripts/normalize_jcdb.py
index 64bf31d6aed..4b221b65f56 100755
--- a/share/analysis-scripts/normalize_jcdb.py
+++ b/share/analysis-scripts/normalize_jcdb.py
@@ -44,7 +44,6 @@ if not arg.exists():
 with open(arg) as data:
     jcdb_json = json.load(data)
 jcdb_dir = arg.parent
-out_json = {}
 
 replacements = set()
 
diff --git a/share/analysis-scripts/source_filter.py b/share/analysis-scripts/source_filter.py
index 5d4badc598c..752cef5b07b 100644
--- a/share/analysis-scripts/source_filter.py
+++ b/share/analysis-scripts/source_filter.py
@@ -37,31 +37,33 @@ the efficiency of regex-based heuristics."""
 # of errors when running the filters. Note that an absent tool
 # does _not_ lead to an error.
 
-import external_tool
+from pathlib import Path
 import sys
 
+import external_tool
+
 
-def filter_with_scc(input_data):
+def filter_with_scc(input_data: str) -> str:
     scc_bin = "scc" if sys.platform != "win32" else "scc.exe"
     scc = external_tool.get_command(scc_bin, "SCC")
     if scc:
-        return external_tool.run_and_check([scc, "-k", "-b"], input_data)
+        return external_tool.run_and_check([str(scc), "-k", "-b"], input_data)
     else:
         return input_data
 
 
-def filter_with_astyle(input_data):
+def filter_with_astyle(input_data: str) -> str:
     astyle_bin = "astyle" if sys.platform != "win32" else "astyle.exe"
     astyle = external_tool.get_command(astyle_bin, "ASTYLE")
     if astyle:
         return external_tool.run_and_check(
-            [astyle, "--keep-one-line-blocks", "--keep-one-line-statements"], input_data
+            [str(astyle), "--keep-one-line-blocks", "--keep-one-line-statements"], input_data
         )
     else:
         return input_data
 
 
-def open_and_filter(filename, apply_filters):
+def open_and_filter(filename: Path, apply_filters: bool) -> str:
     # we ignore encoding errors and use ASCII to avoid issues when
     # opening files with different encodings (UTF-8, ISO-8859, etc)
     with open(filename, "r", encoding="ascii", errors="ignore") as f:
-- 
GitLab