diff --git a/ivette/Makefile b/ivette/Makefile
index 776923eb29e81a1e28ddd98560b9cd81636708e6..93acf3c67c270e96a5f3b130443a4f05abb36e4c 100644
--- a/ivette/Makefile
+++ b/ivette/Makefile
@@ -283,14 +283,4 @@ tests: dist
@echo "[Ivette] running tests"
@dune exec -- yarn playwright test
-tests-e2e: dist
- @cp -r out/renderer/* out/main/
- @echo "[Ivette] running tests (e2e)"
- @dune exec -- yarn playwright test tests/e2e
-
-tests-monkey: dist
- @cp -r out/renderer/* out/main/
- @echo "[Ivette] running tests (monkey)"
- @dune exec -- yarn playwright test tests/monkey
-
# --------------------------------------------------------------------------
diff --git a/ivette/package.json b/ivette/package.json
index 3cc1956ff7af2d0cebded2cf2eb99b9059aaa182..8c6657c0dae258f906451d3161c06eea43e6bb7a 100644
--- a/ivette/package.json
+++ b/ivette/package.json
@@ -79,7 +79,6 @@
"eslint-plugin-jsx-a11y": "^6.8.0",
"eslint-plugin-react": "^7.33.2",
"eslint-plugin-react-hooks": "^4.6.0",
- "gremlins.js": "^2.2.0",
"prettier": "^3.2.4",
"typescript": "^5.3.3",
"vite": "^5.0.12",
diff --git a/ivette/tests/README.md b/ivette/tests/README.md
index 6e827583c84d781131b4b4abdaaa4742fc22cc36..bafa7d941aeea7b79420ea77e4ebf6e9fc17d075 100644
--- a/ivette/tests/README.md
+++ b/ivette/tests/README.md
@@ -3,29 +3,21 @@ overall architecture, and how to launch current tests and write new ones.
## Overall Architecture
-The framework is based on two main testing libraries:
+The framework is based on [Playwright](https://playwright.dev) library.
-- `Playwright`: <https://playwright.dev/>
-- `Gremlins.js`: <https://github.com/marmelab/gremlins.js>
+`Playwright` is meant for end-to-end testing, i.e. for testing specific
+behaviors. Here is how the framework is organized:
-`Playwright` is meant for end-to-end (e2e) testing, i.e. for testing specific
-behaviors, while `Gremlins.js` provides monkey testing, i.e. robustness/stress
-testing of the entire application.
-
-Here is how the framework is organized:
-
-- `e2e/` contains the end-to-end tests,
-- `monkey/` cointains the monkey tests,
-- `lib/` contains utilities for writing tests using the `Playwright`
- infrastructure. These utilities are used both for e2e and monkey testing.
+- `src/` contains the end-to-end tests,
+- `lib/` contains utilities for writing tests using `Playwright`
## Executing tests
**Requirement:** In order to execute the tests, Frama-C has to be built
beforehand.
-Testing Ivette, in terms of both the e2e and monkey tests, amounts to the
-execution of the following commands from the Frama-C root directory:
+Testing Ivette amounts to the execution of the following commands from the
+Frama-C root directory:
```sh
$ make // builds Frama-C, if not already
@@ -33,18 +25,6 @@ $ cd ivette
$ make tests // builds Ivette and execute all tests
```
-To execute the e2e tests only, the last command to execute is the following:
-
-```sh
-$ make tests-e2e
-```
-
-To execute the monkey tests only, the last command to execute is the following:
-
-```sh
-$ make tests-monkey
-```
-
## Executing tests (Advanced)
Whenever the execution of the tests does not need (re-)building Ivette, one can
@@ -135,19 +115,4 @@ window with a `Pick locator` button at the bottom, which provides the necessary
locator any selected element (i.e., it works similarly to the inspect
functionality of most browser developer tools).
-Complete test scenarios are provided in the sub-directory `/tests/e2e/`.
-
-## Monkey Testing
-
-Monkey testing is meant for testing the Ivette robustness by performing a stress
-testing of the UI. This is done by using the `gremlins.js` library.
-
-Please refer to the [library
-documentation](https://github.com/marmelab/gremlins.js) for more details, or
-look at the `/tests/monkey/monkey-testing.spec.ts` for an example.
-
-Note that, to obtain reproducible tests, one should seed a randomizer (see the
-`randomizer` value in `monkey-testing.spec.ts` for an example). For that, one
-should also launch Ivette with either default settings, by using the
-`-settings CLEAN` option, or fixed settings, by using the `--settings
-</path/to/json-file-with-settings>` option.
+Complete test scenarios are provided in the sub-directory `/tests/src/`.
diff --git a/ivette/tests/e2e/launch-app.spec.ts b/ivette/tests/e2e/launch-app.spec.ts
deleted file mode 100644
index e9e2ce889fdec3443b3bdb9d82d2fe9f0757961d..0000000000000000000000000000000000000000
--- a/ivette/tests/e2e/launch-app.spec.ts
+++ /dev/null
@@ -1,37 +0,0 @@
-/* ************************************************************************ */
-/* */
-/* This file is part of Frama-C. */
-/* */
-/* Copyright (C) 2007-2023 */
-/* CEA (Commissariat à l'énergie atomique et aux énergies */
-/* alternatives) */
-/* */
-/* you can redistribute it and/or modify it under the terms of the GNU */
-/* Lesser General Public License as published by the Free Software */
-/* Foundation, version 2.1. */
-/* */
-/* It is distributed in the hope that it will be useful, */
-/* but WITHOUT ANY WARRANTY; without even the implied warranty of */
-/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */
-/* GNU Lesser General Public License for more details. */
-/* */
-/* See the GNU Lesser General Public License version 2.1 */
-/* for more details (enclosed in the file licenses/LGPLv2.1). */
-/* */
-/* ************************************************************************ */
-
-import { test } from "@playwright/test";
-import * as e2eService from "../libs/e2eService";
-
-test("launch app", async () => {
- const launchAppResult = await e2eService.launchApp(
- e2eService.argsDefaultLaunch,
- );
- const electronApp = launchAppResult.app;
- const window = launchAppResult.page;
-
- await window.screenshot({ path: "screenshots/e2e-app-launch.png" });
-
- // Exit app.
- await electronApp.close();
-});
diff --git a/ivette/tests/libs/e2eService.ts b/ivette/tests/libs/e2eService.ts
index e71eb4a2bec5f3acc2adeaf5c012b130cca4ddcf..6b7c6dfe21818c2f76941a6b29700dae30f96146 100644
--- a/ivette/tests/libs/e2eService.ts
+++ b/ivette/tests/libs/e2eService.ts
@@ -25,46 +25,25 @@ import { _electron as electron } from "playwright-core";
import * as locs from "./locatorsUtil";
/**
- * Basic Electron configuration for Playwright e2e tests of Ivette
+ * Basic Electron launch of Ivette for Playwright tests
*/
-export const argsDefaultLaunch: string[] = [
- "./out/main/index.js",
- "--no-sandbox",
-];
-
-/**
- * Electron configuration for Playwright e2e tests of Ivette's default settings
- */
-export const argsLaunchWithDefaultSettings: string[] = [
- "./out/main/index.js",
- "--no-sandbox",
- "--settings",
- "DEFAULT"
-];
-
-/**
- * Electron configuration for Playwright e2e tests of Ivette on a C file
- */
-export const argsLaunchWithTestFile: string[] = [
- "./out/main/index.js",
- "--no-sandbox",
- "--settings",
- "./tests/settings.json",
- "../tests/test/adpcm.c",
-];
-
-/**
- * Basic Electron launch of Ivette for Playwright e2e tests
- */
-export async function launchApp(
- params: string[]
+export async function launchIvette(
+ ...params: string[]
): Promise<{ app: ElectronApplication; page: Page }> {
+ const args: string[] = [
+ "./out/main/index.js",
+ "--no-sandbox",
+ "--settings", "DEFAULT"
+ ];
+ params.forEach(p => {
+ p.trim().split(/\s+/).forEach(a => args.push(a));
+ });
const electronApp = await electron.launch({
env: {
...process.env,
NODE_ENV: "development",
},
- args: params,
+ args: args,
});
// Get the first window that the app opens, wait if necessary
@@ -87,19 +66,20 @@ export async function testServerIsStarted(window: Page): Promise<void> {
// Check the server status in the console view
await expect(
locs.getConsoleComponent(window)
- .getByText("[server] Socket server running.")
+ .getByText("[server] Socket server running.")
).toBeVisible();
// Check the server status in the footer
await expect(locs.getServerStatusLabel(window)).toHaveText("ON");
}
-export async function testFileIsLoaded(window: Page): Promise<void> {
+export async function testFileIsLoaded(window: Page, file: string):
+ Promise<void> {
await locs.getConsoleView(window).click();
// Check if a message is present in the console view to confirm the file is
// loaded
await expect(
- locs.getConsoleComponent(window).getByText("adpcm.c (with preprocessing)")
+ locs.getConsoleComponent(window).getByText(`${file} (with preprocessing)`)
).toBeVisible();
// Check if the main function is visible in the functions view
diff --git a/ivette/tests/libs/locatorsUtil.ts b/ivette/tests/libs/locatorsUtil.ts
index a24317eb2cb350c69425ad885fcd90208d19de56..552f469314525de83b86a097fbe3a02a2541aa4a 100644
--- a/ivette/tests/libs/locatorsUtil.ts
+++ b/ivette/tests/libs/locatorsUtil.ts
@@ -26,8 +26,10 @@ import { Locator, Page } from "@playwright/test";
* Locator to select "Console" in the right menu
*/
export function getConsoleView(window: Page): Locator {
- return window
- .getByText("Console").first();
+ window
+ .getByText("Other Plugins")
+ .click();
+ return window.getByText("Console").first();
}
/**
diff --git a/ivette/tests/monkey/monkey-testing.spec.ts b/ivette/tests/monkey/monkey-testing.spec.ts
deleted file mode 100644
index 0e5d7b9e3e5c19beaef0cad6553f43dae4bf3ea8..0000000000000000000000000000000000000000
--- a/ivette/tests/monkey/monkey-testing.spec.ts
+++ /dev/null
@@ -1,81 +0,0 @@
-/* ************************************************************************ */
-/* */
-/* This file is part of Frama-C. */
-/* */
-/* Copyright (C) 2007-2023 */
-/* CEA (Commissariat à l'énergie atomique et aux énergies */
-/* alternatives) */
-/* */
-/* you can redistribute it and/or modify it under the terms of the GNU */
-/* Lesser General Public License as published by the Free Software */
-/* Foundation, version 2.1. */
-/* */
-/* It is distributed in the hope that it will be useful, */
-/* but WITHOUT ANY WARRANTY; without even the implied warranty of */
-/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */
-/* GNU Lesser General Public License for more details. */
-/* */
-/* See the GNU Lesser General Public License version 2.1 */
-/* for more details (enclosed in the file licenses/LGPLv2.1). */
-/* */
-/* ************************************************************************ */
-
-import { test } from "@playwright/test";
-import * as e2eService from "../libs/e2eService";
-
-/* eslint-disable @typescript-eslint/no-explicit-any */
-let gremlins: any;
-
-test("run gremlins.js", async () => {
- test.slow(); // long timeout for a very slow test.
-
- const launchAppResult = await e2eService.launchApp(
- e2eService.argsLaunchWithTestFile,
- );
- const electronApp = launchAppResult.app;
- const window = launchAppResult.page;
-
- // await window.pause()
-
- // Load gremlins.js script
- await window.addScriptTag(
- { path: "./node_modules/gremlins.js/dist/gremlins.min.js", }
- );
-
- // Launch gremlins.js with default settings
- // await window.evaluate(() => gremlins.createHorde().unleash());
-
- // Launch gremlins.js with custom settings
- // See https://github.com/marmelab/gremlins.js for more info
-
- await window.evaluate(async () => {
- await gremlins
- .createHorde({
- species: [
- gremlins.species.clicker(), // clicks anywhere on the visible area of the document
- gremlins.species.toucher(), // touches anywhere on the visible area of the document
- gremlins.species.scroller(), // scrolls the viewport to reveal another part of the document
- gremlins.species.typer(), // types keys on the keyboard
- gremlins.species.formFiller(), // fills forms by entering data, selecting options, clicking checkboxes, etc
- ],
- mogwais: [
- gremlins.mogwais.alert(), // prevents calls to alert() from blocking the test
- gremlins.mogwais.fps(), // logs the number of frames per seconds (FPS) of the browser
- ],
- strategies: [
- gremlins.strategies.distribution({
- distribution: [0.3, 0.3, 0.3, 0.1, 0.1], // the first three gremlins have more chances to be executed than the last
- delay: 10, // wait 10 ms between each action
- }),
- ],
- randomizer: new gremlins.Chance(1234), // if you want the attack to be repeatable, you need to seed the random number generator
- })
- .unleash();
- });
-
- // Could be useful if you want to review the logs
- // await window.pause()
-
- // Exit app.
- await electronApp.close();
-});
diff --git a/ivette/tests/e2e/server-connection-file.spec.ts b/ivette/tests/src/server-connection-file.spec.ts
similarity index 92%
rename from ivette/tests/e2e/server-connection-file.spec.ts
rename to ivette/tests/src/server-connection-file.spec.ts
index 7a379f024a14cc732092aaa3e8f879659dcda42a..cd59d88d2455f4ca8f3318e3f590eed245d52656 100644
--- a/ivette/tests/e2e/server-connection-file.spec.ts
+++ b/ivette/tests/src/server-connection-file.spec.ts
@@ -24,14 +24,10 @@ import { test } from "@playwright/test";
import * as e2eService from "../libs/e2eService";
test("server connection with a C file to analyze", async () => {
- const launchAppResult = await e2eService.launchApp(
- e2eService.argsLaunchWithTestFile,
- );
+ const launchAppResult =
+ await e2eService.launchIvette("../tests/test/adpcm.c");
const electronApp = launchAppResult.app;
const window = launchAppResult.page;
-
- await e2eService.testFileIsLoaded(window);
-
- // Exit app.
+ await e2eService.testFileIsLoaded(window, "tests/test/adpcm.c");
await electronApp.close();
});
diff --git a/ivette/tests/e2e/server-connection.spec.ts b/ivette/tests/src/server-connection.spec.ts
similarity index 94%
rename from ivette/tests/e2e/server-connection.spec.ts
rename to ivette/tests/src/server-connection.spec.ts
index 263384fd1ce7aaf0b5eae20a878977a8633a55ef..474137284531328684ad2fdcb5703e50a9dd2942 100644
--- a/ivette/tests/e2e/server-connection.spec.ts
+++ b/ivette/tests/src/server-connection.spec.ts
@@ -24,14 +24,9 @@ import { test } from "@playwright/test";
import * as e2eService from "../libs/e2eService";
test("check server connection", async () => {
- const launchAppResult = await e2eService.launchApp(
- e2eService.argsLaunchWithDefaultSettings,
- );
+ const launchAppResult = await e2eService.launchIvette();
const electronApp = launchAppResult.app;
const window = launchAppResult.page;
-
await e2eService.testServerIsStarted(window);
-
- // Exit app.
await electronApp.close();
});
diff --git a/ivette/yarn.lock b/ivette/yarn.lock
index a7ef814f26b39138d10d5feeda2d91a59c62a00a..3637da3ac4476d9cf8d32b7b114fbd1da752c629 100644
--- a/ivette/yarn.lock
+++ b/ivette/yarn.lock
@@ -1841,11 +1841,6 @@ chalk@^4.0.0, chalk@^4.0.2, chalk@^4.1.0, chalk@^4.1.2:
ansi-styles "^4.1.0"
supports-color "^7.1.0"
-chance@^1.1.4:
- version "1.1.11"
- resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.11.tgz#78e10e1f9220a5bbc60a83e3f28a5d8558d84d1b"
- integrity sha512-kqTg3WWywappJPqtgrdvbA380VoXO2eu9VCV895JgbyHsaErXdyHK9LOZ911OvAk6L0obK7kDk9CGs8+oBawVA==
-
chownr@^2.0.0:
version "2.0.0"
resolved "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz"
@@ -2006,11 +2001,6 @@ convert-source-map@^2.0.0:
resolved "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz"
integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==
-core-js@^3.6.4:
- version "3.36.0"
- resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.36.0.tgz#e752fa0b0b462a0787d56e9d73f80b0f7c0dde68"
- integrity sha512-mt7+TUBbTFg5+GngsAxeKBTl5/VS0guFeJacYge9OmHb+m058UwwIm41SE9T4Den7ClatV57B6TYTuJ0CX1MAw==
-
core-util-is@1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
@@ -3291,14 +3281,6 @@ graphlib@^2.1.8:
dependencies:
lodash "^4.17.15"
-gremlins.js@^2.2.0:
- version "2.2.0"
- resolved "https://registry.yarnpkg.com/gremlins.js/-/gremlins.js-2.2.0.tgz#663b09cbc59f09ffa1263e2737286c10c7f6fb79"
- integrity sha512-zVe4+WuCwTheg1OzBOdtiy7nZj52lWyWjCPMw/78wI1uAJdGlc97hVGYsZg0v75Hm+E91y5D+yG4U0lk7j/6xg==
- dependencies:
- chance "^1.1.4"
- core-js "^3.6.4"
-
has-bigints@^1.0.1:
version "1.0.1"
resolved "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz"