Merge pull request #602 from yorukot/testsuite_action

Testsuite in github actions

.github/workflows/testsuite-run.yml

@@ -0,0 +1,46 @@
+name: Python Testsuite Run
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install tmux
+        run: sudo apt-get update && sudo apt-get install -y tmux
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.22.2
+      - name: Build superfile
+        run: ./build.sh
+
+      # timeout command just launches and kills spf, to create the config directories
+      - name: Check installation
+        run: tmux -V; ls; pwd; ls bin/; bin/spf path-list; timeout 1s bin/spf
+        continue-on-error: true
+      - name: set debug
+        run: sed -i 's/debug = false/debug = true/g' /home/runner/.config/superfile/config.toml
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
+      - name: Install Dependencies
+        run: pip install -r testsuite/requirements.txt
+
+      - name: Run Tests
+        run: python testsuite/main.py -d
+
+      - name: Print logs
+        if: always()
+        run: cat ~/.local/state/superfile/superfile.log

src/cmd/main.go

@@ -69,7 +69,7 @@ func Run(content embed.FS) {
 				Name:    "config-file",
 				Aliases: []string{"c"},
 				Usage:   "Specify the path to a different config file",
-				Value:   variable.ConfigFile, // Default to the existing config file path
+				Value:   "", // Default to the blank string indicating non-usage of flag
 			},
 		},
 		Action: func(c *cli.Context) error {
@@ -80,11 +80,15 @@ func Run(content embed.FS) {
 			}
 
 			// Setting the config file path
-			variable.ConfigFile = c.String("config-file")
+			configFileArg := c.String("config-file")
 
 			// Validate the config file exists
-			if _, err := os.Stat(variable.ConfigFile); os.IsNotExist(err) {
-				log.Fatalf("Error: Configuration file '%s' does not exist", variable.ConfigFile)
+			if configFileArg != "" {
+				if _, err := os.Stat(variable.ConfigFile); err != nil {
+					log.Fatalf("Error: While reading config file '%s' from arguement : %v", configFileArg, err)
+				} else {
+					variable.ConfigFile = configFileArg
+				}
 			}
 
 			InitConfigFile()

src/internal/config_function.go

@@ -315,6 +315,12 @@ func LoadAllDefaultConfig(content embed.FS) {
 		}
 	}
 
+	// Prevent failure for first time app run by making sure parent directories exists
+	if err = os.MkdirAll(filepath.Dir(variable.ThemeFileVersion), 0755); err != nil {
+		slog.Error("error creating theme file parent directory", "error", err)
+		return
+	}
+
 	err = os.WriteFile(variable.ThemeFileVersion, []byte(variable.CurrentVersion), 0644)
 	if err != nil {
 		slog.Error("error writing theme file version", "error", err)

testsuite/ReadMe.md

@@ -71,7 +71,7 @@ Notes
 
 ## Tips while running tests
 - Use `-d` or `--debug` to enable debug logs during test run.
-- If you see flakiness in test runs due to superfile being still open, consider using `--close-wait-time` options to increase wait time for superfile to close
+- If you see flakiness in test runs due to superfile being still open, consider using `--close-wait-time` options to increase wait time for superfile to close. Note : For now we have enforing superfile to close within a specific time window in tests to reduce test flakiness 
 - Make sure that your hotkeys are set to default hotkeys. Tests use default hotkeys for now.
 - Use `-t` or `--tests` to only run specific tests
   - Example `python main.py -d -t RenameTest CopyTest`

testsuite/core/base_test.py

@@ -33,16 +33,23 @@ def validate(self) -> bool:
         Returns:
             bool: True if validation passed
         """
+    @abstractmethod
+    def cleanup(self) -> None:
+        """Any required cleanup after test is done
+        """
+
 
 class GenericTestImpl(BaseTest):
     def __init__(self, test_env : Environment,
         test_root : Path,
         start_dir : Path,
         test_dirs : List[Path],
-        test_files : List[Tuple[Path, str]],
-        key_inputs : List[Union[keys.Keys,str]],
-        validate_exists : List[Path] = [],
-        validate_not_exists : List[Path] = []):
+        key_inputs : List[Union[keys.Keys,str]] = None,
+        test_files : List[Tuple[Path, str]] = None,
+        validate_exists : List[Path] = None,
+        validate_not_exists : List[Path] = None,
+        validate_spf_closed: bool = False,
+        close_wait_time : float = tconst.CLOSE_WAIT_TIME ):
         super().__init__(test_env)
         self.test_root = test_root
         self.start_dir = start_dir
@@ -51,37 +58,46 @@ def __init__(self, test_env : Environment,
         self.key_inputs = key_inputs
         self.validate_exists = validate_exists
         self.validate_not_exists = validate_not_exists
+        self.validate_spf_closed = validate_spf_closed
+        self.close_wait_time = close_wait_time
 
     def setup(self) -> None:
         for dir_path in self.test_dirs:
             self.env.fs_mgr.makedirs(dir_path)
-        for file_path, data in self.test_files:
-            self.env.fs_mgr.create_file(file_path, data)
+
+        if self.test_files is not None:
+            for file_path, data in self.test_files:
+                self.env.fs_mgr.create_file(file_path, data)
 
         self.logger.debug("Current file structure : \n%s",
             self.env.fs_mgr.tree(self.test_root))
+
+
+    def start_spf(self) -> None:
+        self.env.spf_mgr.start_spf(self.env.fs_mgr.abspath(self.start_dir))
+        assert self.env.spf_mgr.is_spf_running(), "Superfile is not running"
 
+    def end_execution(self) -> None:
+        self.env.spf_mgr.send_special_input(keys.KEY_ESC)    
+        time.sleep(self.close_wait_time)
+        self.logger.debug("Finished Execution")
 
     def test_execute(self) -> None:
         """Execute the test
         """
-        # Start in DIR1
-        self.env.spf_mgr.start_spf(self.env.fs_mgr.abspath(self.start_dir))
-
-        assert self.env.spf_mgr.is_spf_running(), "Superfile is not running"
-
-        for cur_input in self.key_inputs:
-            if isinstance(cur_input, keys.Keys):
-                self.env.spf_mgr.send_special_input(cur_input)
-            else:
-                assert isinstance(cur_input, str), "Invalid input type"
-                self.env.spf_mgr.send_text_input(cur_input)
-            time.sleep(tconst.KEY_DELAY)
+        self.start_spf()
+        if self.key_inputs is not None:
+            for cur_input in self.key_inputs:
+                if isinstance(cur_input, keys.Keys):
+                    self.env.spf_mgr.send_special_input(cur_input)
+                else:
+                    assert isinstance(cur_input, str), "Invalid input type"
+                    self.env.spf_mgr.send_text_input(cur_input)
+                time.sleep(tconst.KEY_DELAY)
 
         time.sleep(tconst.OPERATION_DELAY)
-        self.env.spf_mgr.send_special_input(keys.KEY_ESC)    
-        time.sleep(tconst.CLOSE_WAIT_TIME)
-        self.logger.debug("Finished Execution")
+        self.end_execution()
+
 
     def validate(self) -> bool:
         """Validate that test passed. Log exception if failed.
@@ -91,17 +107,26 @@ def validate(self) -> bool:
         self.logger.debug("spf_manager info : %s, Current file structure : \n%s",
             self.env.spf_mgr.runtime_info(), self.env.fs_mgr.tree(self.test_root))
         try:
-            assert not self.env.spf_mgr.is_spf_running(), "Superfile is still running"
-            for file_path in self.validate_exists:
-                assert self.env.fs_mgr.check_exists(file_path), f"File {file_path} does not exists"
-
-            for file_path in self.validate_not_exists:
-                assert not self.env.fs_mgr.check_exists(file_path), f"File {file_path} exists" 
+            if self.validate_spf_closed :
+                assert not self.env.spf_mgr.is_spf_running(), "Superfile is still running"
+
+            if self.validate_exists is not None:
+                for file_path in self.validate_exists:
+                    assert self.env.fs_mgr.check_exists(file_path), f"File {file_path} does not exists"
+
+            if self.validate_not_exists is not None:
+                for file_path in self.validate_not_exists:
+                    assert not self.env.fs_mgr.check_exists(file_path), f"File {file_path} exists" 
         except AssertionError as ae:
             self.logger.debug("Test assertion failed : %s", ae, exc_info=True)
             return False
 
         return True
+
+    def cleanup(self) -> None:
+        # Cleanup after test is done
+        if self.env.spf_mgr.is_spf_running():
+            self.env.spf_mgr.close_spf()
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}"

testsuite/core/keys.py

@@ -28,8 +28,10 @@ def __init__(self, ascii_code : int, key_name : str):
 
 KEY_CTRL_A : Keys = CtrlKeys('a')
 KEY_CTRL_C : Keys = CtrlKeys('c')
+KEY_CTRL_E : Keys = CtrlKeys('e')
 KEY_CTRL_D : Keys = CtrlKeys('d')
 KEY_CTRL_M : Keys = CtrlKeys('m')
+KEY_CTRL_P : Keys = CtrlKeys('p')
 KEY_CTRL_R : Keys = CtrlKeys('r')
 KEY_CTRL_V : Keys = CtrlKeys('v')
 KEY_CTRL_W : Keys = CtrlKeys('w')

testsuite/core/runner.py

@@ -37,8 +37,17 @@ def get_testcases(test_env : Environment, only_run_tests : List[str] = None) ->
                 res.append(attr(test_env))
     return res
 
+def run_tests(spf_path : Path, stop_on_fail : bool = True, only_run_tests : List[str] = None) -> bool:
+    """Runs tests
 
-def run_tests(spf_path : Path, stop_on_fail : bool = True, only_run_tests : List[str] = None) -> None:
+    Args:
+        spf_path (Path): Path of spf binary under test
+        stop_on_fail (bool, optional): Whether to stop on failures. Defaults to True.
+        only_run_tests (List[str], optional): Only specific test to run. Defaults to None.
+
+    Returns:
+        bool: Whether run was successful
+    """    
     # is this str conversion needed ?
 
     spf_manager : BaseSPFManager = None 
@@ -50,19 +59,20 @@ def run_tests(spf_path : Path, stop_on_fail : bool = True, only_run_tests : List
     fs_manager = TestFSManager()
 
     test_env = Environment(spf_manager, fs_manager)
-
-    try:
-        cnt_passed : int = 0
-        cnt_executed : int = 0
+    cnt_passed : int = 0
+    cnt_executed : int = 0
+    try:    
         testcases : List[BaseTest] = get_testcases(test_env, only_run_tests=only_run_tests)
         logger.info("Testcases : %s", testcases)
         for t in testcases:
             logger.info("Running test %s", t)
             t.setup()
             t.test_execute()
             cnt_executed += 1
+            passed : bool = t.validate()
+            t.cleanup()
 
-            if t.validate():
+            if passed:
                 logger.info("Passed test %s", t)
                 cnt_passed += 1
             else:
@@ -76,4 +86,7 @@ def run_tests(spf_path : Path, stop_on_fail : bool = True, only_run_tests : List
         # This is still not full proof, as if what happens when TestFSManager __init__ fails ?
         test_env.cleanup()
 
+    return cnt_passed == cnt_executed
+
+
 

testsuite/core/test_constants.py

@@ -4,4 +4,5 @@
 OPERATION_DELAY : float = 0.3 # seconds
 
 # 0.3 second was too less for windows
-CLOSE_WAIT_TIME : float     = 1 # seconds
+# 0.5 second Github workflow failed for with superfile is still running errors
+CLOSE_WAIT_TIME : float     = 0.5 # seconds

testsuite/core/tmux_manager.py

@@ -20,6 +20,7 @@ def __init__(self, spf_path : str):
         super().__init__(spf_path)
         self.logger = logging.getLogger()
         self.server = libtmux.Server(socket_name=TmuxSPFManager.SPF_SOCKET_NAME)
+        self.logger.debug("server object : %s", self.server)
         self.spf_session : libtmux.Session = None
         self.spf_pane : libtmux.Pane = None
 
@@ -28,6 +29,7 @@ def start_spf(self, start_dir : str = None) -> None:
                 window_command=self.spf_path, 
                 start_directory=start_dir)
         time.sleep(TmuxSPFManager.SPF_START_DELAY)
+        self.logger.debug("spf_session initialised : %s", self.spf_session)
 
         self.spf_pane = self.spf_session.active_pane
         self._is_spf_running = True

testsuite/core/utils.py

@@ -0,0 +1,8 @@
+import pyperclip
+# ------ Clipboard utils
+
+# This creates a layer of abstraction.
+# Now the user of the fuction doesn't need to import pyperclip
+# or need to even know what pyperclip was used.
+def get_sys_clipboard_text() -> str :
+    return pyperclip.paste()

testsuite/main.py

@@ -55,7 +55,10 @@ def main():
     # Resolve any symlinks, and make it absolute
     spf_path = spf_path.resolve()
 
-    run_tests(spf_path, only_run_tests=args.tests)
-
+    success = run_tests(spf_path, only_run_tests=args.tests)
+    if success:
+        sys.exit(0)
+    else:
+        sys.exit(1)
 
 main()

testsuite/requirements.txt

@@ -1,2 +1,4 @@
-pyautogui
-libtmux
+pyautogui; sys_platform == "windows"
+libtmux; sys_platform == "linux" or sys_platform == "darwin"
+pyperclip
+assertpy

testsuite/tests/command_test.py

@@ -0,0 +1,22 @@
+from pathlib import Path
+
+from core.base_test import GenericTestImpl
+from core.environment import Environment
+import core.keys as keys
+
+TESTROOT = Path("cmd_ops")
+DIR1 = TESTROOT / "dir1"
+FILE1 = TESTROOT / "file1"
+
+class CommandTest(GenericTestImpl):
+    """Test compression and extraction
+    """
+    def __init__(self, test_env : Environment):
+        super().__init__(
+            test_env=test_env,
+            test_root=TESTROOT,
+            start_dir=TESTROOT,
+            test_dirs=[TESTROOT],
+            key_inputs=[':', 'mkdir dir1', keys.KEY_ENTER, ':', 'touch file1', keys.KEY_ENTER],
+            validate_exists=[DIR1, FILE1]
+        )