diff --git a/README.md b/README.md index 906f854..ca01e11 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ pip install "dyn2py @ git+https://github.com/infeeeee/dyn2py" ``` > dyn2py --help -usage: dyn2py [-h] [-l LOGLEVEL] [-n] [-F] [-b] [-f {py,dyn}] [-u] [-p path/to/folder] source +usage: dyn2py [-h] [-v] [-l LOGLEVEL] [-n] [-F] [-b] [-f {py,dyn}] [-u] [-p path/to/folder] [source ...] Extract python code from Dynamo graphs @@ -49,6 +49,7 @@ dynamo options, only for processing Dynamo graphs: The script by default overwrites older files with newer files. Do not move the source Dynamo graphs, or update won't work with them later. +Multiple sources are supported, separate them by spaces. ``` ### As a python module @@ -61,7 +62,8 @@ Most basic example to extract all nodes next to a dynamo file: import dyn2py dynamo_file = dyn2py.DynamoFile("path/to/dynamofile.dyn") -dynamo_file.extract_python() +python_files = dynamo_file.extract_python() +[python_file.write() for python_file in python_files] ``` Change options like with the command line switches with the `Options` class: @@ -78,9 +80,7 @@ dynamo_file = dyn2py.DynamoFile("path/to/dynamofile.dyn") python_files = dynamo_file.get_related_python_files(options) # Read python files and update the graph: -if python_files: - for python_file in python_files: - python_file.update_dynamo(options) +[python_file.update_dynamo(options) for python_file in python_files] # Don't forget to save at the end: dynamo_file.write(options) @@ -115,9 +115,9 @@ pip install -e .[build] pyinstaller dyn2py.spec ``` -### Generate module documentation +### Live module documentation ``` pip install -e .[doc] -pdoc -d google -o docs dyn2py +pdoc -d google dyn2py ``` diff --git a/TODO.md b/TODO.md index 841e78d..afe610c 100644 --- a/TODO.md +++ b/TODO.md @@ -16,7 +16,7 @@ - [ ] Windows Build - [ ] Windows Installer - [ ] Pip -- [ ] Winget + ## Documentation @@ -27,4 +27,5 @@ ## Extra features maybe later -- [ ] Autocomplete \ No newline at end of file +- [ ] Autocomplete +- [ ] Winget \ No newline at end of file diff --git a/dyn2py/__init__.py b/dyn2py/__init__.py index ff473be..8430abe 100644 --- a/dyn2py/__init__.py +++ b/dyn2py/__init__.py @@ -8,6 +8,7 @@ import pathlib from importlib_metadata import metadata import textwrap import logging +import inspect from dyn2py.files import * from dyn2py.options import * @@ -32,100 +33,129 @@ def __dir__(): return __all__ -def run(options: Options | None = None) -> None: +def __command_line() -> None: + """Private method for running as a console script""" + + parser = argparse.ArgumentParser( + prog=METADATA["Name"], + description=METADATA["Summary"], + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent("""\ + The script by default overwrites older files with newer files. + Do not move the source Dynamo graphs, or update won't work with them later. + Multiple sources are supported, separate them by spaces. + """) + ) + + parser.add_argument("-v", "--version", + action="version", + version=f'{METADATA["Name"]} {METADATA["Version"]}' + ) + + parser.add_argument("-l", "--loglevel", + metavar="LOGLEVEL", + choices=LOGLEVELS, + default=DEFAULT_LOGLEVEL, + help=f"set log level, possible options: {', '.join(LOGLEVELS)} ") + + parser.add_argument("-n", "--dry-run", + help="do not modify files, only show log", + action="store_true" + ) + + parser.add_argument("-F", "--force", + help="overwrite even if the files are older", + action="store_true") + + parser.add_argument("-b", "--backup", + help="create a backup for updated files", + action="store_true") + + parser.add_argument("-f", "--filter", + choices=FILTERS, + help="only check python or Dynamo graphs, skip the others, useful for folders" + ) + + dynamo_options = parser.add_argument_group( + title="dynamo options, only for processing Dynamo graphs") + + dynamo_options.add_argument("-u", "--update", + help="update Dynamo graph from python scripts in the same folder", + action="store_true") + + dynamo_options.add_argument("-p", "--python-folder", + metavar="path/to/folder", + help="extract python scripts to this folder, read python scripts from here with --update", + type=pathlib.Path) + + parser.add_argument("source", + type=pathlib.Path, + help="path to a Dynamo graph, a python script or a folder containing them", + nargs="*" + ) + + options = parser.parse_args(namespace=Options()) + + run(options) + + +def run(options: Options) -> None: """Run an extraction as from the command line Args: options (Options): Options as from the command line. Raises: + TypeError: options is not an Options object FileNotFoundError: If the source file does not exist """ - if not options: + if not isinstance(options, Options): + raise TypeError("Options have to be a dyn2py.Options() object!") - parser = argparse.ArgumentParser( - prog=METADATA["Name"], - description=METADATA["Summary"], - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=textwrap.dedent("""\ - The script by default overwrites older files with newer files. - Do not move the source Dynamo graphs, or update won't work with them later. - """) - ) - - parser.add_argument("-v", "--version", - action="version", - version=f'{METADATA["Name"]} {METADATA["Version"]}' - ) - - parser.add_argument("-l", "--loglevel", - metavar="LOGLEVEL", - choices=LOGLEVELS, - default=DEFAULT_LOGLEVEL, - help=f"set log level, possible options: {', '.join(LOGLEVELS)} ") - - parser.add_argument("-n", "--dry-run", - help="do not modify files, only show log", - action="store_true" - ) - - parser.add_argument("-F", "--force", - help="overwrite even if the files are older", - action="store_true") - - parser.add_argument("-b", "--backup", - help="create a backup for updated files", - action="store_true") - - parser.add_argument("-f", "--filter", - choices=FILTERS, - help="only check python or Dynamo graphs, skip the others, useful for folders" - ) - - dynamo_options = parser.add_argument_group( - title="dynamo options, only for processing Dynamo graphs") - - dynamo_options.add_argument("-u", "--update", - help="update Dynamo graph from python scripts in the same folder", - action="store_true") - - dynamo_options.add_argument("-p", "--python-folder", - metavar="path/to/folder", - help="extract python scripts to this folder, read python scripts from here with --update", - type=pathlib.Path) - - parser.add_argument("source", - type=pathlib.Path, - help="path to a Dynamo graph, a python script or a folder containing them", - action="append" - ) - - options = parser.parse_args(namespace=Options()) + from_command_line = bool(inspect.stack()[1].function == "__command_line") # Set up logging: logging.basicConfig(format='%(levelname)s: %(message)s', level=options.loglevel) - logging.debug(options) - logging.debug(f"Parsed arguments: {vars(options)}") + logging.debug(f"Run options: {vars(options)}") # Set up sources: - files = [] + source_files = [] for source in options.source: if not source.exists(): - raise FileNotFoundError(f"Source file does not exist!") + if from_command_line: + # log only if it was called from command line: + logging.error(f"File does not exist: {source}") + exit(1) + else: + raise FileNotFoundError(f"Source file does not exist!") # Get files from folder: elif source.is_dir(): logging.debug(f"Source is a folder") for f in source.iterdir(): - files.append(File(f)) + source_files.append(f) # It's a single file: else: - files.append(File(source)) + source_files.append(source) + + # Create file objects + files = [] + for f in source_files: + try: + files.append(File(f)) + except DynamoFileException as e: + # It's a dynamo1 file + logging.warning(e) + continue + except PythonNodeNotFoundException as e: + # No python node in this file + logging.warning(e) + continue # Dynamo files come first, sort sources: files.sort(key=lambda f: f.extension) @@ -149,21 +179,21 @@ def run(options: Options | None = None) -> None: files = list(python_files) + if not files and from_command_line: + logging.error("No files to process! See previous warnings!") + exit(1) + # Cycle through files: for f in files: if f.is_dynamo_file(): logging.debug("Source is a Dynamo file") - - try: - f.extract_python(options) - except DynamoFileException as e: - logging.error(f"{e} Skipping") + f.extract_python(options) elif f.is_python_file(): logging.debug("Source is a Python file") f.update_dynamo(options) - # Dynamo files are written only at the end, so they don't get updated too frequently - for f in DynamoFile.open_files: + # Write files at the end: + for f in DynamoFile.open_files | PythonFile.open_files: f.write(options) diff --git a/dyn2py/exceptions.py b/dyn2py/exceptions.py index b2ed827..148d420 100644 --- a/dyn2py/exceptions.py +++ b/dyn2py/exceptions.py @@ -1,10 +1,13 @@ class DynamoFileException(Exception): + """Something wrong in this DynamoFile""" pass class PythonNodeNotFoundException(Exception): + """PythonNode not found""" pass + class PythonNodeException(Exception): pass diff --git a/dyn2py/files.py b/dyn2py/files.py index 2862927..8e107a0 100644 --- a/dyn2py/files.py +++ b/dyn2py/files.py @@ -22,11 +22,12 @@ class File(): """Base class for managing files""" def __init__(self, filepath: pathlib.Path | str, read_from_disk: bool = True) -> None: - """Generate a file object. If the path is correct it will become a DynamoFile or PythonFile object + """Generate a file object. If the path is correct it will become a DynamoFile or PythonFile object. + Calls DynamoFile.read_file() and PythonFile.read_file() Args: filepath (pathlib.Path | str): Path to the python file or Dynamo graph - read_from_disk (bool, optional): Read the file from disk. Set to false to get only metadata. Defaults to True. + read_from_disk (bool, optional): Read the file from disk. False to get only metadata. Defaults to True. """ self.filepath: pathlib.Path @@ -69,7 +70,7 @@ class File(): if self.exists and read_from_disk: self.read_file() - if self.exists and read_from_disk: + if self.exists: logging.debug(f"File exists: {self.filepath}") self.mtime = self.filepath.stat().st_mtime self.mtimeiso = datetime.fromtimestamp(self.mtime).isoformat() @@ -78,11 +79,11 @@ class File(): """Should be implemented in subclasses""" pass - def is_newer(self, other_file: "File") -> bool: + def is_newer(self, other_file: File) -> bool: """Check if this file is newer than the other file Args: - other_file(File): The other file + other_file (File): The other file Returns: bool: True if this file is newer or the other doesn't exist @@ -110,18 +111,21 @@ class File(): """ return bool(self.extension == ".py") - def write(self, options: Options = Options()) -> None: + def write(self, options: Options | None = None) -> None: """Prepare writing file to the disk: create backup, process dry-run, call filetype specific write_file() methods Should be called on subclasses! Args: - options(Options, optional): Run options. Defaults to Options(). + options (Options | None, optional): Run options. Defaults to None. Raises: TypeError: If called on a File object """ + if not options: + options = Options() + # This should only work on subclasses: if type(self).__name__ == "File": raise TypeError("This method shouldn't be called on File objects!") @@ -144,11 +148,16 @@ class File(): f"Should write file, but it's a dry-run: {self.filepath}") else: logging.info(f"Writing file: {self.filepath}") - self.write_file() + self._write_file() - def write_file(self): - """Should be implemented in subclasses""" - pass + def _write_file(self): + """Should be implemented in subclasses + + Raises: + NotImplementedError: If called on a File object + """ + raise NotImplementedError( + "Should be called only on DynamoFile and PythonFile objects!") class DynamoFile(File): @@ -166,14 +175,21 @@ class DynamoFile(File): open_files: set[DynamoFile] = set() """A set of open Dynamo files, before saving. Self added by read()""" - def extract_python(self, options: Options = Options()) -> None: - """Extract and write python files + def extract_python(self, options: Options | None = None) -> list[PythonFile]: + """Extract python files from Dynamo graphs, add them to open_files Args: - options(Options, optional): Run options. Defaults to Options(). + options (Options | None, optional): Run options. Defaults to None. + + Returns: + list[PythonFile]: List of PythonFile objects extracted from this DynamoFile """ + if not options: + options = Options() + logging.info(f"Extracting from file: {self.filepath}") + python_files = [] # Go through nodes in the file: for python_node in self.python_nodes: @@ -190,11 +206,14 @@ class DynamoFile(File): ) if python_file.is_newer(self) and not options.force: - logging.info( + PythonFile.open_files.remove(python_file) + logging.warning( f"Existing file is newer, skipping: {python_file.filepath}") continue - python_file.write(options) + python_files.append(python_file) + + return python_files def read_file(self, reread: bool = False) -> None: """Read Dynamo graph to parameters. Automatically called by __init__() @@ -251,11 +270,11 @@ class DynamoFile(File): dynamo_file=self) self.python_nodes.add(python_node) - def get_python_node_by_id(self, node_id: str) -> "PythonNode": + def get_python_node_by_id(self, node_id: str) -> PythonNode: """Get a PythonNode object from this Dynamo graph, by its id Args: - node_id(str): The id of the python node as string + node_id (str): The id of the python node as string Returns: PythonNode: The PythonNode with the given id @@ -278,7 +297,7 @@ class DynamoFile(File): """Update the code of a PythonNode in this file Args: - python_node(PythonNode): The new node + python_node (PythonNode): The new node Raises: PythonNodeNotFoundException: Existing node not found @@ -303,20 +322,22 @@ class DynamoFile(File): self.modified = True - def write_file(self) -> None: + def _write_file(self) -> None: """Write this file to the disk. Should be called only from File.write()""" with open(self.filepath, "w", encoding="utf-8", newline="") as output_file: json.dump(self.full_dict, output_file, indent=2, use_decimal=True) - def get_related_python_files(self, options: Options = Options()) -> list["PythonFile"]: + def get_related_python_files(self, options: Options | None = None) -> list[PythonFile]: """Get python files exported from this Dynamo file Args: - options(Options, optional): Run options. Defaults to Options(). + options (Options | None, optional): Run options. Defaults to None. Returns: list[PythonFile]: A list of PythonFile objects """ + if not options: + options = Options() # Find the folder of the python files if options.python_folder: @@ -333,11 +354,11 @@ class DynamoFile(File): return related_python_files @staticmethod - def get_open_file_by_uuid(uuid: str) -> "DynamoFile | None": + def get_open_file_by_uuid(uuid: str) -> DynamoFile | None: """Get an open Dynamo graph by its uuid Args: - uuid(str): Uuid of the file + uuid (str): Uuid of the file Returns: DynamoFile: The file. None if not found """ @@ -357,7 +378,7 @@ class PythonFile(File): text: str """Full contents of the file.""" - open_files: set["PythonFile"] = set() + open_files: set[PythonFile] = set() """A set of open Python files.""" def __init__(self, @@ -475,13 +496,16 @@ class PythonFile(File): logging.debug(f"Header data from python file: {self.header_data}") # logging.debug(f"Code from python file: {self.code}") - def update_dynamo(self, options: Options = Options()) -> None: + def update_dynamo(self, options: Options | None = None) -> None: """Update a the source Dynamo graph from this python script Args: - options (Options, optional): Run options. Defaults to Options(). + options (Options | None, optional): Run options. Defaults to None. """ + if not options: + options = Options() + dynamo_file = self.get_source_dynamo_file() new_python_node = PythonNode(python_file=self) @@ -527,7 +551,7 @@ class PythonFile(File): return dynamo_file - def write_file(self) -> None: + def _write_file(self) -> None: """Write this file to the disk. Should be called only from File.write()""" with open(self.filepath, "w", encoding="utf-8", newline="") as output_file: output_file.write(self.text) diff --git a/pyproject.toml b/pyproject.toml index bc37051..febf173 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dyn2py" -version = "0.2.1" +version = "0.3.0" description = "Extract python code from Dynamo graphs" readme = "README.md" requires-python = ">=3.8" @@ -30,7 +30,7 @@ changelog = "https://github.com/infeeeee/dyn2py/releases" [project.scripts] -dyn2py = "dyn2py:run" +dyn2py = "dyn2py:__command_line" [build-system] diff --git a/tests/support.py b/tests/support.py index 16eaf85..2599f7e 100644 --- a/tests/support.py +++ b/tests/support.py @@ -28,7 +28,11 @@ def extract_single_node_dyn(modify_py: bool = False): # Extract py: options = dyn2py.Options(python_folder=OUTPUT_DIR) dyn = dyn2py.DynamoFile(f"{INPUT_DIR}/single_node.dyn") - dyn.extract_python(options) + pythonfiles = dyn.extract_python(options) + pythonfiles[0].write() + + dyn2py.PythonFile.open_files.clear() + dyn2py.DynamoFile.open_files.clear() if modify_py: # Open the extracted file and replace a string: @@ -38,3 +42,6 @@ def extract_single_node_dyn(modify_py: bool = False): if "asd_string" in line: line = line.replace("asd_string", "qwe_string") mod_py.write(line) + + + diff --git a/tests/test_CommandLine.py b/tests/test_CommandLine.py index c3dc351..c638f8a 100644 --- a/tests/test_CommandLine.py +++ b/tests/test_CommandLine.py @@ -1,15 +1,172 @@ import unittest import subprocess +import shutil +import pathlib + +from tests.support import * class TestCommandLine(unittest.TestCase): - def test_help(self): - args = ["-h", "--help"] + # Sources to test: normal dry_run force backup filter update python_folder + # - single dyn file ✅ ✅ ✅ ✅ ✅ + # - multiple dyn files + # - single python file + # - multiple python files + # - single folder + # - multiple folders + + # Options to test: + # dry run + # force + # backup + # filter + # update + # python folder + + def test_help_and_version(self): + + args = ["-h", "--help", "-v", "--version"] for arg in args: p = subprocess.run(f"dyn2py {arg}", capture_output=True, shell=True) - # No error: self.assertFalse(p.stderr) + + dyn_sources = [ + {"filename": "python_nodes.dyn", "py_file_count": 6}, + {"filename": "single_node.dyn", "py_file_count": 1} + ] + + dyn_sources_error = ["dynamo1file.dyn", + "no_python.dyn", + "nonexisting.dyn"] + + @staticmethod + def run_command(args: list = []) -> dict: + argstring = " ".join(args) + process = subprocess.run(f"dyn2py -l WARNING {argstring}", + capture_output=True, shell=True) + + stderr = process.stderr.decode() + + python_files = {p: p.stat().st_mtime for p in pathlib.Path(OUTPUT_DIR).iterdir() + if p.suffix == ".py"} + + output = { + "stderr": stderr, + "python_file_mtimes": python_files + } + + return output + + def test_dyn_error(self): + for s in self.dyn_sources_error: + + cleanup_output_dir() + + if pathlib.Path(f"{INPUT_DIR}/{s}").exists(): + shutil.copy(f"{INPUT_DIR}/{s}", + f"{OUTPUT_DIR}/{s}") + + file_open = self.run_command( + [f"{OUTPUT_DIR}/{s}"]) + + self.assertTrue(bool(file_open["stderr"])) + self.assertEqual( + len(file_open["python_file_mtimes"]), 0) + + def test_single_dyn(self): + + dyn_tests = [] + + for source_dict in self.dyn_sources: + for backup_arg in ["-b", "--backup"]: + for force_arg in ["-F", "--force"]: + for pfolder_option in ["", "-p", "--python-folder"]: + + test_dict = source_dict.copy() + + if pfolder_option: + test_dict["filepath"] = f"{INPUT_DIR}/{source_dict['filename']}" + pfolder_arg = f"{pfolder_option} {OUTPUT_DIR}" + else: + test_dict["filepath"] = f"{OUTPUT_DIR}/{source_dict['filename']}" + pfolder_arg = "" + + test_dict.update({ + "pfolder_arg": pfolder_arg, + "backup_arg": backup_arg, + "force_arg": force_arg + }) + + dyn_tests.append(test_dict) + + for s in dyn_tests: + cleanup_output_dir() + + if not s["pfolder_arg"]: + shutil.copy(f"{INPUT_DIR}/{s['filename']}", + f"{OUTPUT_DIR}/{s['filename']}") + + file_open = self.run_command( + [s["pfolder_arg"], s['filepath']]) + + self.assertFalse(bool(file_open["stderr"])) + self.assertEqual( + len(file_open["python_file_mtimes"]), s["py_file_count"]) + + # Test no overwrite + file_no_overwrite = self.run_command( + [s["pfolder_arg"], s['filepath']]) + + self.assertTrue(bool(file_no_overwrite["stderr"])) + self.assertEqual( + len(file_no_overwrite["python_file_mtimes"]), s["py_file_count"]) + + for p in file_no_overwrite["python_file_mtimes"]: + self.assertEqual( + file_no_overwrite["python_file_mtimes"][p], file_open["python_file_mtimes"][p]) + + # Test force: + file_force = self.run_command( + [s["pfolder_arg"], s["force_arg"], s['filepath']]) + + self.assertFalse(bool(file_force["stderr"])) + self.assertEqual( + len(file_force["python_file_mtimes"]), s["py_file_count"]) + + for p in file_force["python_file_mtimes"]: + self.assertTrue( + file_force["python_file_mtimes"][p] > file_open["python_file_mtimes"][p] + ) + + # Test backup + file_backup = self.run_command( + [s["pfolder_arg"], s["force_arg"], s["backup_arg"], s['filepath']]) + + self.assertFalse(bool(file_backup["stderr"])) + + self.assertEqual( + len(file_backup["python_file_mtimes"]), s["py_file_count"] * 2) + + for p in file_force["python_file_mtimes"]: + self.assertTrue( + file_backup["python_file_mtimes"][p] > file_force["python_file_mtimes"][p] + ) + + def test_single_dyn_dryrun(self): + for s in self.dyn_sources: + for arg in ["-n", "--dry-run"]: + + cleanup_output_dir() + + shutil.copy(f"{INPUT_DIR}/{s['filename']}", + f"{OUTPUT_DIR}/{s['filename']}") + + file_dryrun = self.run_command( + [arg, f"{OUTPUT_DIR}/{s['filename']}"]) + + self.assertFalse(bool(file_dryrun["stderr"])) + self.assertFalse(file_dryrun["python_file_mtimes"]) diff --git a/tests/test_DynamoFile.py b/tests/test_DynamoFile.py index fa1cc13..495948d 100644 --- a/tests/test_DynamoFile.py +++ b/tests/test_DynamoFile.py @@ -3,7 +3,6 @@ import dyn2py import pathlib import shutil import simplejson as json -from dyn2py.files import DynamoFile from tests.support import * @@ -17,7 +16,7 @@ class TestDynamoFile(unittest.TestCase): self.assertEqual(dyn.uuid, "3c3b4c05-9716-4e93-9360-ca0637cb5486") self.assertEqual(dyn.name, "python_nodes") - self.assertTrue(dyn in DynamoFile.open_files) + self.assertTrue(dyn in dyn2py.DynamoFile.open_files) # Dynamo 1 file: with self.assertRaises(dyn2py.DynamoFileException): @@ -45,11 +44,17 @@ class TestDynamoFile(unittest.TestCase): def test_extract_python(self): cleanup_output_dir() + dyn2py.PythonFile.open_files.clear() opt = dyn2py.Options(python_folder=OUTPUT_DIR) dyn = dyn2py.DynamoFile(f"{INPUT_DIR}/python_nodes.dyn") dyn.extract_python(options=opt) + self.assertEqual(len(dyn2py.PythonFile.open_files), 6) + + for f in dyn2py.PythonFile.open_files: + f.write() + output_dir = pathlib.Path(OUTPUT_DIR) self.assertEqual(len(list(output_dir.iterdir())), 6) @@ -72,6 +77,9 @@ class TestDynamoFile(unittest.TestCase): dyn2 = dyn2py.DynamoFile(f"{INPUT_DIR}/single_node.dyn") for dyn in [dyn1, dyn2]: dyn.extract_python(options=opt) + for f in dyn2py.PythonFile.open_files: + f.write() + dyn2py.PythonFile.open_files.clear() python_files1 = dyn1.get_related_python_files(options=opt) python_files2 = dyn2.get_related_python_files(options=opt) diff --git a/tests/test_File.py b/tests/test_File.py index f11b225..c2f791e 100644 --- a/tests/test_File.py +++ b/tests/test_File.py @@ -74,6 +74,8 @@ class TestFile(unittest.TestCase): cleanup_output_dir() opt = dyn2py.Options(python_folder=OUTPUT_DIR) older_file.extract_python(options=opt) # type: ignore + for f in dyn2py.PythonFile.open_files: + f.write() newer_file = dyn2py.File( f"{OUTPUT_DIR}/single_node_1c5d99792882409e97e132b3e9f814b0.py")