diff --git a/python_files/tests/pytestadapter/.data/same_function_new_class_param.py b/python_files/tests/pytestadapter/.data/same_function_new_class_param.py new file mode 100644 index 000000000000..6f85051436b8 --- /dev/null +++ b/python_files/tests/pytestadapter/.data/same_function_new_class_param.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import pytest + + +class TestNotEmpty: + @pytest.mark.parametrize("a, b", [(1, 1), (2, 2)]) # test_marker--TestNotEmpty::test_integer + def test_integer(self, a, b): + assert a == b + + @pytest.mark.parametrize( # test_marker--TestNotEmpty::test_string + "a, b", [("a", "a"), ("b", "b")] + ) + def test_string(self, a, b): + assert a == b + + +class TestEmpty: + @pytest.mark.parametrize("a, b", [(0, 0)]) # test_marker--TestEmpty::test_integer + def test_integer(self, a, b): + assert a == b + + @pytest.mark.parametrize("a, b", [("", "")]) # test_marker--TestEmpty::test_string + def test_string(self, a, b): + assert a == b diff --git a/python_files/tests/pytestadapter/expected_discovery_test_output.py b/python_files/tests/pytestadapter/expected_discovery_test_output.py index d5db7589cca1..3ddceeb060ac 100644 --- a/python_files/tests/pytestadapter/expected_discovery_test_output.py +++ b/python_files/tests/pytestadapter/expected_discovery_test_output.py @@ -541,7 +541,7 @@ "name": "test_adding", "path": os.fspath(parameterize_tests_path), "type_": "function", - "id_": "parametrize_tests.py::TestClass::test_adding", + "id_": os.fspath(parameterize_tests_path) + "::TestClass::test_adding", "children": [ { "name": "[3+5-8]", @@ -638,7 +638,7 @@ ), }, ], - "id_": "parametrize_tests.py::test_string", + "id_": os.fspath(parameterize_tests_path) + "::test_string", }, ], }, @@ -760,7 +760,7 @@ ), }, ], - "id_": "param_same_name/test_param1.py::test_odd_even", + "id_": os.fspath(param1_path) + "::test_odd_even", } ], }, @@ -818,7 +818,7 @@ ), }, ], - "id_": "param_same_name/test_param2.py::test_odd_even", + "id_": os.fspath(param2_path) + "::test_odd_even", } ], }, @@ -1077,3 +1077,200 @@ ], "id_": str(SYMLINK_FOLDER_PATH), } + +same_function_new_class_param_expected_output = { + "name": ".data", + "path": TEST_DATA_PATH_STR, + "type_": "folder", + "children": [ + { + "name": "same_function_new_class_param.py", + "path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"), + "type_": "file", + "id_": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"), + "children": [ + { + "name": "TestNotEmpty", + "path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"), + "type_": "class", + "children": [ + { + "name": "test_integer", + "path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"), + "type_": "function", + "children": [ + { + "name": "[1-1]", + "path": os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + "lineno": find_test_line_number( + "TestNotEmpty::test_integer", + os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + ), + "type_": "test", + "id_": get_absolute_test_id( + "same_function_new_class_param.py::TestNotEmpty::test_integer[1-1]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + "runID": get_absolute_test_id( + "same_function_new_class_param.py::TestNotEmpty::test_integer[1-1]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + }, + { + "name": "[2-2]", + "path": os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + "lineno": find_test_line_number( + "TestNotEmpty::test_integer", + os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + ), + "type_": "test", + "id_": get_absolute_test_id( + "same_function_new_class_param.py::TestNotEmpty::test_integer[2-2]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + "runID": get_absolute_test_id( + "same_function_new_class_param.py::TestNotEmpty::test_integer[2-2]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + }, + ], + "id_": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py") + + "::TestNotEmpty::test_integer", + }, + { + "name": "test_string", + "path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"), + "type_": "function", + "children": [ + { + "name": "[a-a]", + "path": os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + "lineno": find_test_line_number( + "TestNotEmpty::test_string", + os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + ), + "type_": "test", + "id_": get_absolute_test_id( + "same_function_new_class_param.py::TestNotEmpty::test_string[a-a]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + "runID": get_absolute_test_id( + "same_function_new_class_param.py::TestNotEmpty::test_string[a-a]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + }, + { + "name": "[b-b]", + "path": os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + "lineno": find_test_line_number( + "TestNotEmpty::test_string", + os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + ), + "type_": "test", + "id_": get_absolute_test_id( + "same_function_new_class_param.py::TestNotEmpty::test_string[b-b]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + "runID": get_absolute_test_id( + "same_function_new_class_param.py::TestNotEmpty::test_string[b-b]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + }, + ], + "id_": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py") + + "::TestNotEmpty::test_string", + }, + ], + "id_": "same_function_new_class_param.py::TestNotEmpty", + }, + { + "name": "TestEmpty", + "path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"), + "type_": "class", + "children": [ + { + "name": "test_integer", + "path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"), + "type_": "function", + "children": [ + { + "name": "[0-0]", + "path": os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + "lineno": find_test_line_number( + "TestEmpty::test_integer", + os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + ), + "type_": "test", + "id_": get_absolute_test_id( + "same_function_new_class_param.py::TestEmpty::test_integer[0-0]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + "runID": get_absolute_test_id( + "same_function_new_class_param.py::TestEmpty::test_integer[0-0]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + }, + ], + "id_": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py") + + "::TestEmpty::test_integer", + }, + { + "name": "test_string", + "path": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py"), + "type_": "function", + "children": [ + { + "name": "[-]", + "path": os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + "lineno": find_test_line_number( + "TestEmpty::test_string", + os.fspath( + TEST_DATA_PATH / "same_function_new_class_param.py" + ), + ), + "type_": "test", + "id_": get_absolute_test_id( + "same_function_new_class_param.py::TestEmpty::test_string[-]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + "runID": get_absolute_test_id( + "same_function_new_class_param.py::TestEmpty::test_string[-]", + TEST_DATA_PATH / "same_function_new_class_param.py", + ), + }, + ], + "id_": os.fspath(TEST_DATA_PATH / "same_function_new_class_param.py") + + "::TestEmpty::test_string", + }, + ], + "id_": "same_function_new_class_param.py::TestEmpty", + }, + ], + } + ], + "id_": TEST_DATA_PATH_STR, +} + +print(param_same_name_expected_output) diff --git a/python_files/tests/pytestadapter/test_discovery.py b/python_files/tests/pytestadapter/test_discovery.py index e27750dc174c..24960b91c644 100644 --- a/python_files/tests/pytestadapter/test_discovery.py +++ b/python_files/tests/pytestadapter/test_discovery.py @@ -113,6 +113,10 @@ def test_parameterized_error_collect(): "test_multi_class_nest.py", expected_discovery_test_output.nested_classes_expected_test_output, ), + ( + "same_function_new_class_param.py", + expected_discovery_test_output.same_function_new_class_param_expected_output, + ), ( "test_multi_class_nest.py", expected_discovery_test_output.nested_classes_expected_test_output, @@ -187,7 +191,9 @@ def test_pytest_collect(file, expected_const): ), f"Status is not 'success', error is: {actual_item.get('error')}" assert actual_item.get("cwd") == os.fspath(helpers.TEST_DATA_PATH) assert is_same_tree( - actual_item.get("tests"), expected_const + actual_item.get("tests"), + expected_const, + ["id_", "lineno", "name", "runID"], ), f"Tests tree does not match expected value. \n Expected: {json.dumps(expected_const, indent=4)}. \n Actual: {json.dumps(actual_item.get('tests'), indent=4)}" @@ -255,6 +261,7 @@ def test_pytest_root_dir(): assert is_same_tree( actual_item.get("tests"), expected_discovery_test_output.root_with_config_expected_output, + ["id_", "lineno", "name", "runID"], ), f"Tests tree does not match expected value. \n Expected: {json.dumps(expected_discovery_test_output.root_with_config_expected_output, indent=4)}. \n Actual: {json.dumps(actual_item.get('tests'), indent=4)}" @@ -281,6 +288,7 @@ def test_pytest_config_file(): assert is_same_tree( actual_item.get("tests"), expected_discovery_test_output.root_with_config_expected_output, + ["id_", "lineno", "name", "runID"], ), f"Tests tree does not match expected value. \n Expected: {json.dumps(expected_discovery_test_output.root_with_config_expected_output, indent=4)}. \n Actual: {json.dumps(actual_item.get('tests'), indent=4)}" diff --git a/python_files/tests/tree_comparison_helper.py b/python_files/tests/tree_comparison_helper.py index edf6aa8ff869..3d9d1d39194b 100644 --- a/python_files/tests/tree_comparison_helper.py +++ b/python_files/tests/tree_comparison_helper.py @@ -1,29 +1,39 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - - -def is_same_tree(tree1, tree2) -> bool: - """Helper function to test if two test trees are the same. +def is_same_tree(tree1, tree2, test_key_arr, path="root") -> bool: + """Helper function to test if two test trees are the same with detailed error logs. `is_same_tree` starts by comparing the root attributes, and then checks if all children are the same. """ # Compare the root. - if any(tree1[key] != tree2[key] for key in ["path", "name", "type_"]): - return False + for key in ["path", "name", "type_", "id_"]: + if tree1.get(key) != tree2.get(key): + print( + f"Difference found at {path}: '{key}' is '{tree1.get(key)}' in tree1 and '{tree2.get(key)}' in tree2." + ) + return False # Compare child test nodes if they exist, otherwise compare test items. if "children" in tree1 and "children" in tree2: - # sort children by path before comparing since order doesn't matter of children + # Sort children by path before comparing since order doesn't matter of children children1 = sorted(tree1["children"], key=lambda x: x["path"]) children2 = sorted(tree2["children"], key=lambda x: x["path"]) # Compare test nodes. if len(children1) != len(children2): + print( + f"Difference in number of children at {path}: {len(children1)} in tree1 and {len(children2)} in tree2." + ) return False else: - return all(is_same_tree(*pair) for pair in zip(children1, children2)) + for i, (child1, child2) in enumerate(zip(children1, children2)): + if not is_same_tree(child1, child2, test_key_arr, path=f"{path} -> child {i}"): + return False elif "id_" in tree1 and "id_" in tree2: # Compare test items. - return all(tree1[key] == tree2[key] for key in ["id_", "lineno"]) + for key in test_key_arr: + if tree1.get(key) != tree2.get(key): + print( + f"Difference found at {path}: '{key}' is '{tree1.get(key)}' in tree1 and '{tree2.get(key)}' in tree2." + ) + return False - return False + return True diff --git a/python_files/tests/unittestadapter/test_discovery.py b/python_files/tests/unittestadapter/test_discovery.py index 74eb5a5fb4f3..94d0bb89c62a 100644 --- a/python_files/tests/unittestadapter/test_discovery.py +++ b/python_files/tests/unittestadapter/test_discovery.py @@ -129,7 +129,7 @@ def test_simple_discovery() -> None: actual = discover_tests(start_dir, pattern, None) assert actual["status"] == "success" - assert is_same_tree(actual.get("tests"), expected) + assert is_same_tree(actual.get("tests"), expected, ["id_", "lineno", "name"]) assert "error" not in actual @@ -185,7 +185,7 @@ def test_simple_discovery_with_top_dir_calculated() -> None: actual = discover_tests(start_dir, pattern, None) assert actual["status"] == "success" - assert is_same_tree(actual.get("tests"), expected) + assert is_same_tree(actual.get("tests"), expected, ["id_", "lineno", "name"]) assert "error" not in actual @@ -256,7 +256,7 @@ def test_error_discovery() -> None: actual = discover_tests(start_dir, pattern, None) assert actual["status"] == "error" - assert is_same_tree(expected, actual.get("tests")) + assert is_same_tree(expected, actual.get("tests"), ["id_", "lineno", "name"]) assert len(actual.get("error", [])) == 1 @@ -274,6 +274,7 @@ def test_unit_skip() -> None: assert is_same_tree( actual.get("tests"), expected_discovery_test_output.skip_unittest_folder_discovery_output, + ["id_", "lineno", "name"], ) assert "error" not in actual @@ -296,4 +297,5 @@ def test_complex_tree() -> None: assert is_same_tree( actual.get("tests"), expected_discovery_test_output.complex_tree_expected_output, + ["id_", "lineno", "name"], ) diff --git a/python_files/tests/unittestadapter/test_utils.py b/python_files/tests/unittestadapter/test_utils.py index f650f12252f7..1cb9a4686399 100644 --- a/python_files/tests/unittestadapter/test_utils.py +++ b/python_files/tests/unittestadapter/test_utils.py @@ -110,7 +110,7 @@ def test_get_existing_child_node() -> None: tree_copy = tree.copy() # Check that the tree didn't get mutated by get_child_node. - assert is_same_tree(tree, tree_copy) + assert is_same_tree(tree, tree_copy, ["id_", "lineno", "name"]) def test_no_existing_child_node() -> None: @@ -164,7 +164,7 @@ def test_no_existing_child_node() -> None: tree_after["children"] = tree_after["children"][:-1] # Check that all pre-existing items in the tree didn't get mutated by get_child_node. - assert is_same_tree(tree_before, tree_after) + assert is_same_tree(tree_before, tree_after, ["id_", "lineno", "name"]) # Check for the added node. last_child = tree["children"][-1] @@ -226,7 +226,7 @@ def test_build_simple_tree() -> None: suite = loader.discover(start_dir, pattern) tests, errors = build_test_tree(suite, start_dir) - assert is_same_tree(expected, tests) + assert is_same_tree(expected, tests, ["id_", "lineno", "name"]) assert not errors @@ -286,7 +286,7 @@ def test_build_decorated_tree() -> None: suite = loader.discover(start_dir, pattern) tests, errors = build_test_tree(suite, start_dir) - assert is_same_tree(expected, tests) + assert is_same_tree(expected, tests, ["id_", "lineno", "name"]) assert not errors diff --git a/python_files/vscode_pytest/__init__.py b/python_files/vscode_pytest/__init__.py index 1d855232cbd8..3534bf7c699f 100644 --- a/python_files/vscode_pytest/__init__.py +++ b/python_files/vscode_pytest/__init__.py @@ -440,10 +440,23 @@ def build_test_tree(session: pytest.Session) -> TestNode: # parameterized test cases cut the repetitive part of the name off. parent_part, parameterized_section = test_node["name"].split("[", 1) test_node["name"] = "[" + parameterized_section - parent_path = os.fspath(get_node_path(test_case)) + "::" + parent_part + + first_split = test_case.nodeid.rsplit( + "::", 1 + ) # splits the parameterized test name from the rest of the nodeid + second_split = first_split[0].rsplit( + ".py", 1 + ) # splits the file path from the rest of the nodeid + + class_and_method = second_split[1] + "::" # This has "::" separator at both ends + # construct the parent id, so it is absolute path :: any class and method :: parent_part + parent_id = os.fspath(get_node_path(test_case)) + class_and_method + parent_part + # file, middle, param = test_case.nodeid.rsplit("::", 2) + # parent_id = test_case.nodeid.rsplit("::", 1)[0] + "::" + parent_part + # parent_path = os.fspath(get_node_path(test_case)) + "::" + parent_part try: function_name = test_case.originalname # type: ignore - function_test_node = function_nodes_dict[parent_path] + function_test_node = function_nodes_dict[parent_id] except AttributeError: # actual error has occurred ERRORS.append( f"unable to find original name for {test_case.name} with parameterization detected." @@ -451,9 +464,9 @@ def build_test_tree(session: pytest.Session) -> TestNode: raise VSCodePytestError("Unable to find original name for parameterized test case") except KeyError: function_test_node: TestNode = create_parameterized_function_node( - function_name, get_node_path(test_case), test_case.nodeid + function_name, get_node_path(test_case), parent_id ) - function_nodes_dict[parent_path] = function_test_node + function_nodes_dict[parent_id] = function_test_node function_test_node["children"].append(test_node) # Check if the parent node of the function is file, if so create/add to this file node. if isinstance(test_case.parent, pytest.File): @@ -643,17 +656,16 @@ def create_class_node(class_module: pytest.Class) -> TestNode: def create_parameterized_function_node( - function_name: str, test_path: pathlib.Path, test_id: str + function_name: str, test_path: pathlib.Path, function_id: str ) -> TestNode: """Creates a function node to be the parent for the parameterized test nodes. Keyword arguments: function_name -- the name of the function. test_path -- the path to the test file. - test_id -- the id of the test, which is a parameterized test so it + function_id -- the previously constructed function id that fits the pattern- absolute path :: any class and method :: parent_part must be edited to get a unique id for the function node. """ - function_id: str = test_id.split("::")[0] + "::" + function_name return { "name": function_name, "path": test_path,