Identifying duplicate pages is a critical task for maintaining site quality and improving SEO. **Duplicate content** can confuse search engines, dilute ranking signals, and negatively impact your site's visibility. By detecting and managing duplicate pages, you help ensure that your website is optimized for both users and search engines.


import unittest
import user_code
import ast
import re   
import importlib
import csv
import unittest
import importlib

class TestTask(unittest.TestCase):
    def test_simple_duplicates(self):
        import user_code
        importlib.reload(user_code)
        pages = ["Welcome to our site", "About us", "Contact", "Welcome to our site"]
        result = user_code.find_duplicate_pages(pages)
        expected = [3]
        _dynamic_test(
            self,
            result == expected,
            f"Correctly finds duplicate at index 3: {result}",
            f"Expected {expected}, got {result}",
        )

    def test_multiple_duplicates(self):
        import user_code
        importlib.reload(user_code)
        pages = ["Home", "Services", "Blog", "Contact", "Blog"]
        result = user_code.find_duplicate_pages(pages)
        expected = [4]
        _dynamic_test(
            self,
            result == expected,
            f"Correctly finds duplicate at index 4: {result}",
            f"Expected {expected}, got {result}",
        )

    def test_no_duplicates(self):
        import user_code
        importlib.reload(user_code)
        pages = ["Unique A", "Unique B", "Unique C"]
        result = user_code.find_duplicate_pages(pages)
        expected = []
        _dynamic_test(
            self,
            result == expected,
            f"Correctly returns empty list when no duplicates: {result}",
            f"Expected {expected}, got {result}",
        )

    def test_many_duplicates(self):
        import user_code
        importlib.reload(user_code)
        pages = ["A", "B", "A", "C", "B", "A"]
        result = user_code.find_duplicate_pages(pages)
        expected = [2, 4, 5]
        _dynamic_test(
            self,
            result == expected,
            f"Correctly finds multiple duplicates in order: {result}",
            f"Expected {expected}, got {result}",
        )

    def test_duplicates_are_not_first_occurrence(self):
        import user_code
        importlib.reload(user_code)
        pages = ["X", "Y", "Z", "X", "Y", "X"]
        result = user_code.find_duplicate_pages(pages)
        expected = [3, 4, 5]
        _dynamic_test(
            self,
            result == expected,
            f"Does not include first occurrences, only later duplicates: {result}",
            f"Expected {expected}, got {result}",
        )

    def test_empty_list(self):
        import user_code
        importlib.reload(user_code)
        pages = []
        result = user_code.find_duplicate_pages(pages)
        expected = []
        _dynamic_test(
            self,
            result == expected,
            "Correctly returns empty list for empty input.",
            f"Expected empty list, got {result}",
        )

def _dynamic_test(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
        test_case.assertTrue(True, success_message)
    else:
        test_case._testMethodName = failure_message
        test_case.fail(failure_message)

def normalize_text(text):
    text = text.lower()
    text = re.sub(r"\\s{2,}", " ", text)
    text = re.sub(r"\\s*([,:?])\\s*", r"\\1 ", text)
    return text.strip()

def change_var(code: str, var_name: str, value: str) -> str:
    tree = ast.parse(code)
    lines = code.splitlines()
    changed = False
    # Collect all assignment nodes to modify
    assign_nodes = [
        (i, node)
        for i, node in enumerate(tree.body)
        if isinstance(node, ast.Assign)
        and any(isinstance(target, ast.Name) and target.id == var_name for target in node.targets)
    ]

    # If nothing to change, return unmodified code
    if not assign_nodes:
        return code

    # Perform replacements for all matching assignments (from last to first to not break line offsets)
    for i, node in reversed(assign_nodes):
        start_line = node.lineno - 1
        line = lines[start_line]
        indent = ' ' * (len(line) - len(line.lstrip()))
        lines[start_line] = f"{indent}{var_name} = {value}"
        next_line = len(lines)
        for next_node in tree.body[i+1:]:
            if hasattr(next_node, 'lineno'):
                next_line = next_node.lineno - 1
                break
        if next_line > start_line + 1:
            lines[start_line+1:next_line] = []
        changed = True

    return '\\n'.join(lines) if changed else code

if __name__ == "__main__":
    unittest.main()


test_main.py

Master the use of Python to automate, analyze, and optimize SEO workflows. This course guides SEO professionals through practical Python applications for keyword research, on-page analysis, and technical SEO, using real-world scenarios and hands-on coding tasks.

Learn how Python can automate repetitive SEO tasks, from scraping search results to generating meta tags, making your SEO workflow more efficient.

Explore how Python can be used to analyze keyword data, identify opportunities, and visualize keyword trends for SEO campaigns.

Apply Python to analyze on-page SEO elements and perform technical audits, including content checks, tag analysis, and site structure evaluation.

Challenge: Find Duplicate Pages

Solution

Challenge: Find Duplicate Pages

Solution