Extracting titles from SERP (Search Engine Results Page) HTML is a common automation task for SEO monitoring. As an SEO specialist, you often need to collect the titles of search results to analyze ranking changes, competitor content, or simply to track how your pages appear in search results. Automating this process with Python saves time and ensures accuracy, especially when dealing with large volumes of data. In this challenge, you will write a function that parses a provided HTML string, locates each search result, and extracts the title text from each result.


import unittest
import user_code
import ast
import re   
import importlib
import csv
import unittest
import importlib

class TestTask(unittest.TestCase):
    def test_extracts_all_titles(self):
        import user_code
        importlib.reload(user_code)
        html = """
        <div class='result'><a>First Title</a></div>
        <div class='result'><a>Second Title</a></div>
        <div class='result'><a>Third Title</a></div>
        """
        result = user_code.extract_titles(html)
        expected = ["First Title", "Second Title", "Third Title"]
        _dynamic_test(
            self,
            isinstance(result, list) and result == expected,
            "Returns all titles in order from result divs.",
            f"Expected {expected}, got {result}"
        )

    def test_empty_list_when_no_results(self):
        import user_code
        importlib.reload(user_code)
        html = """
        <div class='other'><a>No Title</a></div>
        <a>Alone Title</a>
        """
        result = user_code.extract_titles(html)
        _dynamic_test(
            self,
            isinstance(result, list) and result == [],
            "Returns empty list if there are no result divs.",
            f"Expected empty list, got {result}"
        )

    def test_ignores_a_outside_result_div(self):
        import user_code
        importlib.reload(user_code)
        html = """
        <div class='result'><a>Keep Me</a></div>
        <a>Ignore Me</a>
        <div><a>Also Ignore</a></div>
        """
        result = user_code.extract_titles(html)
        expected = ["Keep Me"]
        _dynamic_test(
            self,
            isinstance(result, list) and result == expected,
            "Ignores <a> tags outside of <div class='result'>.",
            f"Expected {expected}, got {result}"
        )

    def test_multiple_titles_and_spaces(self):
        import user_code
        importlib.reload(user_code)
        html = """
        <div class='result'><a>   Title One   </a></div>
        <div class='result'><a>Title Two</a></div>
        <div class='result'><a>Title Three</a></div>
        """
        result = user_code.extract_titles(html)
        expected = ["Title One", "Title Two", "Title Three"]
        _dynamic_test(
            self,
            isinstance(result, list) and result == expected,
            "Strips spaces and returns all titles correctly.",
            f"Expected {expected}, got {result}"
        )

def _dynamic_test(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
        test_case.assertTrue(True, success_message)
    else:
        test_case._testMethodName = failure_message
        test_case.fail(failure_message)

def normalize_text(text):
    text = text.lower()
    text = re.sub(r"\\s{2,}", " ", text)
    text = re.sub(r"\\s*([,:?])\\s*", r"\\1 ", text)
    return text.strip()

def change_var(code: str, var_name: str, value: str) -> str:
    tree = ast.parse(code)
    lines = code.splitlines()
    changed = False
    # Collect all assignment nodes to modify
    assign_nodes = [
        (i, node)
        for i, node in enumerate(tree.body)
        if isinstance(node, ast.Assign)
        and any(isinstance(target, ast.Name) and target.id == var_name for target in node.targets)
    ]

    # If nothing to change, return unmodified code
    if not assign_nodes:
        return code

    # Perform replacements for all matching assignments (from last to first to not break line offsets)
    for i, node in reversed(assign_nodes):
        start_line = node.lineno - 1
        line = lines[start_line]
        indent = ' ' * (len(line) - len(line.lstrip()))
        lines[start_line] = f"{indent}{var_name} = {value}"
        next_line = len(lines)
        for next_node in tree.body[i+1:]:
            if hasattr(next_node, 'lineno'):
                next_line = next_node.lineno - 1
                break
        if next_line > start_line + 1:
            lines[start_line+1:next_line] = []
        changed = True

    return '\\n'.join(lines) if changed else code

if __name__ == "__main__":
    unittest.main()


test_main.py

Master the use of Python to automate, analyze, and optimize SEO workflows. This course guides SEO professionals through practical Python applications for keyword research, on-page analysis, and technical SEO, using real-world scenarios and hands-on coding tasks.

Learn how Python can automate repetitive SEO tasks, from scraping search results to generating meta tags, making your SEO workflow more efficient.

Explore how Python can be used to analyze keyword data, identify opportunities, and visualize keyword trends for SEO campaigns.

Apply Python to analyze on-page SEO elements and perform technical audits, including content checks, tag analysis, and site structure evaluation.

Challenge: Extract Titles from SERP HTML

Solution

Challenge: Extract Titles from SERP HTML

Solution