Python으로 만드는 AI 웹 스크래퍼

def clean_html(html):
    html = fix_text(html)
    soup = BeautifulSoup(html, "html.parser")
    # Remove obvious noisy tags
    for tag in soup([
        "script", "style", "noscript", "svg", "img", "iframe", "nav", "header", "footer", "aside", "form", "button"
    ]):
        tag.decompose()
    noise_words = [
        "cursor", "modal", "popup", "floating", "signup", "login", "cookie", "banner", "navbar", "menu", "footer", "header", "subscribe", "newsletter", "loading", "wait", "success", "auth", "w-nav", "w-form"
    ]
    # First collect noisy tags
    tags_to_remove = []
    for tag in soup.find_all(True):
        if tag.attrs is None:
            continue
        class_value = tag.get("class", [])
        id_value = tag.get("id", "")
        if isinstance(class_value, list):
            class_text = " ".join(class_value).lower()
        else:
            class_text = str(class_value).lower()
        id_text = str(id_value).lower()
        if any(word in class_text or word in id_text for word in noise_words):
            tags_to_remove.append(tag)
    # Then remove them safely
    for tag in tags_to_remove:
        tag.decompose()
    body = soup.body if soup.body else soup
    return str(body)

def html_to_markdown(html):
    markdown_text = markdownify_html(
        html,
        heading_style="ATX",
        bullets="-"
    )
    markdown_text = fix_text(markdown_text)
    # Remove image markdown
    markdown_text = re.sub(r"!\[.*?\]\(.*?\)", "", markdown_text)
    # Remove extra spaces and blank lines
    markdown_text = re.sub(r"[ \t]+", " ", markdown_text)
    markdown_text = re.sub(r"
{3,}", "

", markdown_text)
    lines = []
    skip_lines = [
        "click to try", "wait...", "you've successfully reserved your spot.", "thank you! your submission has been received!", "oops! something went wrong while submitting the form.", "product", "resources", "company"
    ]
    for line in markdown_text.splitlines():
        line = line.strip()
        if not line: continue
        if line.lower() in skip_lines: continue
        lines.append(line)
    return "
".join(lines)

def answer_query_from_page(markdown_text, user_query):
    prompt = f"""
You are an AI web scraping assistant. You will receive Markdown extracted from a webpage. Your task is to answer the user's query using only the useful page content.
User query: {user_query}
Webpage Markdown: {markdown_text}
Instructions:
- Return only clean Markdown.
- Use only information from the webpage Markdown.
- Do not invent missing details.
- Ignore navigation links, buttons, CTAs, popups, decorative labels, image captions, and repeated marketing fragments.
- Ignore lines like "Start for free", "Contact Sales", "Your AI Agent", and decorative workflow examples unless they directly answer the query.
- Focus on headings, paragraphs, product descriptions, feature sections, pricing details, documentation text, and factual claims.
- If the page does not contain the answer, say: "The page does not contain this information."
- Keep the answer short, clear, and focused.
"""
    response = client.responses.create(
        model=MODEL_NAME,
        input=prompt
    )
    return response.output_text

def ai_web_scraper(url, user_query):
    raw_html = fetch_page(url)
    cleaned_html = clean_html(raw_html)
    markdown_text = html_to_markdown(cleaned_html)
    answer = answer_query_from_page(markdown_text, user_query)
    return answer