import requests
from openai import OpenAI
import os
from bs4 import BeautifulSoup

os.environ["OPENAI_API_KEY"] = "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"

client = OpenAI()

urls = ['https://klix.ba/', 'https://srpskainfo.com/', 'https://bljesak.info/']


for url in urls:
    response = requests.get(url)
    html = response.text
    soup = BeautifulSoup(html, 'html.parser')
    tags = soup.find_all(['h2', 'p'])

    prompt_text = ''
    for tag in tags:
        text = tag.get_text(strip=True)
        prompt_text = prompt_text + text

    completion = client.chat.completions.create(
        model="gpt-3.5-turbo",
        messages=[
            {"role": "system", "content": "Data analytic, Journalist and News reporter"},
            {"role": "user", "content": f"Extract for me evry title and full content for evry title from {prompt_text},without shortening,remove all thing that are not connected to news,  make it clear for reading"}
        ]
    )
    generated_text = completion.choices[0].message.content
    print(f"Text for {url}: \n {generated_text}\n")