adding multiple urls

pyth/singlearticle.py

@@ -5,40 +5,28 @@ from bs4 import BeautifulSoup
 
 os.environ["OPENAI_API_KEY"] = "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"
 
-
 client = OpenAI()
 
-from openai import OpenAI
-client = OpenAI()
-
-url = 'https://srpskainfo.com/tuzilastvo-ih-tek-jutros-obavijestilo-da-istrazuje-debeveca-disciplinski-tuzilac-vsts-trazi-suspenziju-predsjednika-suda-bih/' 
+urls = ['https://klix.ba/', 'https://srpskainfo.com/', 'https://bljesak.info/']
 
 
-response = requests.get(url)
-html = response.text
+for url in urls:
+    response = requests.get(url)
+    html = response.text
+    soup = BeautifulSoup(html, 'html.parser')
+    tags = soup.find_all(['h2', 'p'])
 
-soup = BeautifulSoup(html, 'html.parser')
-onlyText = ' '.join(filter(None, [text.strip() for text in soup.stripped_strings]))
+    prompt_text = ''
+    for tag in tags:
+        text = tag.get_text(strip=True)
+        prompt_text = prompt_text + text
 
-
-def takeDataFrom(url):
-   try:
-        response = requests.get(url)
-        response.raise_for_status()
-        data = response.text
-        print(f"Here:{data}")
-        return data
-   except Exception as e:
-    print(f"Error! :{e}")
-    return None
-
-
-completion = client.chat.completions.create(
-    model="gpt-3.5-turbo",
-    messages=[
-     {"role": "system", "content": "Data analytic"},
-     {"role": "user", "content": f"Extract for me title and full text from :{onlyText},make it clear for reading"}
-    ]
-   )
-
-print(completion.choices[0].message.content)
+    completion = client.chat.completions.create(
+        model="gpt-3.5-turbo",
+        messages=[
+            {"role": "system", "content": "Data analytic, Journalist and News reporter"},
+            {"role": "user", "content": f"Extract for me evry title and full content for evry title from {prompt_text},without shortening,remove all thing that are not connected to news,  make it clear for reading"}
+        ]
+    )
+    generated_text = completion.choices[0].message.content
+    print(f"Text for {url}: \n {generated_text}\n")