old-svevijesti/pyth/__pycache__/scrapingsingle.cpython-310.pyc

o
<00><0C>e<EFBFBD><00>@s<>dd
lm
Z

ddlZddlmZ
ddlmZmZ
ddlZddl	m
Z

ddlmZm
Z
mZmZmZmZmZ
ddlZddlmZ
ddlZe<15>
e<11>
e<08>d<08>
Ze<06>Ze
<EFBFBD>Zgd	<09>
Zd
di
Zd1d
edefdd<10>Zdd<12>Z dd<14>Z!dd<16>Z"e#<23>Z$e#<23>Z%dd<18>Z&e#<23>Z'eD]Z(e&e(e'<27>Z)e)r<>e$<24>*e)<29>

q<>dd<1A>e$D<00>
Z+e#e<10><00>
Z,e+e,Z-e-Z+e#e+<2B>
Z+e"e+<2B>
Z+e.dk<02>
rUe+D]<5D>Z/e<02>0e/e<1C>Z1e
e1j2d<1C>Z3e3<65>4gd<1D>
<01>
Z5d<1E>6dd <20>e5D<00>
<01>
Z7e3<65>4d!g
<01>
Z8d<1E>6d"d <20>e8D<00>
<01>
Z9e9Z9e7Z7e!e7<65>
Z7e e9<65>
Z9e!ee9<65>
<01>
Z9zIej:j;j<dd#d$d%<25>d&d'e7<65>d(e9<65>d)<29>d%<25>gd*<2A>Z=e=j>dj?j@ZAeAZAe<13>BeA<65>
ZCeCd+ZDeCd,Z2e<1A>EeA<65>
ZFe
eDe2e/eFd-d.<2E><05>
s9d/ZGeeDe2e/eFeG<65>
Wq<>eH<65>
yT
ZI
z
eJd0eI<65><00><02>

WYdZI[Iq<49>dZI[Iw
wdSdS)2<>)
<01>
BeautifulSoupN)
<01>urljoin)<02>OpenAI<41>APIError)
<01>OpenAIEmbeddings)<07>insert_data<74>is_similar_data<74>get_similar<61>get_specific_data<74>
get_all_links<6B>	cleansing<6E>modify_similar_data)
<01>load_dotenv<6E>OPENAI_API_KEY)zhttps://klix.bazhttps://srpskainfo.comzhttps://bljesak.infoz
User-Agentz<74>Mozilla/5.0 (Linux; Android 5.1.1; SM-G928X Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.83 Mobile Safari/537.36<EFBFBD>
gpt-3.5-turbo<62>string<6E>returncCst<00>
|
<01>
}t|<02>|<00>
<01>
S)
N)<04>tiktoken<65>encoding_for_model<65>len<65>encode)r<00>model<65>encoding<6E>r<00>>/home/asabani/Desktop/svevijesti-master/pyth/scrapingsingle.py<70>num_tokens_from_strings


rc
CsHd
}
d}t<00>
|
<01>
}|<03>|<00>
}t|<04>
|k
r|g
S|d|<02>}|<03>|<05>
}|S)Nri<>)rrrr<00>decode)<07>text<78>
encoding_name<6D>
max_tokensr<00>tokens<6E>
sliced_tokens<6E>sliced_textrrr<00>slice_text_at_2k_tokenss







r#c
s d
<01>d<02><00>f
dd<04>|D<00>
<01>
}
|
S)NuYABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzČčĆćDždžĐ𩹮ž0123456789 <20>c
3s <00>|]}
|
<01>vr
|
n
dV
qd
S)<02>
 Nr)<02>.0<EFBFBD>char<61>
<01>
allowed_charsrr<00>	<genexpr>0s<02>z&replace_with_spaces.<locals>.<genexpr>)
<01>join)r<00>cleaned_textrr(r<00>replace_with_spaces.s


r-c
Cs>t<00>}
|D]}d
|vr|<02>
dd<03>}|
<01>|<03>

q|
<01>|<02>

q|
S)N<>wwwzwww.r$)<03>set<65>replace<63>add)<04>	links_set<65>modified_links<6B>link<6E>
modified_linkrrr<00>	fix_links4s



r6c
Cs<>t<00>
|t<02>}|jd
kr@t|jd<02>}|<03>d<03>
}g}|D]#}|jddd<06>}|D]}t||d<00>}	|	|
v
r<|<05>|	<09>

|
<01>	|	<09>

q%q|SdS)N<><4E><00>html.parser<65>article<6C>
aT)
<01>hrefr;)
<EFBFBD>requests<74>get<65>headers<72>status_coderr<00>find_allr<00>appendr1)
<EFBFBD>url<72>already_checked<65>response<73>soup<75>articles<65>
link_storer9<00>linksr4<00>
link_valuerrr<00>get_article_linksDs 














<02><02><04>rJc
Csh|]}
|
r|
<01>qSrr)r&<00>itemrrr<00>	<setcomp>\srL<00>__main__r8)<03>h2<68>h1<68>h3r%c
C<00>g|]}
|
jdd
<01>
<01>qS<00>T)
<01>strip<69>
<01>get_text)r&<00>titlerrr<00>
<listcomp>l<00>rW<00>
pc
CrQrRrT)r&rrrrrWorX<00>systemz+Data analytic, Journalist and News reporter)<02>role<6C>content<6E>userz>Extract relevant information from the following input: Title: z, Text: z<>. Remove any non-news element related to the current text and title, and provide the cleaned data as a JSON object with 'title' and 'content' fields.)r<00>messagesrVr\g\<5C><><EFBFBD>(\<5C>?)
<01>	threshold<6C>NOzError in completion: )
r)K<>bs4rr<<00>urllib.parser<00>openairr<00>os<6F>langchain.embeddingsr<00>vectDatarrr	r
rrr
<00>json<6F>dotenvrr<00>getenvr<00>client<6E>
embeddings<EFBFBD>dlinksr><00>str<74>intrr#r-r6r/<00>total_links<6B>collected_newsrJrC<00>dlink<6E>
temp_links<EFBFBD>update<74>final_links<6B>db_links<6B>	new_links<6B>__name__r4r=rDrrEr@<00>titlesr+<00>
title_text<EFBFBD>texts<74>	text_text<78>chat<61>completions<6E>create<74>
completion<EFBFBD>choices<65>messager\<00>generated_text<78>loads<64>
response_datarV<00>embed_query<72>vector<6F>	similar_d<5F>	Exception<6F>
e<>printrrrr<00><module>
s<>




$













<02>













<02><06>




<04>

<08><02><04>