64 lines
5.8 KiB
Plaintext
64 lines
5.8 KiB
Plaintext
|
o
|
|||
|
|
;<3B>e<EFBFBD>$ <00> @ s\ d dl Z d dlZd dlmZ d dlmZ d dlZd dlm Z m
|
|||
|
|
Z
|
|||
|
|
d dl m Z d dl
mZmZmZmZmZmZ d dlZd dlmZmZ d dlZe<06> e<07>d<08>Ze <09> Ze <0C> Z e<07>d <09>Z e<07>d
|
|||
|
|
<EFBFBD>Z e<07>d <0B>Z e<07>d <0C>Z e<07>d
<0A>Z!dd<0F> Z"dd<11> Z#dd<13> Z$d dd<18>Z%dd<1A> Z&e&<26> e<13> Z'e'r<>e'D ] Z(e)de(d <00> <00><02> e)d e(d <00> <00><02> e)d e(d <00> <00><02> q<>dS dS ) <20> N)<01>cosine_similarity)<01> load_dotenv)<02>OpenAI<41>APIError)<01>OpenAIEmbeddings)<06>get_specific_data<74>modify_similar_data<74> insert_data<74>preparing_articles<65>get_source_data<74>get_ready_data)<02>num_tokens_from_string<6E>slice_text_at_2k_tokens<6E>OPENAI_API_KEY<45>DB_HOST<53>DB_PORT<52>DB_USER<45> DB_PASSWORD<52>DB_NAMEc C s: | t j<01>| <00> }|t j<01>|<01> }t|g|g<01>d d }|S )Nr )<04>np<6E>linalg<6C>normr )<05>v1<76>v2<76>
v1_normalized<65>
v2_normalized<65>
|
|||
|
|
similarity<EFBFBD> r <00>8/home/asabani/Desktop/svevijesti-master/pyth/articles.py<70>calculate_cosine_similarity s r c C sH t | t<01>rdd<02> | dd<04> <00>d<05>D <00>}t<03>|<01>S t | tj<05>r | S td<06><01>)Nc S s g | ]}t |<01><01>qS r )<01>float)<02>.0<EFBFBD>numr r r <00>
|
|||
|
|
<listcomp>" <00> z*parse_embedding_string.<locals>.<listcomp><3E> <00><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>,zAInvalid type for embedding_str. Must be either str or np.ndarray.)<07>
|
|||
|
|
isinstance<EFBFBD>str<74>splitr <00>array<61>ndarray<61>
|
|||
|
|
ValueError)<02>
embedding_str<74>numbersr r r <00>parse_embedding_string s
|
|||
|
|
|
|||
|
|
r0 c C sj t jtttttd<01>} | <00><07> }|<01>d<02> |<01> <09> }|<01>
|
|||
|
|
<EFBFBD> dd<04> |D <00>}dd<04> |D <00>}dd<04> |D <00>}|||fS )N<><05>host<73>port<72>user<65>password<72>dbnamezGSELECT title, link, embedding FROM vectorsvevijesti WHERE ready = True;c S <00> g | ]}|d <00>qS )r r <00>r! <00>rowr r r r# 7 r$ z/get_titles_links_embeddings.<locals>.<listcomp>c S r7 )r% r r8 r r r r# 8 r$ c S s g | ]}t |d <00><01>qS )<01> )r0 r8 r r r r# 9 s ) <0B>psycopg2<67>connectr2 r3 r4 r5 r6 <00>cursor<6F>execute<74>fetchall<6C>close)<06>connr= <00>data<74>titles<65>links<6B>
|
|||
|
|
embeddingsr r r <00>get_titles_links_embeddings* s <06>
|
|||
|
|
|
|||
|
|
rF <00> <00>?r: <00>ffffff<66>?c
|
|||
|
|
C s~ z<>t jtttttd<01>}|<03><> |<03><07> <00>u}t<08> \}}}t <09> }g } t
|
|||
|
|
t |||<07><03>D ]O\}
|
|||
|
|
\} } }
| | f|vru|<08> | | f<02> | | fg}t
|
|||
|
|
t |||<07><03>D ](\}\}}}|
|
|||
|
|
|kro||f|vrot
|
|<12>}||kro|<08> ||f<02> |<0E>||f<02> qG| <09>|<0E> q&| W d <00> W d <00> W S 1 s<>w Y W d <00> W d S 1 s<>w Y W d S t jy<> } ztd|<14> <00><02> g W Y d }~S d }~ww )Nr1 <00>Error: )r; r< r2 r3 r4 r5 r6 r= rF <00>set<65> enumerate<74>zip<69>addr <00>append<6E>Error<6F>print)<15>eps<70> min_samples<65> thresholdrA r= rC rD rE <00>processed_articles<65>grouped_similar_articles<65>i<>title1<65>link1<6B>
|
|||
|
|
embedding1<EFBFBD>group<75>j<>title2<65>link2<6B>
|
|||
|
|
embedding2r <00>er r r <00> find_and_group_similar_articles= s@ <06>
|
|||
|
|
|
|||
|
|
<02>
|
|||
|
|
<02>X<><08><02>r` c ! C s t <00> } | <00>r| D <00>]<5D>}g }t|<01>dk<04>r<>|D ]}t|<03>dkr/|d d<02> \}}||g}|<02>|<06> qt|<02>}|dkr<>td<03> |d d }|d d } t|<08>}
|
|||
|
|
t| <09>} |
|
|||
|
|
d d } | d d }
|
|
|||
|
|
d d }| d d }||krt|<0E> d|<0F> <00>}n|}t| <0C>}t|
<0A>}|| }d|<08> d| <09> <00>}t||<08> td|<08> t|| <09> td| <09> t|<12> |dkr<>| <0C> d |
<0A> <00>}t|<14>}d
|
|||
|
|
|<14> d <0B>}n d | <0C> d |
<0A> d
<0A>}|dk<02>r<>td<0F> |d d }|d d } |d d }t|<08>}
|
|||
|
|
t| <09>} t|<16>}|
|
|||
|
|
d d } | d d }
|d d }|
|
|||
|
|
d d }| d d }|d d }||k<03>r)||k<03>r!|<0E> d|<0F> d|<19> <00>}n|<0E> d|<0F> <00>}n||k<03>r6|<0E> d|<19> <00>}n|}t| <0C>}t|
<0A>}t|<18>}|| | }d|<08> d| <09> d|<16> <00>}t||<08> td|<08> t|| <09> td| <09> t||<16> td|<16> t|<12> |dk<04>r<>| <0C> d |
<0A> d |<18> <00>}t|<14>}d
|
|||
|
|
|<14> d <0B>}n d| <0C> d |
<0A> d|<18> d
<0A>}z9t j
|
|||
|
|
j j dddd<15>d|d<15>gd<17>}|j
d jj} t<10>| <1C>} |}| d } t<12>| <1C>} t|| || |<13> td|<04> <00><02> W q t<15>y<> } ztd| <20> <00><02> td|<08> <00><02> td | <1C> <00><02> W Y d } ~ qd } ~ ww qd S td <1D> d S ) Nr% r: <00>2r z, zC: Fi<46> <00> z
Here is text zn, combined from 3 sources, filter text, and make news content, return as JSON only with single 'content' fieldzHere are 2 texts z<>, combine the following texts into a cohesive news remove any non-news related to both texts and provide the cleaned data as a JSON only with single 'content' field.<2E> <00>3zHere are 3 texts z and z
gpt-3.5-turbo<62>systemz+Data analytic, Journalist and News reporter)<02>role<6C>contentr4 )<02>model<65>messagesrg zInserting combined: rI <00>Title: zAnswer: zNo similar articles found.)r` <00>lenrN rP r r
r r
|
|||
|
|
r <00>client<6E>chat<61> completions<6E>create<74>choices<65>messagerg <00>json<6F>loadsrE <00> embed_queryr <00> Exception)!<21> grouped_similar_articles_resultrZ <00>articles<65>
article_tuple<6C>title<6C>link<6E>article<6C>l<>a_one<6E>a_two<77>get_one<6E>get_two<77>text1<74>text2rX r] <00>ftoks<6B>stoks<6B>tokens<6E> similar_d<5F>
combined_text<78> user_message<67>a_three<65> get_three<65>text3<74>link3<6B>ttoks<6B>
|
|||
|
|
completion<EFBFBD>generated_text<78>
response_data<74>text<78>vectorr_ r r r <00>processing_similarb s<>
|
|||
|
|
|
|||
|
|
<02>
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
<02><06>
|
|||
|
|
|
|||
|
|
<08><02><02><04> vr<76> rj zLink: zStatus: rc )rG r: rH )*r; <00>numpyr <00>sklearn.metrics.pairwiser <00>dotenvr <00>os<6F>openair r <00>langchain.embeddingsr <00>vectDatar r r r
|
|||
|
|
r r <00>tiktoken<65>scrapingsingler
r rr <00>getenvr rl rE r2 r3 r4 r5 r6 r r0 rF r` r<> <00>ready<64>arP r r r r <00><module> sD
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
%}<04>
|