old-nlp/venv/lib/python3.7/site-packages/nltk/__pycache__/collocations.cpython-37.pyc

B
D(<28>]
9<00>@sF
dZd
dl
mZ
d
dlZd
dlmZ
d
dlmZ
d
dl	m
Z

d
dlmZm
Z
mZmZ
d
dlmZmZ
Gd	d
<EFBFBD>d
e<13>ZGdd<0C>de<14>ZGd
d<0E>de<14>ZGdd<10>de<14>Zddd<12>
Zedk<02>
r8d
dlZd
dlm
Z

yedejd<00>
ZWnek
r<EFBFBD>


dZYnXyedejd<00>
ZWnek
<EFBFBD>
r,


dZYnXeee<1F>
dddgZ dS)a<>
Tools to identify collocations --- words that often appear consecutively
--- within corpora. They may also be used to find other associations between
word occurrences.
See Manning and Schutze ch. 5 at http://nlp.stanford.edu/fsnlp/promo/colloc.pdf
and the Text::NSP Perl package at http://ngram.sourceforge.net

Finding collocations requires first calculating the frequencies of words and
their appearance in the context of other words. Often the collection of words
will then requiring filtering to only retain useful content terms. Each ngram
of words may then be scored according to some association measure, in order
to determine the relative likelihood of each ngram being a collocation.

The ``BigramCollocationFinder`` and ``TrigramCollocationFinder`` classes provide
these functionalities, dependent on being provided a function which scores a
ngram given appropriate frequency counts. A number of standard association
measures are provided in bigram_measures and trigram_measures.
<EFBFBD>)
<01>print_functionN)
<01>	iteritems)
<01>FreqDist)
<01>ngrams)<04>ContingencyMeasures<65>BigramAssocMeasures<65>TrigramAssocMeasures<65>QuadgramAssocMeasures)<02>ranks_from_scores<65>spearman_correlationc@s<>eZ
dZd
Zdd<03>Zeddd<07>
<01>
Zedd	<09><00>
Zed
d<0B><00>
Z	dd
<0A>f
dd<0F>
Z
dd<11>Zdd<13>Zdd<15>Z
dd<17>Zdd<19>Zdd<1B>Zdd<1D>ZdS)<1F>AbstractCollocationFindera<72>

    An abstract base class for collocation finders whose purpose is to
    collect collocation candidate frequencies, filter and rank them.

    As a minimum, collocation finders require the frequencies of each
    word in a corpus, and the joint frequency of word tuples. This data
    should be provided through nltk.probability.FreqDist objects or an
    identical interface.
    cCs|
|_|
<01>
<01>|_
||_dS)
N)<03>word_fd<66>
N<>ngram_fd)<03>selfr
r<00>r<00>3/tmp/pip-install-4m6m_5d_/nltk/nltk/collocations.py<70>__init__7s



z"AbstractCollocationFinder.__init__FNcsN|f
|d
<00>|r,tj
<01><02>f
dd<03>|
D<00>
<01>
S|rJtj
<01><02>f
dd<03>|
D<00>
<01>
SdS)zU
        Pad the document with the place holder according to the window_size
        <20>
c
3s|]}
t<00>
|
<01><00>V
qdS)
N)<02>
_itertools<EFBFBD>chain)<02>.0<EFBFBD>doc)
<01>paddingrr<00>	<genexpr>FszAAbstractCollocationFinder._build_new_documents.<locals>.<genexpr>c
3s|]}
t<00>
<01>|
<01>V
qdS)
N)rr)rr)
rrrrJsN)rr<00>
from_iterable)<06>cls<6C>	documents<74>window_sizeZpad_left<66>	pad_rightZ
pad_symbolr)
rr<00>_build_new_documents<s




z.AbstractCollocationFinder._build_new_documentscCs|<00>|j
|
|jd
d<02><03>
S)z<>Constructs a collocation finder given a collection of documents,
        each of which is a list (or iterable) of tokens.
        T)
r)<03>
from_wordsr <00>
default_ws)rrrrr<00>from_documentsMs
z(AbstractCollocationFinder.from_documentscs$t<00><00>
fd
d<02>t
t<02>
<01>
d<00>
D<00>
<01>
S)Nc
3s"|]}
t<00>
|
|
<01><00><00>
V
qdS)
N)
<01>tuple)r<00>
i)<02>
n<>wordsrrrYsz<AbstractCollocationFinder._ngram_freqdist.<locals>.<genexpr>r)r<00>range<67>len)r'r&r)r&r'r<00>_ngram_freqdistWsz)AbstractCollocationFinder._ngram_freqdistc
Csd
S)NFr)<02>ngram<61>freqrrr<00><lambda>[<00>z"AbstractCollocationFinder.<lambda>cCs:t<00>}x(t
|j<02>
D]\}}|
||<04>s|||<qW||_d
S)z<>Generic filter removes ngrams from the frequency distribution
        if the function returns True when passed an ngram tuple.
        N)rrr)r<00>fnZ	tmp_ngramr+r,rrr<00>
_apply_filter[s





z'AbstractCollocationFinder._apply_filtercs|<00><00>f
d
d<02><08>

dS)zARemoves candidate ngrams which have frequency less than min_freq.cs|
<01>kS)
Nr)<02>ngr,)
<01>min_freqrrr-gr.z=AbstractCollocationFinder.apply_freq_filter.<locals>.<lambda>N)
r0)rr2r)
r2r<00>apply_freq_filteresz+AbstractCollocationFinder.apply_freq_filtercs|<00><00>f
d
d<02><08>

dS)z`Removes candidate ngrams (w1, w2, ...) where fn(w1, w2, ...)
        evaluates to True.
        cs<00>|<00>S)
Nr)r1<00>
f)
r/rrr-mr.z>AbstractCollocationFinder.apply_ngram_filter.<locals>.<lambda>N)
r0)rr/r)
r/r<00>apply_ngram_filterisz,AbstractCollocationFinder.apply_ngram_filtercs|<00><00>f
d
d<02><08>

dS)zmRemoves candidate ngrams (w1, w2, ...) where any of (fn(w1), fn(w2),
        ...) evaluates to True.
        cst<00>f
d
d<02>|D<00>
<01>
S)Nc
3s|]}
<01>|
<01>
V
qdS)
Nr)r<00>
w)
r/rrrsszPAbstractCollocationFinder.apply_word_filter.<locals>.<lambda>.<locals>.<genexpr>)
<01>any)r1r4)
r/rrr-sr.z=AbstractCollocationFinder.apply_word_filter.<locals>.<lambda>N)
r0)rr/r)
r/r<00>apply_word_filterosz+AbstractCollocationFinder.apply_word_filterccs6x0|jD]&}|j
|
f
|<02><02>}|d
k	r||fV
qWd
S)zbGenerates of (ngram, score) pairs as determined by the scoring
        function provided.
        N)r<00>score_ngram)r<00>score_fn<66>tup<75>scorerrr<00>
_score_ngramsus


z'AbstractCollocationFinder._score_ngramscCst|<00>
|
<01>
d
d<02>d<03>S)z<>Returns a sequence of (ngram, score) pairs ordered from highest to
        lowest score, as determined by the scoring function provided.
        c

Ss|d
|dfS)Nrr
r)
<01>
trrrr-<00>r.z8AbstractCollocationFinder.score_ngrams.<locals>.<lambda>)
<01>key)<02>sortedr=)rr:rrr<00>score_ngrams~sz&AbstractCollocationFinder.score_ngramscCsd
d<02>|<00>|
<01>
d|<02>D<00>
S)z;Returns the top n ngrams when scored by the given function.c
Ssg|]\}
}|
<01>qSrr)r<00>
p<>
srrr<00>
<listcomp><3E>sz3AbstractCollocationFinder.nbest.<locals>.<listcomp>N)
rA)rr:r&rrr<00>nbest<73>szAbstractCollocationFinder.nbestccs.x(|<00>|
<01>
D]\}}||kr$|V
qPqWd
S)z}Returns a sequence of ngrams, ordered by decreasing score, whose
        scores each exceed the given minimum score.
        N)
rA)rr:Z	min_scorer+r<rrr<00>above_score<72>s

z%AbstractCollocationFinder.above_score)FFN)<12>__name__<5F>
__module__<EFBFBD>__qualname__<5F>__doc__r<00>classmethodr r#<00>staticmethodr*r0r3r5r8r=rArErFrrrrr,s	

	rc@s4eZ
dZd
ZdZd
dd<04>
Zeddd<06>
<01>
Zdd<08>Zd	S)<0C>BigramCollocationFinderz<72>A tool for the finding and ranking of bigram collocations or other
    association measures. It is often useful to use from_words() rather than
    constructing an instance directly.
    <20>cCst<00>
||
|<02>
||_d
S)z<>Construct a BigramCollocationFinder, given FreqDists for
        appearances of words and (possibly non-contiguous) bigrams.
        N)rrr)rr
<00>	bigram_fdrrrrr<00>s
z BigramCollocationFinder.__init__cCs<>t<00>}t<00>}|d
krt
d<02>
<01>
xjt|
|dd<04>D]X}|d}|dkrBq,||d7<x0|dd<06>D] }|dk	r`|||fd7<q`Wq,W||||d<08>S)	z<>Construct a BigramCollocationFinder for all bigrams in the given
        sequence.  When window_size > 2, count non-contiguous bigrams, in the
        style of Church and Hanks's (1990) association ratio.
        rNzSpecify window_size at least 2T)
rr
Nr)
r)r<00>
ValueErrorr)rr'r<00>wfd<66>bfd<66>window<6F>w1<77>w2rrrr!<00>s









z"BigramCollocationFinder.from_wordscCsJ|j}|j
||f|jd
}|s&dS|j|}|j|}|
|||f|<04>S)z<>Returns the score for a given bigram using the given scoring
        function.  Following Church and Hanks (1990), counts are scaled by
        a factor of 1/(window_size - 1).
        g<00>?N)rrrr
)rr:rTrU<00>n_allZn_iiZn_ixZn_xirrrr9<00>s







z#BigramCollocationFinder.score_ngramN)
rN)
rN)	rGrHrIrJr"rrKr!r9rrrrrM<00>s

rMc@s:eZ
dZd
ZdZdd<04>Zeddd<06>
<01>
Zdd<08>Zd	d
<EFBFBD>Z	dS)
<0A>TrigramCollocationFinderz<72>A tool for the finding and ranking of trigram collocations or other
    association measures. It is often useful to use from_words() rather than
    constructing an instance directly.
    <20>cCst<00>
||
|<04>
||_||_d
S)z<>Construct a TrigramCollocationFinder, given FreqDists for
        appearances of words, bigrams, two words with any word between them,
        and trigrams.
        N)rr<00>wildcard_fdrO)rr
rOrYZ
trigram_fdrrrr<00>s

z!TrigramCollocationFinder.__init__cCs<>|d
krtd<02>
<01>
t
<01>}t
<01>}t
<01>}t
<01>}x<>t|
|dd<04>D]<5D>}|d}|dkrNq8x<38>t<03>|dd<06>d<08>D]j\}	}
||d7<|	dkr<>qd|||	fd7<|
dkr<>qd|||
fd7<|||	|
fd7<qdWq8W|||||<06>S)	z]Construct a TrigramCollocationFinder for all trigrams in the given
        sequence.
        rXzSpecify window_size at least 3T)
rr
NrrN)rPrrr<00>combinations)rr'rrQZwildfdrRZtfdrSrTrU<00>w3rrrr!<00>s(

















z#TrigramCollocationFinder.from_wordsc

Cst|j
|j<02>S)
z<>Constructs a bigram collocation finder with the bigram and unigram
        data from this finder. Note that this does not include any filtering
        applied to this finder.
        )rMr
rO)
rrrr<00>
bigram_finder<65>sz&TrigramCollocationFinder.bigram_finderc
Cs<>|j}|j
|||f}|sd
S|j||f}|j||f}|j||f}	|j|}
|j|}|j|}|
||||	f|
||f|<05>S)zXReturns the score for a given trigram using the given scoring
        function.
        N)rrrOrYr
)
rr:rTrUr[rVZn_iiiZn_iixZn_ixiZn_xiiZn_ixxZn_xixZn_xxirrrr9<00>s












z$TrigramCollocationFinder.score_ngramN)
rX)
rGrHrIrJr"rrKr!r\r9rrrrrW<00>s	
rWc@s2eZ
dZd
ZdZdd<04>Zed
dd<06>
<01>
Zdd<08>Zd	S)<0B>QuadgramCollocationFinderz<72>A tool for the finding and ranking of quadgram collocations or other association measures.
    It is often useful to use from_words() rather than constructing an instance directly.
    <20>c		Cs6t<00>
||
|<02>
||_||_||_||_||_||_d
S)z<>Construct a QuadgramCollocationFinder, given FreqDists for appearances of words,
        bigrams, trigrams, two words with one word and two words between them, three words
        with a word between them in both variations.
        N)rr<00>iii<69>ii<69>ixi<78>ixxi<78>iixi<78>ixii)	rr
Zquadgram_fdr`r_rarbrcrdrrrr
s





z"QuadgramCollocationFinder.__init__c	Csj
|d
krtd<02>
<01>
t
<01>}t
<01>}t
<01>}t
<01>}t
<01>}t
<01>}t
<01>}	t
<01>}
<EFBFBD>
xt|
|dd<04>D]<5D>}|d}|dkrhqRx<52>t<03>|dd<00>d<07>D]<5D>\}
}}||d7<|
dkr<>q~|||
fd7<|dkr<>q~|||
|fd7<|||fd7<|dkr<>q~|||
||fd7<|||fd7<|
|||fd7<|	||
|fd7<q~WqRW||||||||	|
<EFBFBD>S)Nr^zSpecify window_size at least 4T)
rr
rrX)rPrrrrZ)rr'rZixxxZiiiir`r_rarbrcrdrSrTrUr[<00>w4rrrr!
s<












 













z$QuadgramCollocationFinder.from_wordsc	Cs
|j}|j
||||f}|s dS|j|||f}|j|||f}	|j|||f}
|j|||f}|j||f}|j||f}
|j||f}|j||f}|j||f}|j||f}|j|}|j|}|j|}|j|}|
|||
||	f|||||
|f||||f|<06>S)
N)	rrr_rcrdr`rarbr
)rr:rTrUr[rerVZn_iiiiZn_iiixZn_xiiiZn_iixiZn_ixiiZn_iixxZn_xxiiZn_xiixZn_ixixZn_ixxiZn_xixiZn_ixxxZn_xixxZn_xxixZn_xxxirrrr9A
s0



























z%QuadgramCollocationFinder.score_ngramN)
r^)	rGrHrIrJr"rrKr!r9rrrrr]

s

"r]cs<>d
dlm
}m}m}
|dkr"|j}|
dkr0|j}
d
dlm}m}
|<05>	d<05>
<01><00>f
dd<07>}x<>|<06>
<EFBFBD>D]<5D>}dd	<09>|<06>	|<08>
D<00>
}	t<0B>|	<09>
}
|
<EFBFBD>
d
<EFBFBD>

|
<EFBFBD>|<07>

|||
<EFBFBD>|<00>
<01>
||
<EFBFBD>|
<01>
<01>
<01>}t|<08>

tddd	<09>|
<EFBFBD>|d
<0A>D<00>
<01>
td|
j|f<00>

q`WdS)z=Finds bigram collocations in the files of the WebText corpus.r
)rrr
N)<02>	stopwords<64>webtext<78>englishc

st|<00>
d
kp|<00>
<01><00>kS)NrX)r)<00>lower)
r6)
<01>
ignored_wordsrrr-o
r.zdemo.<locals>.<lambda>c
Ssg|]}
|
<01><00><00>qSr)
ri)r<00>wordrrrrDr
szdemo.<locals>.<listcomp>rX<00>
	c
Ssg|]}
d<00>|
<01>
<01>qS)
<01>
 )
<01>join)rr;rrrrD}
s<00>z	 Correlation to %s: %0.4f)<13>nltk.metricsrrr
Zlikelihood_ratioZraw_freqZnltk.corpusrfrgr'ZfileidsrMr!r3r8rA<00>printrErG)<0C>scorer<65>compare_scorerrrr
rfrgZword_filter<65>filer'<00>cfZcorrr)
rjr<00>demo_
s&














rv<00>__main__)
rzBigramAssocMeasures.rrN)NN)!rJ<00>
__future__r<00>	itertoolsr<00>sixrZnltk.probabilityrZ	nltk.utilrrprrrr	Znltk.metrics.spearmanr
r<00>objectrrMrWr]rvrG<00>sys<79>eval<61>argvrr<00>
IndexErrorrs<00>__all__rrrr<00><module>s6



g3DU
&