senaduka/old-nlp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Senad Uka
2019-10-20 13:16:49 +02:00
commit 233066caf4
2099 changed files with 360824 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
venv/lib/python3.7/site-packages/nltk/test/unit/test_disagreement.py Normal file
View File

@@ -0,0 +1,142 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
import unittest
from nltk.metrics.agreement import AnnotationTask
class TestDisagreement(unittest.TestCase):
'''
Class containing unit tests for nltk.metrics.agreement.Disagreement.
'''
def test_easy(self):
'''
Simple test, based on
https://github.com/foolswood/krippendorffs_alpha/raw/master/krippendorff.pdf.
'''
data = [('coder1', 'dress1', 'YES'),
('coder2', 'dress1', 'NO'),
('coder3', 'dress1', 'NO'),
('coder1', 'dress2', 'YES'),
('coder2', 'dress2', 'NO'),
('coder3', 'dress3', 'NO'),
]
annotation_task = AnnotationTask(data)
self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)
def test_easy2(self):
'''
Same simple test with 1 rating removed.
Removal of that rating should not matter: K-Apha ignores items with
only 1 rating.
'''
data = [('coder1', 'dress1', 'YES'),
('coder2', 'dress1', 'NO'),
('coder3', 'dress1', 'NO'),
('coder1', 'dress2', 'YES'),
('coder2', 'dress2', 'NO'),
]
annotation_task = AnnotationTask(data)
self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)
def test_advanced(self):
'''
More advanced test, based on
http://www.agreestat.com/research_papers/onkrippendorffalpha.pdf
'''
data = [('A', '1', '1'),
('B', '1', '1'),
('D', '1', '1'),
('A', '2', '2'),
('B', '2', '2'),
('C', '2', '3'),
('D', '2', '2'),
('A', '3', '3'),
('B', '3', '3'),
('C', '3', '3'),
('D', '3', '3'),
('A', '4', '3'),
('B', '4', '3'),
('C', '4', '3'),
('D', '4', '3'),
('A', '5', '2'),
('B', '5', '2'),
('C', '5', '2'),
('D', '5', '2'),
('A', '6', '1'),
('B', '6', '2'),
('C', '6', '3'),
('D', '6', '4'),
('A', '7', '4'),
('B', '7', '4'),
('C', '7', '4'),
('D', '7', '4'),
('A', '8', '1'),
('B', '8', '1'),
('C', '8', '2'),
('D', '8', '1'),
('A', '9', '2'),
('B', '9', '2'),
('C', '9', '2'),
('D', '9', '2'),
('B', '10', '5'),
('C', '10', '5'),
('D', '10', '5'),
('C', '11', '1'),
('D', '11', '1'),
('C', '12', '3'),
]
annotation_task = AnnotationTask(data)
self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)
def test_advanced2(self):
'''
Same more advanced example, but with 1 rating removed.
Again, removal of that 1 rating shoudl not matter.
'''
data = [('A', '1', '1'),
('B', '1', '1'),
('D', '1', '1'),
('A', '2', '2'),
('B', '2', '2'),
('C', '2', '3'),
('D', '2', '2'),
('A', '3', '3'),
('B', '3', '3'),
('C', '3', '3'),
('D', '3', '3'),
('A', '4', '3'),
('B', '4', '3'),
('C', '4', '3'),
('D', '4', '3'),
('A', '5', '2'),
('B', '5', '2'),
('C', '5', '2'),
('D', '5', '2'),
('A', '6', '1'),
('B', '6', '2'),
('C', '6', '3'),
('D', '6', '4'),
('A', '7', '4'),
('B', '7', '4'),
('C', '7', '4'),
('D', '7', '4'),
('A', '8', '1'),
('B', '8', '1'),
('C', '8', '2'),
('D', '8', '1'),
('A', '9', '2'),
('B', '9', '2'),
('C', '9', '2'),
('D', '9', '2'),
('B', '10', '5'),
('C', '10', '5'),
('D', '10', '5'),
('C', '11', '1'),
('D', '11', '1'),
('C', '12', '3'),
]
annotation_task = AnnotationTask(data)
self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)