senaduka/old-nlp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Senad Uka
2019-10-20 13:16:49 +02:00
commit 233066caf4
2099 changed files with 360824 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
venv/lib/python3.7/site-packages/googletrans/__init__.py Normal file
View File

@@ -0,0 +1,7 @@
"""Free Google Translate API for Python. Translates totally free of charge."""
__all__ = 'Translator',
__version__ = '2.4.0'
from googletrans.client import Translator
from googletrans.constants import LANGCODES, LANGUAGES

BIN
venv/lib/python3.7/site-packages/googletrans/__pycache__/__init__.cpython-37.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.7/site-packages/googletrans/__pycache__/adapters.cpython-37.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.7/site-packages/googletrans/__pycache__/client.cpython-37.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.7/site-packages/googletrans/__pycache__/compat.cpython-37.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.7/site-packages/googletrans/__pycache__/constants.cpython-37.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.7/site-packages/googletrans/__pycache__/gtoken.cpython-37.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.7/site-packages/googletrans/__pycache__/models.cpython-37.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.7/site-packages/googletrans/__pycache__/urls.cpython-37.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.7/site-packages/googletrans/__pycache__/utils.cpython-37.pyc Normal file
View File

Binary file not shown.

16
venv/lib/python3.7/site-packages/googletrans/adapters.py Normal file
View File

@@ -0,0 +1,16 @@
from requests.adapters import HTTPAdapter
class TimeoutAdapter(HTTPAdapter):
"""HTTP adapter that adds timeout to each query."""
def __init__(self, timeout=None, *args, **kwargs):
"""HTTP adapter that adds timeout to each query.
:param timeout: Timeout that will be added to each query
"""
self.timeout = timeout
super(TimeoutAdapter, self).__init__(*args, **kwargs)
def send(self, *args, **kwargs):
kwargs['timeout'] = self.timeout
return super(TimeoutAdapter, self).send(*args, **kwargs)

262
venv/lib/python3.7/site-packages/googletrans/client.py Normal file
View File

@@ -0,0 +1,262 @@
# -*- coding: utf-8 -*-
"""
A Translation module.
You can translate text using this module.
"""
import requests
import random
from googletrans import urls, utils
from googletrans.adapters import TimeoutAdapter
from googletrans.compat import PY3
from googletrans.gtoken import TokenAcquirer
from googletrans.constants import DEFAULT_USER_AGENT, LANGCODES, LANGUAGES, SPECIAL_CASES
from googletrans.models import Translated, Detected
EXCLUDES = ('en', 'ca', 'fr')
class Translator(object):
"""Google Translate ajax API implementation class
You have to create an instance of Translator to use this API
:param service_urls: google translate url list. URLs will be used randomly.
For example ``['translate.google.com', 'translate.google.co.kr']``
:type service_urls: a sequence of strings
:param user_agent: the User-Agent header to send when making requests.
:type user_agent: :class:`str`
:param proxies: proxies configuration.
Dictionary mapping protocol or protocol and host to the URL of the proxy
For example ``{'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}``
:type proxies: dictionary
:param timeout: Definition of timeout for Requests library.
Will be used by every request.
:type timeout: number or a double of numbers
"""
def __init__(self, service_urls=None, user_agent=DEFAULT_USER_AGENT,
proxies=None, timeout=None):
self.session = requests.Session()
if proxies is not None:
self.session.proxies = proxies
self.session.headers.update({
'User-Agent': user_agent,
})
if timeout is not None:
self.session.mount('https://', TimeoutAdapter(timeout))
self.session.mount('http://', TimeoutAdapter(timeout))
self.service_urls = service_urls or ['translate.google.com']
self.token_acquirer = TokenAcquirer(session=self.session, host=self.service_urls[0])
# Use HTTP2 Adapter if hyper is installed
try: # pragma: nocover
from hyper.contrib import HTTP20Adapter
self.session.mount(urls.BASE, HTTP20Adapter())
except ImportError: # pragma: nocover
pass
def _pick_service_url(self):
if len(self.service_urls) == 1:
return self.service_urls[0]
return random.choice(self.service_urls)
def _translate(self, text, dest, src):
if not PY3 and isinstance(text, str): # pragma: nocover
text = text.decode('utf-8')
token = self.token_acquirer.do(text)
params = utils.build_params(query=text, src=src, dest=dest,
token=token)
url = urls.TRANSLATE.format(host=self._pick_service_url())
r = self.session.get(url, params=params)
data = utils.format_json(r.text)
return data
def _parse_extra_data(self, data):
response_parts_name_mapping = {
0: 'translation',
1: 'all-translations',
2: 'original-language',
5: 'possible-translations',
6: 'confidence',
7: 'possible-mistakes',
8: 'language',
11: 'synonyms',
12: 'definitions',
13: 'examples',
14: 'see-also',
}
extra = {}
for index, category in response_parts_name_mapping.items():
extra[category] = data[index] if (index < len(data) and data[index]) else None
return extra
def translate(self, text, dest='en', src='auto'):
"""Translate text from source language to destination language
:param text: The source text(s) to be translated. Batch translation is supported via sequence input.
:type text: UTF-8 :class:`str`; :class:`unicode`; string sequence (list, tuple, iterator, generator)
:param dest: The language to translate the source text into.
The value should be one of the language codes listed in :const:`googletrans.LANGUAGES`
or one of the language names listed in :const:`googletrans.LANGCODES`.
:param dest: :class:`str`; :class:`unicode`
:param src: The language of the source text.
The value should be one of the language codes listed in :const:`googletrans.LANGUAGES`
or one of the language names listed in :const:`googletrans.LANGCODES`.
If a language is not specified,
the system will attempt to identify the source language automatically.
:param src: :class:`str`; :class:`unicode`
:rtype: Translated
:rtype: :class:`list` (when a list is passed)
Basic usage:
>>> from googletrans import Translator
>>> translator = Translator()
>>> translator.translate('안녕하세요.')
<Translated src=ko dest=en text=Good evening. pronunciation=Good evening.>
>>> translator.translate('안녕하세요.', dest='ja')
<Translated src=ko dest=ja text=こんにちは。 pronunciation=Kon'nichiwa.>
>>> translator.translate('veritas lux mea', src='la')
<Translated src=la dest=en text=The truth is my light pronunciation=The truth is my light>
Advanced usage:
>>> translations = translator.translate(['The quick brown fox', 'jumps over', 'the lazy dog'], dest='ko')
>>> for translation in translations:
... print(translation.origin, ' -> ', translation.text)
The quick brown fox -> 빠른 갈색 여우
jumps over -> 이상 점프
the lazy dog -> 게으른 개
"""
dest = dest.lower().split('_', 1)[0]
src = src.lower().split('_', 1)[0]
if src != 'auto' and src not in LANGUAGES:
if src in SPECIAL_CASES:
src = SPECIAL_CASES[src]
elif src in LANGCODES:
src = LANGCODES[src]
else:
raise ValueError('invalid source language')
if dest not in LANGUAGES:
if dest in SPECIAL_CASES:
dest = SPECIAL_CASES[dest]
elif dest in LANGCODES:
dest = LANGCODES[dest]
else:
raise ValueError('invalid destination language')
if isinstance(text, list):
result = []
for item in text:
translated = self.translate(item, dest=dest, src=src)
result.append(translated)
return result
origin = text
data = self._translate(text, dest, src)
# this code will be updated when the format is changed.
translated = ''.join([d[0] if d[0] else '' for d in data[0]])
extra_data = self._parse_extra_data(data)
# actual source language that will be recognized by Google Translator when the
# src passed is equal to auto.
try:
src = data[2]
except Exception: # pragma: nocover
pass
pron = origin
try:
pron = data[0][1][-2]
except Exception: # pragma: nocover
pass
if not PY3 and isinstance(pron, unicode) and isinstance(origin, str): # pragma: nocover
origin = origin.decode('utf-8')
if dest in EXCLUDES and pron == origin:
pron = translated
# for python 2.x compatbillity
if not PY3: # pragma: nocover
if isinstance(src, str):
src = src.decode('utf-8')
if isinstance(dest, str):
dest = dest.decode('utf-8')
if isinstance(translated, str):
translated = translated.decode('utf-8')
# put final values into a new Translated object
result = Translated(src=src, dest=dest, origin=origin,
text=translated, pronunciation=pron, extra_data=extra_data)
return result
def detect(self, text):
"""Detect language of the input text
:param text: The source text(s) whose language you want to identify.
Batch detection is supported via sequence input.
:type text: UTF-8 :class:`str`; :class:`unicode`; string sequence (list, tuple, iterator, generator)
:rtype: Detected
:rtype: :class:`list` (when a list is passed)
Basic usage:
>>> from googletrans import Translator
>>> translator = Translator()
>>> translator.detect('이 문장은 한글로 쓰여졌습니다.')
<Detected lang=ko confidence=0.27041003>
>>> translator.detect('この文章は日本語で書かれました。')
<Detected lang=ja confidence=0.64889508>
>>> translator.detect('This sentence is written in English.')
<Detected lang=en confidence=0.22348526>
>>> translator.detect('Tiu frazo estas skribita en Esperanto.')
<Detected lang=eo confidence=0.10538048>
Advanced usage:
>>> langs = translator.detect(['한국어', '日本語', 'English', 'le français'])
>>> for lang in langs:
... print(lang.lang, lang.confidence)
ko 1
ja 0.92929292
en 0.96954316
fr 0.043500196
"""
if isinstance(text, list):
result = []
for item in text:
lang = self.detect(item)
result.append(lang)
return result
data = self._translate(text, dest='en', src='auto')
# actual source language that will be recognized by Google Translator when the
# src passed is equal to auto.
src = ''
confidence = 0.0
try:
src = ''.join(data[8][0])
confidence = data[8][-2][0]
except Exception: # pragma: nocover
pass
result = Detected(lang=src, confidence=confidence)
return result

11
venv/lib/python3.7/site-packages/googletrans/compat.py Normal file
View File

@@ -0,0 +1,11 @@
# -*- coding: utf-8 -*-
import sys
try: # pragma: nocover
from urllib.parse import quote
except: # pragma: nocover
from urllib import quote
PY3 = sys.version_info > (3, )
unicode = str if PY3 else unicode

116
venv/lib/python3.7/site-packages/googletrans/constants.py Normal file
View File

@@ -0,0 +1,116 @@
DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
SPECIAL_CASES = {
'ee': 'et',
}
LANGUAGES = {
'af': 'afrikaans',
'sq': 'albanian',
'am': 'amharic',
'ar': 'arabic',
'hy': 'armenian',
'az': 'azerbaijani',
'eu': 'basque',
'be': 'belarusian',
'bn': 'bengali',
'bs': 'bosnian',
'bg': 'bulgarian',
'ca': 'catalan',
'ceb': 'cebuano',
'ny': 'chichewa',
'zh-cn': 'chinese (simplified)',
'zh-tw': 'chinese (traditional)',
'co': 'corsican',
'hr': 'croatian',
'cs': 'czech',
'da': 'danish',
'nl': 'dutch',
'en': 'english',
'eo': 'esperanto',
'et': 'estonian',
'tl': 'filipino',
'fi': 'finnish',
'fr': 'french',
'fy': 'frisian',
'gl': 'galician',
'ka': 'georgian',
'de': 'german',
'el': 'greek',
'gu': 'gujarati',
'ht': 'haitian creole',
'ha': 'hausa',
'haw': 'hawaiian',
'iw': 'hebrew',
'hi': 'hindi',
'hmn': 'hmong',
'hu': 'hungarian',
'is': 'icelandic',
'ig': 'igbo',
'id': 'indonesian',
'ga': 'irish',
'it': 'italian',
'ja': 'japanese',
'jw': 'javanese',
'kn': 'kannada',
'kk': 'kazakh',
'km': 'khmer',
'ko': 'korean',
'ku': 'kurdish (kurmanji)',
'ky': 'kyrgyz',
'lo': 'lao',
'la': 'latin',
'lv': 'latvian',
'lt': 'lithuanian',
'lb': 'luxembourgish',
'mk': 'macedonian',
'mg': 'malagasy',
'ms': 'malay',
'ml': 'malayalam',
'mt': 'maltese',
'mi': 'maori',
'mr': 'marathi',
'mn': 'mongolian',
'my': 'myanmar (burmese)',
'ne': 'nepali',
'no': 'norwegian',
'ps': 'pashto',
'fa': 'persian',
'pl': 'polish',
'pt': 'portuguese',
'pa': 'punjabi',
'ro': 'romanian',
'ru': 'russian',
'sm': 'samoan',
'gd': 'scots gaelic',
'sr': 'serbian',
'st': 'sesotho',
'sn': 'shona',
'sd': 'sindhi',
'si': 'sinhala',
'sk': 'slovak',
'sl': 'slovenian',
'so': 'somali',
'es': 'spanish',
'su': 'sundanese',
'sw': 'swahili',
'sv': 'swedish',
'tg': 'tajik',
'ta': 'tamil',
'te': 'telugu',
'th': 'thai',
'tr': 'turkish',
'uk': 'ukrainian',
'ur': 'urdu',
'uz': 'uzbek',
'vi': 'vietnamese',
'cy': 'welsh',
'xh': 'xhosa',
'yi': 'yiddish',
'yo': 'yoruba',
'zu': 'zulu',
'fil': 'Filipino',
'he': 'Hebrew'
}
LANGCODES = dict(map(reversed, LANGUAGES.items()))

202
venv/lib/python3.7/site-packages/googletrans/gtoken.py Normal file
View File

@@ -0,0 +1,202 @@
# -*- coding: utf-8 -*-
import ast
import math
import re
import time
import requests
from googletrans.compat import PY3
from googletrans.compat import unicode
from googletrans.utils import rshift
class TokenAcquirer(object):
"""Google Translate API token generator
translate.google.com uses a token to authorize the requests. If you are
not Google, you do have this token and will have to pay for use.
This class is the result of reverse engineering on the obfuscated and
minified code used by Google to generate such token.
The token is based on a seed which is updated once per hour and on the
text that will be translated.
Both are combined - by some strange math - in order to generate a final
token (e.g. 744915.856682) which is used by the API to validate the
request.
This operation will cause an additional request to get an initial
token from translate.google.com.
Example usage:
>>> from googletrans.gtoken import TokenAcquirer
>>> acquirer = TokenAcquirer()
>>> text = 'test'
>>> tk = acquirer.do(text)
>>> tk
950629.577246
"""
RE_TKK = re.compile(r'tkk:\'(.+?)\'', re.DOTALL)
RE_RAWTKK = re.compile(r'tkk:\'(.+?)\'', re.DOTALL)
def __init__(self, tkk='0', session=None, host='translate.google.com'):
self.session = session or requests.Session()
self.tkk = tkk
self.host = host if 'http' in host else 'https://' + host
def _update(self):
"""update tkk
"""
# we don't need to update the base TKK value when it is still valid
now = math.floor(int(time.time() * 1000) / 3600000.0)
if self.tkk and int(self.tkk.split('.')[0]) == now:
return
r = self.session.get(self.host)
raw_tkk = self.RE_TKK.search(r.text)
if raw_tkk:
self.tkk = raw_tkk.group(1)
return
# this will be the same as python code after stripping out a reserved word 'var'
code = unicode(self.RE_TKK.search(r.text).group(1)).replace('var ', '')
# unescape special ascii characters such like a \x3d(=)
if PY3: # pragma: no cover
code = code.encode().decode('unicode-escape')
else: # pragma: no cover
code = code.decode('string_escape')
if code:
tree = ast.parse(code)
visit_return = False
operator = '+'
n, keys = 0, dict(a=0, b=0)
for node in ast.walk(tree):
if isinstance(node, ast.Assign):
name = node.targets[0].id
if name in keys:
if isinstance(node.value, ast.Num):
keys[name] = node.value.n
# the value can sometimes be negative
elif isinstance(node.value, ast.UnaryOp) and \
isinstance(node.value.op, ast.USub): # pragma: nocover
keys[name] = -node.value.operand.n
elif isinstance(node, ast.Return):
# parameters should be set after this point
visit_return = True
elif visit_return and isinstance(node, ast.Num):
n = node.n
elif visit_return and n > 0:
# the default operator is '+' but implement some more for
# all possible scenarios
if isinstance(node, ast.Add): # pragma: nocover
pass
elif isinstance(node, ast.Sub): # pragma: nocover
operator = '-'
elif isinstance(node, ast.Mult): # pragma: nocover
operator = '*'
elif isinstance(node, ast.Pow): # pragma: nocover
operator = '**'
elif isinstance(node, ast.BitXor): # pragma: nocover
operator = '^'
# a safety way to avoid Exceptions
clause = compile('{1}{0}{2}'.format(
operator, keys['a'], keys['b']), '', 'eval')
value = eval(clause, dict(__builtin__={}))
result = '{}.{}'.format(n, value)
self.tkk = result
def _lazy(self, value):
"""like lazy evalution, this method returns a lambda function that
returns value given.
We won't be needing this because this seems to have been built for
code obfuscation.
the original code of this method is as follows:
... code-block: javascript
var ek = function(a) {
return function() {
return a;
};
}
"""
return lambda: value
def _xr(self, a, b):
size_b = len(b)
c = 0
while c < size_b - 2:
d = b[c + 2]
d = ord(d[0]) - 87 if 'a' <= d else int(d)
d = rshift(a, d) if '+' == b[c + 1] else a << d
a = a + d & 4294967295 if '+' == b[c] else a ^ d
c += 3
return a
def acquire(self, text):
a = []
# Convert text to ints
for i in text:
val = ord(i)
if val < 0x10000:
a += [val]
else:
# Python doesn't natively use Unicode surrogates, so account for those
a += [
math.floor((val - 0x10000)/0x400 + 0xD800),
math.floor((val - 0x10000)%0x400 + 0xDC00)
]
b = self.tkk if self.tkk != '0' else ''
d = b.split('.')
b = int(d[0]) if len(d) > 1 else 0
# assume e means char code array
e = []
g = 0
size = len(text)
while g < size:
l = a[g]
# just append if l is less than 128(ascii: DEL)
if l < 128:
e.append(l)
# append calculated value if l is less than 2048
else:
if l < 2048:
e.append(l >> 6 | 192)
else:
# append calculated value if l matches special condition
if (l & 64512) == 55296 and g + 1 < size and \
a[g + 1] & 64512 == 56320:
g += 1
l = 65536 + ((l & 1023) << 10) + (a[g] & 1023) # This bracket is important
e.append(l >> 18 | 240)
e.append(l >> 12 & 63 | 128)
else:
e.append(l >> 12 | 224)
e.append(l >> 6 & 63 | 128)
e.append(l & 63 | 128)
g += 1
a = b
for i, value in enumerate(e):
a += value
a = self._xr(a, '+-a^+6')
a = self._xr(a, '+-3^+b+-f')
a ^= int(d[1]) if len(d) > 1 else 0
if a < 0: # pragma: nocover
a = (a & 2147483647) + 2147483648
a %= 1000000 # int(1E6)
return '{}.{}'.format(a, a ^ b)
def do(self, text):
self._update()
tk = self.acquire(text)
return tk

43
venv/lib/python3.7/site-packages/googletrans/models.py Normal file
View File

@@ -0,0 +1,43 @@
class Translated(object):
"""Translate result object
:param src: source langauge (default: auto)
:param dest: destination language (default: en)
:param origin: original text
:param text: translated text
:param pronunciation: pronunciation
"""
def __init__(self, src, dest, origin, text, pronunciation, extra_data=None):
self.src = src
self.dest = dest
self.origin = origin
self.text = text
self.pronunciation = pronunciation
self.extra_data = extra_data
def __str__(self): # pragma: nocover
return self.__unicode__()
def __unicode__(self): # pragma: nocover
return u'Translated(src={src}, dest={dest}, text={text}, pronunciation={pronunciation}, ' \
u'extra_data={extra_data})'.format(
src=self.src, dest=self.dest, text=self.text, pronunciation=self.pronunciation,
extra_data='"' + repr(self.extra_data)[:10] + '..."')
class Detected(object):
"""Language detection result object
:param lang: detected language
:param confidence: the confidence of detection result (0.00 to 1.00)
"""
def __init__(self, lang, confidence):
self.lang = lang
self.confidence = confidence
def __str__(self): # pragma: nocover
return self.__unicode__()
def __unicode__(self): # pragma: nocover
return u'Detected(lang={lang}, confidence={confidence})'.format(
lang=self.lang, confidence=self.confidence)

6
venv/lib/python3.7/site-packages/googletrans/urls.py Normal file
View File

@@ -0,0 +1,6 @@
# -*- coding: utf-8 -*-
"""
Predefined URLs used to make google translate requests.
"""
BASE = 'https://translate.google.com'
TRANSLATE = 'https://{host}/translate_a/single'

70
venv/lib/python3.7/site-packages/googletrans/utils.py Normal file
View File

@@ -0,0 +1,70 @@
"""A conversion module for googletrans"""
from __future__ import print_function
import re
import json
def build_params(query, src, dest, token):
params = {
'client': 't',
'sl': src,
'tl': dest,
'hl': dest,
'dt': ['at', 'bd', 'ex', 'ld', 'md', 'qca', 'rw', 'rm', 'ss', 't'],
'ie': 'UTF-8',
'oe': 'UTF-8',
'otf': 1,
'ssel': 0,
'tsel': 0,
'tk': token,
'q': query,
}
return params
def legacy_format_json(original):
# save state
states = []
text = original
# save position for double-quoted texts
for i, pos in enumerate(re.finditer('"', text)):
# pos.start() is a double-quote
p = pos.start() + 1
if i % 2 == 0:
nxt = text.find('"', p)
states.append((p, text[p:nxt]))
# replace all weired characters in text
while text.find(',,') > -1:
text = text.replace(',,', ',null,')
while text.find('[,') > -1:
text = text.replace('[,', '[null,')
# recover state
for i, pos in enumerate(re.finditer('"', text)):
p = pos.start() + 1
if i % 2 == 0:
j = int(i / 2)
nxt = text.find('"', p)
# replacing a portion of a string
# use slicing to extract those parts of the original string to be kept
text = text[:p] + states[j][1] + text[nxt:]
converted = json.loads(text)
return converted
def format_json(original):
try:
converted = json.loads(original)
except ValueError:
converted = legacy_format_json(original)
return converted
def rshift(val, n):
"""python port for '>>>'(right shift with padding)
"""
return (val % 0x100000000) >> n