Entry
Language detection module
Apr 2nd, 2009 06:50
engatoo engatoo, osososo didididi, Nathan Wallace, Hans Nowak, Snippet 192, Dinu C. Gherman
"""
Packages: text;miscellaneous.mundane
"""
"""
Is there anything already like a function that I can pass an
arbitrary string and it will tell me wether it is written in
English, French, German, etc.?
I imagine this could be rather simply implemented with some
dicts containing common prefixes and suffixes (as well as most
often used words like 'you', 'me', etc.) used in the respective
natural language. One could then calculate some likelihood for
the text to be in any of these languages classes and then return
the most likely one, or a list of them. I'm not sure, though,
how accents would be represented in a "portable" way (accross
multiple platforms), maybe in HTML...?
While writing this I started a small experiment to code what
I think about. Below you'll find what came out of it. Is there
anything more sophisticated out there, including a better scor-
ing/weighting method, maybe also for combinations of words, even
handling accents, perhaps?
"""
# langdetect.py -- Detect a natural language of a written text.
import string
en, fr, de = 'en', 'fr', 'de'
wordDict = {
'i':en, 'you':en, 'me':en, 'the':en, 'a':en,
'moi':fr, 'je':fr, 'toi':fr, 'vouz':fr, 'sur':fr, 'en':fr,
'sie':de, 'ich':de, 'um':de, 'an':de, 'ab':de}
prefixDict = {
'off':en, 'to':en, 'under':en, 'in':en, 'thou':en,
'mont':fr, 'contr':fr, 'mal':fr,
'ver':de, 'zu':de, 'los':de, 'gut':de}
suffixDict = {
'son':en, 'day':en, 'ing':en, 'ly':en, 'ght':en,
'ique':fr, 'tude':fr, 'ont':fr, 'nal':fr,
'tung':de, 'heim':de, 'zeug':de}
punct = """.,!?"()[]{}!§$%&/*+#"""
trans = string.maketrans(punct, ' '*len(punct))
def detectLanguage(input):
inp0 = string.lower(input)
inp1 = string.translate(inp0, trans)
inp2 = string.strip(inp1)
inp3 = string.split(inp2, ' ')
res = {en:0, fr:0, de:0}
explain = {en:[], fr:[], de:[]}
for word in inp3:
try :
v = wordDict[word]
res[v] = res[v] + 1
explain[v].append(word)
except KeyError:
pass
for p in prefixDict.keys():
try:
wp = word[:len(p)]
if p == wp:
prefixDict[wp]
res[v] = res[v] + 1
explain[v].append(word)
except KeyError:
pass
for s in suffixDict.keys():
try:
ws = word[-len(s):]
if s == ws:
suffixDict[ws]
res[v] = res[v] + 1
explain[v].append(word)
except KeyError:
pass
return res, explain
for phrase in ("I am in a good mood today.",
"Je suis en plaine forme.",
"Ich bin heute gut drauf."):
result, explain = detectLanguage(phrase)
print "Input:", phrase
print "Hypothesis:", result
print "Reasons:", explain
print
# Should print something like this:
#
# Input: I am in a good mood today.
# Hypothesis: {'en': 5, 'fr': 0, 'de': 0}
# Reasons: {'en': ['i', 'in', 'a', 'today', 'today'],
# 'fr': [],
# 'de': []}
#
# Input: Je suis en plaine forme.
# Hypothesis: {'en': 0, 'fr': 2, 'de': 0}
# Reasons: {'en': [], 'fr': ['je', 'en'], 'de': []}
#
# Input: Ich bin heute gut drauf.
# Hypothesis: {'en': 0, 'fr': 0, 'de': 2}
# Reasons: {'en': [], 'fr': [], 'de': ['ich', 'gut']}
http://dl3.com/link/category/voice
http://dl3.com/link/category/chat
http://dl3.com/link/category/song
http://dl3.com/link
http://www.dl3.com/smiles
http://www.dl3.com/cards
http://games.dl3.com
http://www.chataluae.com
http://www.dl3.com/vb/index.php
http://www.dl3.com/vb
http://www.dl3.com/vb
http://www.dl3.com
http://www.stop10.com
http://topics.ll6.com
http://topics.ll6.com
http://www.stop10.com
http://www.ll6.com/java.shtml
http://dlel.ll6.com/c21.html
http://dlel.ll6.com/c20.html
http://dlel.ll6.com/c19.html
http://dlel.ll6.com/c18.html
http://dlel.ll6.com/c17.html
http://dlel.ll6.com/categories.xml
http://dlel.ll6.com/sitemap.xml
http://dlel.ll6.com/c10.html
http://dlel.ll6.com/c9.html
http://dlel.ll6.com/c8.html
http://dlel.ll6.com/c7.html
http://dlel.ll6.com/c6.html
http://dlel.ll6.com/c5.html
http://dlel.ll6.com/c4.html
http://dlel.ll6.com/c3.html
http://dlel.ll6.com/c2.html
http://dlel.ll6.com/c10.html
http://dlel.ll6.com/c16.html
http://dlel.ll6.com/c15.html
http://dlel.ll6.com/c14.html
http://dlel.ll6.com/c13.html
http://dlel.ll6.com/c12.html
http://dlel.ll6.com/c11.html
http://voice.ll6.com
http://www.ll6.com
http://vb.ll6.com/forumdisplay.php?f=50
http://vb.ll6.com/forumdisplay.php?f=41
http://vb.ll6.com/forumdisplay.php?f=35
http://vb.ll6.com/forumdisplay.php?f=34
http://vb.ll6.com/forumdisplay.php?f=33
http://vb.ll6.com/forumdisplay.php?f=32
http://vb.ll6.com/forumdisplay.php?f=31
http://vb.ll6.com/forumdisplay.php?f=30
http://vb.ll6.com/forumdisplay.php?f=29
http://vb.ll6.com/forumdisplay.php?f=28
http://vb.ll6.com/forumdisplay.php?f=27
http://vb.ll6.com/forumdisplay.php?f=26
http://vb.ll6.com/forumdisplay.php?f=25
http://vb.ll6.com/forumdisplay.php?f=24
http://vb.ll6.com/forumdisplay.php?f=23
http://vb.ll6.com/forumdisplay.php?f=22
http://vb.ll6.com/forumdisplay.php?f=21
http://vb.ll6.com/forumdisplay.php?f=20
http://vb.ll6.com/forumdisplay.php?f=19
http://vb.ll6.com/forumdisplay.php?f=18
http://vb.ll6.com/forumdisplay.php?f=17
http://vb.ll6.com/forumdisplay.php?f=5
http://vb.ll6.com/forumdisplay.php?f=6
http://vb.ll6.com/forumdisplay.php?f=10
http://vb.ll6.com/forumdisplay.php?f=4
http://vb.ll6.com/index.php
http://ll6.com
http://www.alshalah.com
http://www.alshalah.com/chat1420.html
http://www.alshalah.com/music22.html
http://www.alshalah.com/music21.html
http://www.alshalah.com/chat1355.html
http://www.alshalah.com/chat1507.html
http://www.alshalah.com/download7.html
http://www.alshalah.com/download7.html
http://www.alshalah.com/chat1337.html
http://www.alshalah.com/chat100.html
http://www.alshalah.com/music12.html
http://www.alshalah.com/music12.html
http://www.alshalah.com/download5.html
http://www.alshalah.com/music23.html
http://www.alshalah.com/chat1339.html
http://www.alshalah.com/chat103.html
http://www.alshalah.com/music15.html
http://www.alshalah.com/chat1528.html
http://www.alshalah.com/chat1359.html
http://www.alshalah.com/chat1519.html
http://www.alshalah.com/chat1422.html
http://www.alshalah.com/music4.html
http://www.alshalah.com/chat101.html
http://www.alshalah.com/chat106.html
http://www.alshalah.com/chat102.html
http://www.alshalah.com/chat1166.html
http://www.alshalah.com/vote.php
http://www.alshalah.com/visit.php
http://www.alshalah.com/chat2821.html
http://www.alshalah.com/chat2821.html
http://www.alshalah.com/chat2821.html
http://www.alshalah.com/chat2821.html
http://www.alshalah.com/chat2820.html
http://www.alshalah.com/chat2820.html
http://www.alshalah.com/chat2819.html
http://www.alshalah.com/chat2819.html
http://www.alshalah.com/chat2818.html
http://www.alshalah.com/chat2818.html
http://www.alshalah.com/chat2817.html
http://www.alshalah.com/chat2817.html
http://www.alshalah.com/chat2816.html
http://www.alshalah.com/chat2816.html
http://www.alshalah.com/chat2815.html
http://www.alshalah.com/chat2815.html
http://www.alshalah.com/chat2814.html
http://www.alshalah.com/chat2814.html
http://www.alshalah.com/chat2813.html
http://www.alshalah.com/chat2813.html
http://www.alshalah.com/chat2812.html
http://www.alshalah.com/chat2812.html
http://www.alshalah.com/chat2813.html
http://www.alshalah.com/chat2811.html
http://www.alshalah.com/chat2811.html
http://www.alshalah.com/chat2810.html
http://www.alshalah.com/chat2810.html
http://www.alshalah.com/chat2809.html
http://www.alshalah.com/chat2809.html
http://www.alshalah.com/chat2808.html
http://www.alshalah.com/chat2808.html
http://www.alshalah.com/chat2807.html
http://www.alshalah.com/chat2807.html
http://www.alshalah.com/chat2806.html
http://www.alshalah.com/chat2806.html
http://www.alshalah.com/chat2805.html
http://www.alshalah.com/chat2804.html
http://www.alshalah.com/chat2803.html
http://www.alshalah.com/chat2803.html
http://www.alshalah.com/chat2802.html
http://www.alshalah.com/voice
http://www.alshalah.com/vb
http://www.alshalah.com/
http://www.alshalah.com/download3.html
http://www.alshalah.com/music6.html
http://www.alshalah.com/music6.html
http://www.alshalah.com/music5.html
http://www.alshalah.com/music5.html
http://www.alshalah.com/music4.html
http://www.alshalah.com/music4.html
http://www.alshalah.com/music3.html
http://www.alshalah.com/music3.html
http://www.alshalah.com/music22.html
http://www.alshalah.com/music21.html
http://www.alshalah.com/music19.html
http://www.alshalah.com/music18.html
http://www.alshalah.com/vb/sex202
http://www.alshalah.com/vb/sex205
http://www.alshalah.com/vb/sitemap
http://www.alshalah.com/vb/sitemap/f-183.html
http://www.alshalah.com/chat101.html
http://www.alshalah.com/chat101.html
http://www.alshalah.com/vb/picturet38947.html
http://www.alshalah.com/vb/picturet25902.html
http://www.alshalah.com/chat2801.html
http://www.alshalah.com/chat2800.html
http://www.alshalah.com/chat2799.html
http://www.alshalah.com/download10.html
http://www.alshalah.com/chat259.html
http://www.alshalah.com/vb/picturet939.html
http://www.alshalah.com/vb/picturet7317.html
http://www.alshalah.com/chat102.html
http://www.alshalah.com/chat312.html
http://www.alshalah.com/vb/picturet706.html
http://www.alshalah.com/vb/picturet22974-2.html
http://www.alshalah.com/vb/picturet26805.html
http://www.alshalah.com/chat601.html
http://www.alshalah.com/vb/picturet29167.html
http://www.alshalah.com/vb/picturet21358.html
http://www.alshalah.com/vb/picturet42591.html
http://www.alshalah.com/vb/picturet10141.html
http://www.alshalah.com/chat960.html
http://www.alshalah.com/chat658.html
http://www.alshalah.com/chat102.html
http://www.alshalah.com/chat1166.html
http://www.alshalah.com/vb/picturet34614.html
http://www.alshalah.com/chat2470.html
http://www.alshalah.com/vb/picturet44915.html
http://www.alshalah.com/download5.html
http://www.alshalah.com/chat2408.html
http://www.alshalah.com/chat2408.html
http://www.alshalah.com/vb/picturet24559.html
http://www.alshalah.com/music11.html
http://www.alshalah.com/vb/picturet40695.html
http://www.alshalah.com/vb/picturet1067.html
http://www.stop10.com
http://www.alshalah.com/chat27.html
http://www.alshalah.com/vb/picturet10871.html
http://www.alshalah.com/vb/picturet46798.html
http://www.alshalah.com/vb/picturet4995.html
http://www.alshalah.com/vb/picturet4445.html
http://www.alshalah.com/vb/picturet17029.html
http://www.alshalah.com/chat2794.html
http://www.alshalah.com/vb/picturet42026.html
http://www.alshalah.com/vb/picturet28063.html
http://www.alshalah.com/vb/picturet27011.html
http://www.alshalah.com/vb/picturet3521.html
http://www.alshalah.com/vb/picturet25360.html
http://www.alshalah.com/chat2793.html
http://www.alshalah.com/chat886.html
http://www.alshalah.com/vb/picturet274.htmll
http://www.alshalah.com/vb/picturet274.html
http://www.alshalah.com/music16.html
http://www.alshalah.com/download7.html
http://www.alshalah.com/vb/picturet43943.html
http://www.alshalah.com/chat2666.html
http://www.alshalah.com/music28.html
http://www.alshalah.com/music28.html
http://www.alshalah.com/music15.html
http://www.alshalah.com/music28.html
http://www.roo7e.com
http://www.roo7e.com
http://www.al33ab.com
http://ksacam.com
http://chat.ksacam.com
http://ksacam.info
http://oo2o.com
http://s4cam.com
http://saudi4cam.com
http://voice.ksacam.com
http://سعودي-كام.oo2o.com
http://سعوديكام.com
http://xn--mgbply5cnrr.com
http://www.3rabstarz.com/vb
http://www.3rabstarz.com/vb
http://www.3rabstarz.com/
http://www.3rabstarz.com/vb
http://www.3rabstarz.com/
http://www.dir.3rabstarz.com/
http://www.up.3rabstarz.com
http://www.3rabstarz.com/vb/forumdisplay.php?f=5
http://www.3rabstarz.com/vb/forumdisplay.php?f=116
http://www.3rabstarz.com/vb/forumdisplay.php?f=66
http://www.3rabstarz.com/vb/forumdisplay.php?f=78
http://www.3rabstarz.com/vb/forumdisplay.php?f=147
http://www.3rabstarz.com/vb/forumdisplay.php?f=2
http://www.3rabstarz.com/vb/forumdisplay.php?f=31
http://www.3rabstarz.com/vb/forumdisplay.php?f=136
http://www.3rabstarz.com/vb/forumdisplay.php?f=34
http://www.3rabstarz.com/vb/forumdisplay.php?f=89
http://www.3rabstarz.com/vb/forumdisplay.php?f=26
http://www.3rabstarz.com/vb/forumdisplay.php?f=72
http://www.3rabstarz.com/vb/forumdisplay.php?f=73
http://www.3rabstarz.com/vb/forumdisplay.php?f=149
http://www.3rabstarz.com/vb/forumdisplay.php?f=150
http://www.3rabstarz.com/vb/forumdisplay.php?f=151
http://www.3rabstarz.com/vb/forumdisplay.php?f=152
http://www.3rabstarz.com/vb/forumdisplay.php?f=67
http://www.3rabstarz.com/vb/forumdisplay.php?f=3
http://www.3rabstarz.com/vb/forumdisplay.php?f=37
http://www.3rabstarz.com/vb/forumdisplay.php?f=115
http://www.3rabstarz.com/vb/forumdisplay.php?f=76
http://www.3rabstarz.com/vb/forumdisplay.php?f=77
http://www.3rabstarz.com/vb/forumdisplay.php?f=36
http://www.3rabstarz.com/vb/forumdisplay.php?f=153
http://www.3rabstarz.com/vb/forumdisplay.php?f=122
http://www.3rabstarz.com/vb/forumdisplay.php?f=118
http://www.3rabstarz.com/vb/forumdisplay.php?f=119
http://www.3rabstarz.com/vb/forumdisplay.php?f=120
http://www.3rabstarz.com/vb/forumdisplay.php?f=121
http://www.3rabstarz.com/vb/forumdisplay.php?f=35
http://www.3rabstarz.com/vb/forumdisplay.php?f=65
http://www.3rabstarz.com/vb/forumdisplay.php?f=7
http://www.3rabstarz.com/vb/forumdisplay.php?f=9
http://www.3rabstarz.com/vb/forumdisplay.php?f=11
http://www.3rabstarz.com/vb/forumdisplay.php?f=142
http://www.3rabstarz.com/vb/forumdisplay.php?f=143
http://www.3rabstarz.com/vb/forumdisplay.php?f=146
http://www.3rabstarz.com/vb/forumdisplay.php?f=140
http://www.3rabstarz.com/vb/forumdisplay.php?f=137
http://www.3rabstarz.com/vb/forumdisplay.php?f=114
http://www.3rabstarz.com/vb/forumdisplay.php?f=117
http://www.3rabstarz.com/vb/forumdisplay.php?f=113
http://www.3rabstarz.com/vb/forumdisplay.php?f=51
http://www.3rabstarz.com/vb/forumdisplay.php?f=84
http://www.3rabstarz.com/vb/forumdisplay.php?f=138
http://www.3rabstarz.com/vb/forumdisplay.php?f=139
http://www.3rabstarz.com/vb/forumdisplay.php?f=15
http://www.3rabstarz.com/vb/forumdisplay.php?f=130
http://www.3rabstarz.com/vb/forumdisplay.php?f=97
http://www.3rabstarz.com/vb/forumdisplay.php?f=102
http://www.3rabstarz.com/vb/forumdisplay.php?f=101
http://www.3rabstarz.com/vb/forumdisplay.php?f=104
http://www.3rabstarz.com/vb/forumdisplay.php?f=131
http://www.3rabstarz.com/vb/forumdisplay.php?f=103
http://www.3rabstarz.com/vb/forumdisplay.php?f=99
http://www.3rabstarz.com/vb/forumdisplay.php?f=59
http://www.3rabstarz.com/vb/forumdisplay.php?f=60
http://www.3rabstarz.com/vb/forumdisplay.php?f=39
http://www.3rabstarz.com/vb/forumdisplay.php?f=40
http://www.3rabstarz.com/vb/forumdisplay.php?f=42
http://www.3rabstarz.com/vb/forumdisplay.php?f=4
http://www.3rabstarz.com/vb/forumdisplay.php?f=1
http://www.3rabstarz.com/vb/forumdisplay.php?f=88
http://www.3rabstarz.com/vb/forumdisplay.php?f=148
http://www.3rabstarz.com/vb/forumdisplay.php?f=62
http://www.3rabstarz.com/vb/forumdisplay.php?f=63
http://www.3rabstarz.com/vb/forumdisplay.php?f=64
http://www.3rabstarz.com/vb/forumdisplay.php?f=6
http://www.3rabstarz.com/vb/forumdisplay.php?f=12
http://www.3rabstarz.com/vb/forumdisplay.php?f=96
http://www.3rabstarz.com/vb/forumdisplay.php?f=58
http://www.3rabstarz.com/vb/forumdisplay.php?f=38
