source: internals/2016/aptoideimagesdetector/trunk/Source Code/Linguage Extractor/NP Extractor/np_extractor.py @ 16308

Last change on this file since 16308 was 16308, checked in by dferreira, 3 years ago

Adition of relevant links to linguage processing and text mining. To be understood and tested.

File size: 3.4 KB
Line 
1# coding=UTF-8
2import nltk
3from nltk.corpus import brown
4
5# This is a fast and simple noun phrase extractor (based on NLTK)
6# Feel free to use it, just keep a link back to this post
7# http://thetokenizer.com/2013/05/09/efficient-way-to-extract-the-main-topics-of-a-sentence/
8# Create by Shlomi Babluki
9# May, 2013
10
11
12# This is our fast Part of Speech tagger
13#############################################################################
14brown_train = brown.tagged_sents(categories='news')
15regexp_tagger = nltk.RegexpTagger(
16    [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'),
17     (r'(-|:|;)$', ':'),
18     (r'\'*$', 'MD'),
19     (r'(The|the|A|a|An|an)$', 'AT'),
20     (r'.*able$', 'JJ'),
21     (r'^[A-Z].*$', 'NNP'),
22     (r'.*ness$', 'NN'),
23     (r'.*ly$', 'RB'),
24     (r'.*s$', 'NNS'),
25     (r'.*ing$', 'VBG'),
26     (r'.*ed$', 'VBD'),
27     (r'.*', 'NN')
28])
29unigram_tagger = nltk.UnigramTagger(brown_train, backoff=regexp_tagger)
30bigram_tagger = nltk.BigramTagger(brown_train, backoff=unigram_tagger)
31#############################################################################
32
33
34# This is our semi-CFG; Extend it according to your own needs
35#############################################################################
36cfg = {}
37cfg["NNP+NNP"] = "NNP"
38cfg["NN+NN"] = "NNI"
39cfg["NNI+NN"] = "NNI"
40cfg["JJ+JJ"] = "JJ"
41cfg["JJ+NN"] = "NNI"
42#############################################################################
43
44
45class NPExtractor(object):
46
47    def __init__(self, sentence):
48        self.sentence = sentence
49
50    # Split the sentence into singlw words/tokens
51    def tokenize_sentence(self, sentence):
52        tokens = nltk.word_tokenize(sentence)
53        return tokens
54
55    # Normalize brown corpus' tags ("NN", "NN-PL", "NNS" > "NN")
56    def normalize_tags(self, tagged):
57        n_tagged = []
58        for t in tagged:
59            if t[1] == "NP-TL" or t[1] == "NP":
60                n_tagged.append((t[0], "NNP"))
61                continue
62            if t[1].endswith("-TL"):
63                n_tagged.append((t[0], t[1][:-3]))
64                continue
65            if t[1].endswith("S"):
66                n_tagged.append((t[0], t[1][:-1]))
67                continue
68            n_tagged.append((t[0], t[1]))
69        return n_tagged
70
71    # Extract the main topics from the sentence
72    def extract(self):
73
74        tokens = self.tokenize_sentence(self.sentence)
75        tags = self.normalize_tags(bigram_tagger.tag(tokens))
76
77        merge = True
78        while merge:
79            merge = False
80            for x in range(0, len(tags) - 1):
81                t1 = tags[x]
82                t2 = tags[x + 1]
83                key = "%s+%s" % (t1[1], t2[1])
84                value = cfg.get(key, '')
85                if value:
86                    merge = True
87                    tags.pop(x)
88                    tags.pop(x)
89                    match = "%s %s" % (t1[0], t2[0])
90                    pos = value
91                    tags.insert(x, (match, pos))
92                    break
93
94        matches = []
95        for t in tags:
96            if t[1] == "NNP" or t[1] == "NNI":
97            #if t[1] == "NNP" or t[1] == "NNI" or t[1] == "NN":
98                matches.append(t[0])
99        return matches
100
101
102# Main method, just run "python np_extractor.py"
103def main():
104
105    sentence = "Swayy is a beautiful new dashboard for discovering and curating online content."
106    np_extractor = NPExtractor(sentence)
107    result = np_extractor.extract()
108    print "This sentence is about: %s" % ", ".join(result)
109
110if __name__ == '__main__':
111    main()
Note: See TracBrowser for help on using the repository browser.