NLP_Example.py

from ggs import *
import numpy as np
import matplotlib.pyplot as plt
import gensim


#FILL IN HERE WITH THE WORD EMBEDDING VECTOR THAT YOU WANT TO USE
#Google News embedding available for download at https://code.google.com/archive/p/word2vec/
model = gensim.models.KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True)

#Preprocessing of Wikipedia data

numThresh = 5 #If word re-occurs more than numThresh times and in multiple articles, then remove it
repeats = dict()
section = 0
with open('Data/WikipediaText.txt','r') as f:
    for line in f:
        for raw in line.split():
            word = filter(str.isalpha, raw).lower()
            if (word == 'breakpointsection'): #Pre-defined "code word" to indicate a new article!
                section = section + 1
            if word in repeats:
                if (section != repeats[word][1]):
                    repeats[word] = [repeats[word][0]+1, -1]
                else:
                    repeats[word] = [repeats[word][0]+1, section]
            else:
                repeats[word] = [1,section]

with open('ProcessedText.txt', 'w') as fileName:
    with open('Data/WikipediaText.txt','r') as f:
        for line in f:
            for raw in line.split():
                word = filter(str.isalpha, raw)

                if(word == 'BREAKPOINTSECTION'):
                    fileName.write(word+' ')
                elif (repeats[word.lower()][0] <= numThresh):
                    fileName.write(word+' ')
                elif (repeats[word.lower()][1] != -1):
                    fileName.write(word+' ')
                # else:
                #     print "Skipping", word
                
#Get Word2Vec Embedding
numWords = 1282
data = np.zeros((numWords,300))
counter = 0
with open('ProcessedText.txt','r') as f:
    for line in f:
        for raw in line.split():
            word = filter(str.isalpha, raw)
            if (word == 'BREAKPOINTSECTION'):
                print "Breakpoint at", counter
                continue
            try:
                val = model.word_vec(word)
                data[counter,:] = val
                counter = counter + 1
            except KeyError:
                continue

#Run GGS
print "Running GGS"
a = GGS(data.T, 7, 0.0001)
print a