Summarizing the Law

#Goal: create word vectors
from __future__ import absolute_import, division, print_function
#for word encoding
import codecs
#regex
#import glob
#concurrency
import multiprocessing
import os 
import pprint
import re
import nltk
import gensim
import sklearn.manifold
import numpy as np
import matplotlib.pyplot as plt 
import pandas as pd 
import seaborn as sns
#currdir = os.path.dirname(__file__)

f = open('constitution.txt', 'r')
text = f.read()

#process and clean data 
nltk.download('punkt') #pretained tokenizer
nltk.download('stopwords') #words like and, or, an, a

tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')

#Split the text file into sentences
raw_sentences = tokenizer.tokenize(text)
print(raw_sentences[1])

#converting the constitution to a list of words 
def sentence_to_wordList(textFile):
  words = re.sub("[^\w]", " ",  textFile).split()
  return words

#create an instance where for every sentense, each word is tokenized
sentence = []
for raw_sentence in raw_sentences:
  sentence.append(sentence_to_wordList(raw_sentence))

print(raw_sentences[5])
print(sentence[5])

import sys
from os import path
import numpy as np
from PIL import Image
from wordcloud import WordCloud, STOPWORDS

# get path to script's directory
#currdir = path.dirname(__file__)
def create_wordcloud(text_file):
    # create numpy araay for wordcloud mask image
    #mask = np.array(Image.open(path.join(currdir, "cloud.png")))
    mask = np.array(Image.open("cloud.png"))
    # create set of stopwords	
    stopwords = set(STOPWORDS)

    # create wordcloud object
    wc = WordCloud(background_color="white",
                    max_words=200, 
                    mask=mask,
                    stopwords=stopwords)
    
    # generate wordcloud
    wc.generate(text)

    # save wordcloud
    wc.to_file("wc.png")


if __name__ == "__main__":

    # generate wordcloud
    create_wordcloud(text)

from IPython.display import Image
Image("wc.png")

#vectors help with Distance, Similarity and Ranking of Words

#the more features with have, the accurate our model gets but also the more expensive to train 
num_features = 1000
min_word_count = 2

#more workers, the faster we train
num_workers = multiprocessing.cpu_count()
#Size of words to look at, at a time
context_size = 7
#Downsample setting for frequent words
downsampling = 1e-3
#Seed, random number generator, good for debugging
seed = 1

model = gensim.models.Word2Vec(sg=1, seed=seed, workers=num_workers, size=num_features, min_count = min_word_count, window=context_size, sample= downsampling)

model.build_vocab(sentence)
model.train(sentence, total_examples=len(sentence), epochs= 10)

model.most_similar("constitution")

model.most_similar('Commander')