# Connection to Reddit's API
import pandas as pd
import praw

# I've had to clear this section out since it contains person information
# To run this code, the person would need to go to Reddit.com and register
#  a new application, which would provide an API key to input below.
reddit = praw.Reddit(
    client_id="<fill me out>",
    client_secret="<fill me out>",
    user_agent="SentimentTest/0.0.1",
)

print(reddit.read_only)
# Output: True if successful connection

from nltk.stem.wordnet import WordNetLemmatizer
from nltk.corpus import twitter_samples, stopwords
from nltk.tag import pos_tag
from nltk.tokenize import word_tokenize, TweetTokenizer
from nltk import FreqDist, classify, NaiveBayesClassifier

import re, string, random

tweet_tknzr = TweetTokenizer() #Use this over word_tokenize because of contractions

def remove_noise(tokens):
    """
    1. Removes 'noise' such as HTML/URL/Emojis to perform analysis
    2. Lemmatize/Normalize like words
    3. Remove English stop words
    """
    cleaned_tokens = []
    stop_words = stopwords.words('english')

    for token, tag in pos_tag(tokens):
        token = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[[email protected]&+#]|[!*\(\),]|'\
                       '(?:%[0-9a-fA-F][0-9a-fA-F]))+','', token)
        token = re.sub("(@[A-Za-z0-9_]+)","", token)

        if tag.startswith("NN"):
            pos = 'n'
        elif tag.startswith('VB'):
            pos = 'v'
        else:
            pos = 'a'
            
        lemmatizer = WordNetLemmatizer()
        token = lemmatizer.lemmatize(token, pos)

        if len(token) > 0 and token not in string.punctuation and token.lower() not in stop_words:
            cleaned_tokens.append(token.lower())
    return cleaned_tokens

def get_all_words(cleaned_tokens_list):
    # Return all words from cleaned token list
    for tokens in cleaned_tokens_list:
        for token in tokens:
            yield token

def get_tokens_for_model(cleaned_tokens_list):
    # Return all tokens for model classification
    for tokens in cleaned_tokens_list:
        yield dict([token, True] for token in tokens)

"""
Utilize Positive / Negative tweets that come packaged with
the NLTK to help train the model for classification. This consists
of a total of 14,000 tweets - 7000 positive and 7000 negative.
"""

positive_tweet_tokens = twitter_samples.tokenized('positive_tweets.json')
negative_tweet_tokens = twitter_samples.tokenized('negative_tweets.json')

positive_cleaned_tokens_list = []
negative_cleaned_tokens_list = []

#Run through cleaning algorithm
for tokens in positive_tweet_tokens:
    positive_cleaned_tokens_list.append(remove_noise(tokens))

#Run through cleaning algorithm
for tokens in negative_tweet_tokens:
    negative_cleaned_tokens_list.append(remove_noise(tokens))

#Test to show words leftover
#all_pos_words = get_all_words(positive_cleaned_tokens_list)

# Return tokens for model
positive_tokens_for_model = get_tokens_for_model(positive_cleaned_tokens_list)
negative_tokens_for_model = get_tokens_for_model(negative_cleaned_tokens_list)

# Form dataset
positive_dataset = [(tweet_dict, "Positive")
                     for tweet_dict in positive_tokens_for_model]

negative_dataset = [(tweet_dict, "Negative")
                     for tweet_dict in negative_tokens_for_model]

dataset = positive_dataset + negative_dataset

# Shuffle to randomize
random.shuffle(dataset)

# Break into training / testing datasets
train_data = dataset[:7000]
test_data = dataset[7000:]

# Form classifier model
classifier = NaiveBayesClassifier.train(train_data)

# Print accuracy based on test data
print("Accuracy is:", classify.accuracy(classifier, test_data))
print(classifier.show_most_informative_features(10))

results = []

sample_1 = 'I really like this idea, good job Steve!'
sample_2 = 'This is the worst idea I have ever heard of, stop talking to me!'

custom_tokens_1 = remove_noise(tweet_tknzr.tokenize(sample_1))
custom_tokens_2 = remove_noise(tweet_tknzr.tokenize(sample_2))

classifier.classify(dict([token, True] for token in custom_tokens_1))

classifier.classify(dict([token, True] for token in custom_tokens_2))

# Assign subreddits and print out some basic info to test
subreddits = [reddit.subreddit("Liberal"),reddit.subreddit("Conservative"),reddit.subreddit("NeutralPolitics")]

for subreddit in subreddits:
    print(subreddit.display_name)
    print(subreddit.description)

# Grab ID's for 100 stories in each of the three subreddits (the "hot" 100)
story_ids = {}

for subreddit in subreddits:
    for submission in subreddit.hot(limit=100):
        
        #Loop through comment hierarchy
        while True:
            try:
                submission.comments.replace_more(limit=None) #Remove 'show more comments'
                break
            except PossibleExceptions:
                print("Handling replace_more exception")
                sleep(1)

        story_ids[submission.id] = [subreddit.display_name,submission.title,submission.comments.list()] #Put comments in dictionary
        
# These ID's allow us to pull the sentiment for each individual story per subreddit
for key, value in story_ids.items():
     print(key, '->', value)

# Dump comments into a pandas dataframe
df_rows = []

for key, value in story_ids.items():
    # Loop through comment lists in value[2]
    for comment in value[2]:
        df_rows.append([key, value[0], value[1], comment.id, comment.score, comment.created, comment.body])

df = pd.DataFrame(df_rows, columns=['Post ID', 'Subreddit', 'Post Title', 'Comment ID', 'Score', 'Created', 'Body'])
df = df[(df['Body'] == '[deleted]') | (df['Body'] == '[removed]')==False].reset_index(drop=True) #Exclude any deleted/removed comments
df.head()

# Here we can see we have a total of ~18k comments
df.info()

# Gather sentiment for every comment and dump in list

custom_tokens = []
results = []
num_pos = 0
num_neg = 0
per_pos = 0.0
per_neg = 0.0

for index, row in df.iterrows():
    custom_tokens.append(remove_noise(tweet_tknzr.tokenize(row['Body'])))
    results.append([row['Post ID'], row['Subreddit'], row['Body'], classifier.classify(dict([token, True] for token in custom_tokens[index]))])

# Analyze sentiment gathered for all comments on all stories by subreddit
# Display percentage

df_r = pd.DataFrame([result[1:] for result in results], columns=['Subreddit','Body','Sentiment'])
df_r_p = df_r[['Subreddit','Sentiment']].pivot_table(index='Subreddit', columns='Sentiment', aggfunc=len, fill_value=0)
df_r_p['% Negative'] = round((df_r_p['Negative'] / (df_r_p['Positive'] + df_r_p['Negative'])) * 100,2)
df_r_p['% Positive'] = round((df_r_p['Positive'] / (df_r_p['Positive'] + df_r_p['Negative'])) * 100,2)
df_r_p

# Sampling of Pos/Neg Comments
df_r[df_r['Sentiment'] == 'Negative']['Body'].tolist() #Negative

df_r[df_r['Sentiment'] == 'Positive']['Body'].tolist() #Positive

# Analyze sentiment gathered for all comments on all stories by subreddit AND story ID
# Display percentage

df_r = pd.DataFrame(results, columns=['Post ID','Subreddit','Body','Sentiment'])
df_r_p = df_r[['Post ID','Subreddit','Sentiment']].pivot_table(index=['Subreddit','Post ID'], columns='Sentiment', aggfunc=len, fill_value=0)
df_r_p['% Negative'] = round((df_r_p['Negative'] / (df_r_p['Positive'] + df_r_p['Negative'])) * 100,2)
df_r_p['% Positive'] = round((df_r_p['Positive'] / (df_r_p['Positive'] + df_r_p['Negative'])) * 100,2)
df_r_p

# Group by Subreddit and take average of averages
df_r_p.groupby('Subreddit').agg({'Negative':'sum', 
                         'Positive':'sum', 
                         '% Negative':'mean', 
                         '% Positive':'mean'}).round(decimals=2)

# Assign subreddits and print out some basic info to test
subreddits = [reddit.subreddit("Liberal"),reddit.subreddit("Conservative"),reddit.subreddit("NeutralPolitics")]

# Grab ID's for 100 stories in each subreddit (the "hot" 100)
story_ids = {}

for subreddit in subreddits:
    for submission in subreddit.hot(limit=100):
        
        #Loop through comment hierarchy and pull all levels
        while True:
            try:
                submission.comments.replace_more(limit=None) #Remove 'show more comments'
                break
            except PossibleExceptions:
                print("Handling replace_more exception")
                sleep(1)
        
        #Pull only top-level comments
        story_ids[submission.id] = [subreddit.display_name,submission.title,[top_level_comment for top_level_comment in submission.comments]] #Put in dictionary
        
# These ID's allow us to pull the sentiment for each individual story per subreddit
for key, value in story_ids.items():
     print(key, '->', value)

# Dump comments into a pandas dataframe
df_rows = []

for key, value in story_ids.items():
    # Loop through comment lists in value[2]
    for comment in value[2]:
        df_rows.append([key, value[0], value[1], comment.id, comment.score, comment.created, comment.body])

df = pd.DataFrame(df_rows, columns=['Post ID', 'Subreddit', 'Post Title', 'Comment ID', 'Score', 'Created', 'Body'])
df = df[(df['Body'] == '[deleted]') | (df['Body'] == '[removed]')==False].reset_index(drop=True) #Exclude any deleted/removed comments
df.head()

# Very limited set now at 2890 comments, compared to 18k
df.info()

# Gather sentiment for every comment and dump in list

custom_tokens = []
results = []
num_pos = 0
num_neg = 0
per_pos = 0.0
per_neg = 0.0

for index, row in df.iterrows():
    custom_tokens.append(remove_noise(tweet_tknzr.tokenize(row['Body'])))
    results.append([row['Post ID'], row['Subreddit'], row['Body'], classifier.classify(dict([token, True] for token in custom_tokens[index]))])

# Analyze sentiment gathered for top-level comments on all stories by subreddit
# Display percentage

df_r = pd.DataFrame([result[1:] for result in results], columns=['Subreddit','Body','Sentiment'])
df_r_p = df_r[['Subreddit','Sentiment']].pivot_table(index='Subreddit', columns='Sentiment', aggfunc=len, fill_value=0)
df_r_p['% Negative'] = round((df_r_p['Negative'] / (df_r_p['Positive'] + df_r_p['Negative'])) * 100,2)
df_r_p['% Positive'] = round((df_r_p['Positive'] / (df_r_p['Positive'] + df_r_p['Negative'])) * 100,2)
df_r_p

# Analyze sentiment gathered for top-level comments on all stories by subreddit AND story ID
# Display percentage

df_r = pd.DataFrame(results, columns=['Post ID','Subreddit','Body','Sentiment'])
df_r_p = df_r[['Post ID','Subreddit','Sentiment']].pivot_table(index=['Subreddit','Post ID'], columns='Sentiment', aggfunc=len, fill_value=0)
df_r_p['% Negative'] = round((df_r_p['Negative'] / (df_r_p['Positive'] + df_r_p['Negative'])) * 100,2)
df_r_p['% Positive'] = round((df_r_p['Positive'] / (df_r_p['Positive'] + df_r_p['Negative'])) * 100,2)
df_r_p

# Group by Subreddit and take average of averages
df_r_p.groupby('Subreddit').agg({'Negative':'sum', 
                         'Positive':'sum', 
                         '% Negative':'mean', 
                         '% Positive':'mean'}).round(decimals=2)

def returnAllComments():
    '''
    Return all comments from a given submission into a DataFrame
    '''
    #Loop through comment hierarchy and pull all levels
    df_rows = []

    #Loop through comment hierarchy
    while True:
        try:
            submission.comments.replace_more(limit=None) #Flatten comment tree
            break
        except PossibleExceptions:
            print("Handling replace_more exception")
            sleep(1)

    comments = submission.comments.list() #Move to list

    # Loop through comment lists
    for comment in comments:
        df_rows.append([comment.id, comment.score, comment.created, comment.body])

    df = pd.DataFrame(df_rows, columns=['Comment ID', 'Score', 'Created', 'Body'])
    return df[(df['Body'] == '[deleted]') | (df['Body'] == '[removed]')==False].reset_index(drop=True) #Exclude any deleted/removed comments

def runSingleStorySentimentAnalysis(df):
    '''
    Run sentiment analysis on a given DataFrame and return analysis
    '''
    # Gather sentiment for every comment and dump in list
    custom_tokens = []
    results = []
    num_pos = 0
    num_neg = 0
    per_pos = 0.0
    per_neg = 0.0

    for index, row in df.iterrows():
        custom_tokens.append(remove_noise(tweet_tknzr.tokenize(row['Body'])))
        results.append([classifier.classify(dict([token, True] for token in custom_tokens[index]))])

    # Display percentages
    df_r = pd.DataFrame([result for result in results], columns=['Sentiment'])
    df_r_p = pd.DataFrame(df_r.groupby('Sentiment').size(), columns=['Count'])
    df_r_p['% of Total'] = round((df_r_p / df_r_p.sum())*100,2)
    return df_r_p

def returnComments(df):
    '''
    Run sentiment analysis on a given DataFrame and return detail results to review
    '''
    # Gather sentiment for every comment and dump in list
    custom_tokens = []
    results = []
    num_pos = 0
    num_neg = 0
    per_pos = 0.0
    per_neg = 0.0

    for index, row in df.iterrows():
        custom_tokens.append(remove_noise(tweet_tknzr.tokenize(row['Body'])))
        results.append([row['Body'], classifier.classify(dict([token, True] for token in custom_tokens[index]))])
       
    #df_r = pd.DataFrame([result for result in results], columns=['Body','Sentiment'])
    return results

# https://www.reddit.com/r/Conservative/comments/hi3u79/reddit_bans_the_donald_forum_as_part_of_major/
    
submission = reddit.submission(id="hi3u79")
print(submission.title)

df = returnAllComments()
df

runSingleStorySentimentAnalysis(df)

returnComments(df)

#https://www.reddit.com/r/Liberal/comments/hi4eva/reddit_finally_bans_hate_speech_removes_2000/

submission = reddit.submission(id="hi4eva")
print(submission.title)

df = returnAllComments()
df

runSingleStorySentimentAnalysis(df)

# Show details of how comments were classified
returnComments(df)