Upload NComparer.py

#!/usr/bin/env python

# coding: utf-8

# In[2]:

import os

import glob

import os

import subprocess

import sys

import argparse

import re

import csv

import gzip

import igraph

from igraph import *

import matplotlib

from matplotlib import *

import spacy

from spacy.lang.en.stop_words import STOP_WORDS

from spacy.lang.en import English

# In[4]:

path_to_txt =input("Pfad für TXT Files: ") 

for filename in os.listdir(path_to_txt):

    essay = os.path.join(path_to_txt, filename)

    if os.path.isfile(essay) and filename.endswith('.txt') and not filename.endswith('_lastline.txt'):

        #print(filename)

        read_essay = open(essay, "r")

        absatz = read_essay.read()

        if absatz[-1] == ".":

            absatz = absatz.split('\n')[-1]

        else:

            absatz = absatz.split('\n')[-2]    

        make_file = open((essay.rsplit( ".", 1 )[ 0 ] ) + "_lastline.txt", "w") 

        make_file.write(absatz)

        make_file.close()

# In[6]:

def ontology_create(data):

    """

    Function to create an Ontology of Concepts from Conceptnet 5.6.0, 

    but only those with the label 'EN' (all concepts in english)

    Has to be in the same folder as Conceptnet_assertions 5.6.0 

    Parameters

    ----------

    data : 

        input data from which the concepts will be extracted

    Returns

    -------

    unique_nodes : list

        unique concepts with label 'EN' from ConceptNet 5.6.0

    """

    graph = []

    clean_graph=[]

    n=0

    a=0

    words=[]

    new_tags=[]

    tags=[]

    tag=''

    flag=False

    count=0

    nodes_1=[]

    nodes_2=[]

    vertices=[]

    relation=[]

    g = Graph()

    edges=[]

    with open(data, 'r', encoding ='utf8') as csvfile:

        read = csv.reader(csvfile, delimiter='\t', quotechar='\t') 

        for line in read: 

               if '/c/en/' in line[2] and '/c/en/' in line[3]:

                        nodes_1.append(line[2].replace('/c/en/','').replace('/n','').replace('/v','').replace('/a','').replace('/s','').replace('/r',''))

                        nodes_2.append(line[3].replace('/c/en/','').replace('/n','').replace('/v','').replace('/a','').replace('/s','').replace('/r',''))

                        relation.append(line[1])

                        count=count+1

#                         if count==50:

#                             break

    edges=zip(nodes_1,nodes_2)

    unique_nodes = list(set(nodes_1).union(nodes_2))

    vertex = unique_nodes[:]

    #print(unique_nodes)

    for i in range(len(unique_nodes)):

         g.add_vertices(unique_nodes[i])

         #store it in dictionary

    return unique_nodes

onto_list = ontology_create('conceptnet-assertions-5.6.0.csv')

# In[26]:

sentence_list = []

for filename in os.listdir(path_to_txt):

    essay = os.path.join(path_to_txt, filename)

    if os.path.isfile(essay) and filename.endswith('_lastline.txt'):

        read_essay = open(essay, "r")

        absatz = read_essay.read()

        sentence_list.append(absatz)

word_list = []

for sentence in sentence_list:

    word_list.append(sentence.lower().split())

print(sentence_list[:5])

# In[22]:

stop_word_list = ['i',',',';','.','-', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", "you've", "you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', "she's", 'her', 'hers', 'herself', 'it', "it's", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', "that'll", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', "don't", 'should', "should've", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', "aren't", 'couldn', "couldn't", 'didn', "didn't", 'doesn', "doesn't", 'hadn', "hadn't", 'hasn', "hasn't", 'haven', "haven't", 'isn', "isn't", 'ma', 'mightn', "mightn't", 'mustn', "mustn't", 'needn', "needn't", 'shan', "shan't", 'shouldn', "shouldn't", 'wasn', "wasn't", 'weren', "weren't", 'won', "won't", 'wouldn', "wouldn't"]

# In[15]:

def matching_lists(sentence_list):

    """

    Function which matches a list of words from the last paragraphs of the 

    Students Essay Corpus Texts with concepts from ConceptNet

    Parameters

    ----------

    sentence_list : list

        List containing the sentences from the last paragraphs in the 

        Students Essay Corpus

    Returns

    -------

    matches : list

        list containing words matching with ConceptNet concepts

        Elements themselves are lists, these lists contain ConceptNet matches

    """

    matches= []

    temp_matches = []

    for sentence in sentence_list:

        for word in sentence: 

            #use dictionary

            if word in onto_list and word not in stop_word_list:

                temp_matches.append(word)

        matches.append(temp_matches)

        temp_matches = []

    new_file = r'C:\Users\PC\Desktop\Coli neu\SWP\sentences2.txt'

    with open(new_file, 'w') as file:

        count = 0

        for sentence in matches:

            file.write(str(count) + "\t"+ str(sentence) + "\n")

            count += 1

    return matches

# In[23]:

def matching_dicts(sentence_list):

    """

    Function which matches a list of words from the last paragraphs of the 

    Students Essay Corpus Texts with concepts from ConceptNet

    Parameters

    ----------

    sentence_list : list

        List containing the sentences from the last paragraphs in the 

        Students Essay Corpus

    Returns

    -------

    matches : list

        list containing words matching with ConceptNet concepts

        Elements themselves are lists, these lists contain ConceptNet matches

    """

    matches= []

    temp_matches = []

    for sentence in sentence_list:

        for word in sentence: 

            #use dictionary

            if word in s_dict.keys() and word not in stop_word_list:

                temp_matches.append(word)

        matches.append(temp_matches)

        temp_matches = []

#     new_file = r'C:\Users\PC\Desktop\Coli neu\SWP\sentences2.txt'

#     with open(new_file, 'w') as file:

#         count = 0

#         for sentence in matches:

#             file.write(str(count) + "\t"+ str(sentence) + "\n")

#             count += 1

    return matches

concepts = matching_dicts(word_list)

print(concepts[:5])

# In[24]:

x = onto_list 

y = 0

s_dict = dict.fromkeys(x,y)

#print(s_dict)

# In[20]:

liste =  ['He invited other friends with trucks to go with him.', 'He invited other friends with trucks to go with him.', 'They went outside the city areas.', 'They went outside the city areas.', 'John had a lot of fun with his new truck.', 'The little girl went hiking in the mountains with her dad.', 'The little girl went hiking in the mountains with her dad.', 'She was walking ahead of him when she heard a buzzing.', 'She was walking ahead of him when she heard a buzzing.', 'On the trail in front of her was a rattle snake.', 'On the trail in front of her was a rattle snake.', 'Her father told her to run but her legs would not move.', 'Her father told her to run but her legs would not move.', 'Her dad came and carried her to safety.', 'Her dad came and carried her to safety.', 'Kurt had to attend a late evening meeting at work.', 'After dinner, he put on his suit and tie again.', 'He drove to work in the dark.', 'Kurt sat quietly in the meeting.', 'Kurt hoped the meeting would end soon so that he could go home to bed.', 'Keith was deciding whether he wanted a new car or not.', 'Then his old car broke down on the highway.', 'Keith was fed up with the situation.', 'He went straight to the dealership in a tow truck.', 'He went and bought the first car he saw.', 'Brent wanted a girlfriend.', 'So he asked out a lot of girls.', 'One finally said yes.', 'We told our son we were going to visit relatives.', 'We told our son we were going to visit relatives.', 'He was ok with it, and completely believed us.', 'We packed up for the long trip.', 'We drove for several hours.', 'His eyes lit up with joy when he saw his best friend instead.', 'His eyes lit up with joy when he saw his best friend instead.', "Kim's parents were having a party.", 'The adults were playing music and dancing.', "But they weren't very good dancers.", 'They were jerking, and sliding all around the room.', 'She laughed at the adults horrible dancing skills.', 'Mary was getting her kids dressed for a family photo.', 'While she tended to the girl, the boy went in the kitchen.', 'While she tended to the girl, the boy went in the kitchen.', 'He got into a big bag of flour which got all over him.', 'Mary did not have time to wash his clothes.', 'They took their family photo, although the boy was mismatched.', 'They took their family photo, although the boy was mismatched.', 'Katrina always wanted children.', 'Her and her husband tried for several years but had no luck.', 'They went to the doctor for help.', 'They went to the doctor for help.', 'The doctor gave Katrina several treatments.', 'The doctor gave Katrina several treatments.', 'Now, Katrina has a baby girl.', 'Now, Katrina has a baby girl.', 'Bill wrote a book.', 'He sent the book to a book publisher.', 'He sent the book to a book publisher.', 'The publisher read his book and liked it.', 'The publisher read his book and liked it.', "He offered to publish Bill's book.", "He offered to publish Bill's book.", 'Bill became a famous author.', 'John and his wife had tickets for an upcoming play.', 'John and his wife had tickets for an upcoming play.', 'John was excited, but his wife was skeptical.', 'John was excited, but his wife was skeptical.', 'My wife and I moved to our new house two summers ago.', 'My wife and I moved to our new house two summers ago.', 'When we arrived the sky let loose and started to rain.', 'When we arrived the sky let loose and started to rain.', 'We finished unpacking boxes while it continued to pour.', 'We finished unpacking boxes while it continued to pour.', 'The next day we woke to a dull roar.', 'The next day we woke to a dull roar.', 'When I looked outside there were loudly croaking frogs everywhere!', 'When I looked outside there were loudly croaking frogs everywhere!', "Martin didn't like doing his homework.", 'One day he decided to pay Will to do his homework for him.', 'One day he decided to pay Will to do his homework for him.', 'Martin enjoyed not having to do his homework.', "It didn't take long for Martin's bad decision to catch up with him.", 'When test time came Martin had no know of the material and failed.', "Allie's boyfriend invited her to play some golf.", 'Allie agreed but was hesitant.', 'When they got there she was so bored.', 'She did not want to play anymore.', 'So she excused herself to the bathroom and never came back.', 'I was working on a costume.', 'I used a hot glue gun to attach a piece of fabric to the costume.', 'I accidentally squirted glue onto my hand.', 'It hurt, but did not burn me too badly.', 'I learned to be more careful with the glue.', 'Cal slipped his socked feet into his shoes.', 'He felt a rather sharp pain suddenly.', 'He took off the shoes and a spider crawled out from one!', 'Cal drove to the hospital in his socks to get treated for the bite.', "My daughter's friend wanted to be in a fine arts class.", "My daughter's friend wanted to be in a fine arts class.", "My daughter's friend wanted to be in a fine arts class.", "She didn't have enough money to register.", "She didn't have enough money to register.", 'She was offered a babysitting job for $40 a week.', 'She was offered a babysitting job for $40 a week.', "She accepted the job with her parent's permission.", "She accepted the job with her parent's permission.", "She accepted the job with her parent's permission.", "She accepted the job with her parent's permission.", 'Now she is able to take the fine arts class!', 'Now she is able to take the fine arts class!', 'Carl wanted a motorcycle.', 'So he saved up money and bought one.', 'But his friend warned him that motorcycles were dangerous to drive.', 'But his friend warned him that motorcycles were dangerous to drive.', "Carl didn't listen and broke the speed limit on a freeway.", 'But a near miss with a car made him rethink driving a motorcycle.', 'My school was hosting a small tournament one day!', 'They were inviting a small amount of people to compete.', 'On the day of the tournament the shout-casters were sick!', 'The organizer was scrambling to find people so I offered to do it.', 'The organizer was scrambling to find people so I offered to do it.', 'He was very happy and I went on to cast the tournament!', 'He was very happy and I went on to cast the tournament!', 'Ursula had a twin sister.', 'Ursula had a twin sister.', "Until Ursula fell in love and didn't want to play pranks anymore.", 'But Ursula cut her hair very short and her boyfriend recognized her.', 'But Ursula cut her hair very short and her boyfriend recognized her.', 'Bob and his friends always played golf.', 'They would meet up every Saturday.', 'They liked one course in particular.', "But one round Bob's drive hit the barn.", "The starter said he hadn't seen someone hit that in Years.", "Lucy's grandpa wouldn't take her to blockbuster.", "Lucy's grandpa wouldn't take her to blockbuster.", 'Her grandma rarely drove anywhere.', 'She expected her grandma to decline, but she asked anyway.', 'She expected her grandma to decline, but she asked anyway.', 'Her grandma was silent and gave her a meaningful look.', 'Her grandma was silent and gave her a meaningful look.', 'Lucy was surprised, but pleased when she agreed to drive her.', 'Sarah is swimming.', 'Sarah gets attacked by a shark.', 'Sarah gets attacked by a shark.', 'Sarah fights off the shark.', 'Sarah fights off the shark.', 'Sarah escapes the attack.', 'Sarah lost her eye battling the shark.', 'Sarah lost her eye battling the shark.', 'Anna hate cleaning, but her closet was terribly messy.', "She decided she'd have to bite the bullet and clean it.", 'She took everything out.', 'Then she organized her clothes and put them back in.', "Afterwards, Anna was glad she'd finally cleaned it.", 'All her life Susan wanted to be a teacher.', 'She had just graduated college.', 'Susan had gotten a job at the local high school.', 'She began teaching.', 'Susan loved her job teaching and did very well at it.', 'Larry told his wife he had to work late.', 'Larry was not working late.', 'Larry was out with another woman.', 'Larry was out with another woman.', "Larry's wife was out and seen the two together.", "Larry's wife was out and seen the two together.", 'Larry was caught cheating.', 'Kevin went to the beach with his wife one day.', 'Kevin went to the beach with his wife one day.', 'They went into the water and began to swim around.', 'They went into the water and began to swim around.', 'Suddenly, Kevin felt a sharp pain in his foot.', 'He looked down and saw he was badly stabbed by a pufferfish.', 'He looked down and saw he was badly stabbed by a pufferfish.', "They swam to shore, and Kevin's girlfriend called 911 for help.", "They swam to shore, and Kevin's girlfriend called 911 for help.", "They swam to shore, and Kevin's girlfriend called 911 for help.", 'Winston bought a brand new car today.', 'He drove it all around town to show it off to his friends.', 'When he was finished he went home.', 'As he opened the door he dented it on the curb out front.', 'Sadly, he now has to call his insurance agent to get it fixed.', 'Sadly, he now has to call his insurance agent to get it fixed.', 'Marcus was collecting shells on the beach.', 'He picked up a large beautiful shell.', 'He put it in his pocket to save for later.', 'Suddenly he felt a sharp pinch.', 'A crab was inside the shell pinching his leg.', 'A crab was inside the shell pinching his leg.']

# In[31]:

import spacy

nlp = spacy.load('en_core_web_sm')

doc_list = []

temp =[]

for sentence in sentence_list:

    doc = nlp(sentence)

    for token in doc:

        temp.append(token.lemma_)

    doc_list.append(temp)

    temp =[]

    doc= ""

print(doc_list)

# In[30]:

new_file2 = r'C:\Users\PC\Desktop\Coli neu\SWP\test.txt'

with open(new_file2, 'w') as file:

    for i in range(len(sentence_list)):

        file.write(str(i) + "\t" + "None" + "\t" + str(sentence_list[i]) + "\t" + str(concepts[i]) + "\n")

# In[ ]: