Aktualisieren scripts/ActorOnly+CEL/main_CrossEntropy.py,...

import torch

import stanza

from gensim.models import KeyedVectors

from datasets import load_dataset

from structures import *

# loads skipgram gensim

file_name = "data/1-billion-word-language-modeling-benchmark-r13output.word2vec.vec"

model_gensim = KeyedVectors.load_word2vec_format(file_name)

# initialize tokenizer, => sentences splitting and tokenizing, !pip install stanza 

nlp = stanza.Pipeline(lang='en', processors='tokenize')

# loads dataset cnn_dailymail, !pip install datasets

dataset = load_dataset('ccdv/cnn_dailymail', '3.0.0', split='train[:10%]') # extract subset for testing

x = PreprocessedDataSet('/workspace/students/kreuzer/train0', dataset, model_gensim, nlp)

#!/bin/bash

#SBATCH --job-name=pre0

#SBATCH --output=pre0.txt

#SBATCH --mail-user=kreuzer@cl.uni-heidelberg.de

#SBATCH --mail-type=ALL

#SBATCH --partition=compute

#SBATCH --time=2-23:59:59

#SBATCH --ntasks=1

#SBATCH --mem=128000

srun python3 main_preprocess0.py