From 2ba704a6fced5722a1f1e3da464185e3724a57d2 Mon Sep 17 00:00:00 2001 From: Yigit Sever Date: Tue, 17 Sep 2019 21:36:20 +0300 Subject: Hide away the scripts --- get_data.sh | 16 +++--- prep_lookup.py | 61 ---------------------- scripts/prep_lookup.py | 61 ++++++++++++++++++++++ scripts/tab_creator.pl | 76 ++++++++++++++++++++++++++++ scripts/train_dic_creator.pl | 118 +++++++++++++++++++++++++++++++++++++++++++ tab_creator.pl | 76 ---------------------------- train_dic_creator.pl | 118 ------------------------------------------- 7 files changed, 263 insertions(+), 263 deletions(-) delete mode 100644 prep_lookup.py create mode 100644 scripts/prep_lookup.py create mode 100755 scripts/tab_creator.pl create mode 100755 scripts/train_dic_creator.pl delete mode 100755 tab_creator.pl delete mode 100755 train_dic_creator.pl diff --git a/get_data.sh b/get_data.sh index f943722..068c776 100755 --- a/get_data.sh +++ b/get_data.sh @@ -23,6 +23,7 @@ # ROOT="$(pwd)" +SCRIPTS="${ROOT}/scripts" WNET="${ROOT}/wordnets" mkdir -p "${WNET}" @@ -45,24 +46,23 @@ done rm -rf "${WNET}/ita/" # comes alongside iwn, not useful for us mv "${WNET}/iwn" "${WNET}/ita" - echo "Creating .def files" TAB_DIR="${WNET}/tab_files" mkdir -p "${TAB_DIR}" -for filename in ${WNET}/*/wn-data*.tab; do - ${ROOT}/tab_creator.pl $filename +for filename in "${WNET}"/*/wn-data*.tab; do + "${SCRIPTS}/tab_creator.pl" "${filename}" "${TAB_DIR}" done for PAIR in en,bg en,el en,it, en,ro, en,sl en,sq, bg,el bg,it bg,ro el,it el,ro el,sq it,ro ro,sl ro,sq; do IFS=',' read -r source_lang target_lang <<< "${PAIR}" - python ${ROOT}/prep_lookup.py -s "${source_lang}" -t "${target_lang}" + python "${SCRIPTS}/prep_lookup.py" -s "${source_lang}" -t "${target_lang}" done READY="${WNET}/ready" mkdir -p "${READY}" -mv ${ROOT}/*.def "${READY}" +mv "${ROOT}"/*.def "${READY}" echo "Downloading dictionaries" @@ -84,8 +84,8 @@ wget -nc -q https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/dic/it-ro.dic.g wget -nc -q https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/dic/ro-sl.dic.gz -P "${DICT}" # Romanian - Albanian wget -nc -q https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/dic/ro-sq.dic.gz -P "${DICT}" # Romanian - Albanian -for FILE in ${DICT}/*; do - gunzip -q "${FILE}" +for file in "${DICT}"/*; do + gunzip -q "${file}" done export LC_CTYPE=en_US.UTF-8 @@ -95,7 +95,7 @@ echo "Creating dictionaries" for PAIR in en,bg en,el en,it, en,ro, en,sl en,sq, bg,el bg,it bg,ro el,it el,ro el,sq it,ro ro,sl ro,sq; do IFS=',' read -r source_lang target_lang <<< "${PAIR}" - perl "${ROOT}/train_dic_creator.pl" "${source_lang}" "${target_lang}" + perl "${SCRIPTS}/train_dic_creator.pl" "${source_lang}" "${target_lang}" "${DICT}" done TRAIN_DIR="${DICT}/train" diff --git a/prep_lookup.py b/prep_lookup.py deleted file mode 100644 index 7fdfeec..0000000 --- a/prep_lookup.py +++ /dev/null @@ -1,61 +0,0 @@ -import argparse -from pathlib import Path -import collections -import os - -def en_and_other(other, dirname): - from nltk.corpus import wordnet as wn - other_file = os.path.join(dirname, other + "." + 'tab') - lookup = collections.defaultdict(dict) - - with open(other_file, 'r') as f: - for line in f: - (pos, offset, rest) = line.split(' ', 2) - offset = int(offset) - # part of speech + offset is unique, so keys are combination of both - en_def = wn.synset_from_pos_and_offset(pos, offset).definition() - lookup[(pos, offset)]['en'] = en_def - lookup[(pos,offset)][other] = rest.rstrip() - return lookup - -def both_lookup(source, target, dirname): - from_file = os.path.join(dirname, source + "." + 'tab') - to_file = os.path.join(dirname, target + "." + 'tab') - lookup = collections.defaultdict(dict) - - for tab_file, lang_code in zip((from_file, to_file), (source, target)): - with open(tab_file, 'r') as f: - for line in f: - (pos, offset, rest) = line.split(' ', 2) - offset = int(offset) - # part of speech + offset is unique, so keys are combination of both - lookup[(pos,offset)][lang_code] = rest.rstrip() - return lookup - -def main(args): - - dirname = args.tab_directory - source_lang = args.source_lang - target_lang = args.target_lang - - if (source_lang == 'en'): - lookup = en_and_other(target_lang, dirname) - elif (target_lang == 'en'): - lookup = en_and_other(source_lang, dirname) - else: - lookup = both_lookup(source_lang, target_lang, dirname) - - with open(f'{source_lang}_to_{target_lang}.def', 'w') as sf, open(f'{target_lang}_to_{source_lang}.def', 'w') as tf: - for (pos, offset), overlap in lookup.items(): - if source_lang in overlap and target_lang in overlap: - print(overlap[source_lang], file=sf) - print(overlap[target_lang], file=tf) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Create a pair of .def files for 2 given languages') - parser.add_argument('--tab_directory', help='directory of the .tab files', default='wordnets/tab_files') - parser.add_argument('-s', '--source_lang', help='source language 2 letter code') - parser.add_argument('-t', '--target_lang', help='target language 2 letter code') - args = parser.parse_args() - - main(args) diff --git a/scripts/prep_lookup.py b/scripts/prep_lookup.py new file mode 100644 index 0000000..7fdfeec --- /dev/null +++ b/scripts/prep_lookup.py @@ -0,0 +1,61 @@ +import argparse +from pathlib import Path +import collections +import os + +def en_and_other(other, dirname): + from nltk.corpus import wordnet as wn + other_file = os.path.join(dirname, other + "." + 'tab') + lookup = collections.defaultdict(dict) + + with open(other_file, 'r') as f: + for line in f: + (pos, offset, rest) = line.split(' ', 2) + offset = int(offset) + # part of speech + offset is unique, so keys are combination of both + en_def = wn.synset_from_pos_and_offset(pos, offset).definition() + lookup[(pos, offset)]['en'] = en_def + lookup[(pos,offset)][other] = rest.rstrip() + return lookup + +def both_lookup(source, target, dirname): + from_file = os.path.join(dirname, source + "." + 'tab') + to_file = os.path.join(dirname, target + "." + 'tab') + lookup = collections.defaultdict(dict) + + for tab_file, lang_code in zip((from_file, to_file), (source, target)): + with open(tab_file, 'r') as f: + for line in f: + (pos, offset, rest) = line.split(' ', 2) + offset = int(offset) + # part of speech + offset is unique, so keys are combination of both + lookup[(pos,offset)][lang_code] = rest.rstrip() + return lookup + +def main(args): + + dirname = args.tab_directory + source_lang = args.source_lang + target_lang = args.target_lang + + if (source_lang == 'en'): + lookup = en_and_other(target_lang, dirname) + elif (target_lang == 'en'): + lookup = en_and_other(source_lang, dirname) + else: + lookup = both_lookup(source_lang, target_lang, dirname) + + with open(f'{source_lang}_to_{target_lang}.def', 'w') as sf, open(f'{target_lang}_to_{source_lang}.def', 'w') as tf: + for (pos, offset), overlap in lookup.items(): + if source_lang in overlap and target_lang in overlap: + print(overlap[source_lang], file=sf) + print(overlap[target_lang], file=tf) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Create a pair of .def files for 2 given languages') + parser.add_argument('--tab_directory', help='directory of the .tab files', default='wordnets/tab_files') + parser.add_argument('-s', '--source_lang', help='source language 2 letter code') + parser.add_argument('-t', '--target_lang', help='target language 2 letter code') + args = parser.parse_args() + + main(args) diff --git a/scripts/tab_creator.pl b/scripts/tab_creator.pl new file mode 100755 index 0000000..6efce46 --- /dev/null +++ b/scripts/tab_creator.pl @@ -0,0 +1,76 @@ +#!/usr/bin/env perl +# +# +# Copyright © 2019 Yiğit Sever +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +use strict; +use warnings; +use File::Basename; + +my %language_codes = ( + als => "sq", + bul => "bg", + ell => "el", + ita => "it", + ron => "ro", + slv => "sl", +); + +my ($tab_file, $tab_dir) = @ARGV; + +if (not defined $tab_file or not defined $tab_file) { + die "usage: ./tab_creator.pl "; +} + +if (not -e $tab_file) { + die "'$tab_file' does not exist"; +} + +if (not defined $tab_dir && $tab_dir ne '') { + $tab_dir = './wordnets/tab_files'; +} + +open (my $fh, '<', $tab_file) or die "Could not open '$tab_file' $!"; + +my $filename = basename($tab_file); + +my $lang_code; +if ($filename =~ m/wn-data-(\w{3})\.tab/) { + $lang_code = $1; +} + + +my $short_lang_code = $language_codes{$lang_code}; + +my $outfilename = $tab_dir . '/' . $short_lang_code . '.tab'; +open (my $out_fh, '>', $outfilename) or die "Could not open '$outfilename', $!"; + +while (my $row = <$fh>) { + chomp $row; + if ($row =~ m/$lang_code:def/) { + if ($row =~ m/^(\d+)-(\w)\s+$lang_code:def\s*\d\s+(.*)$/) { + my $offset = $1; + my $pos = $2; + my $def = $3; + print $out_fh "$pos $offset $def\n"; + } + } +} diff --git a/scripts/train_dic_creator.pl b/scripts/train_dic_creator.pl new file mode 100755 index 0000000..448fecf --- /dev/null +++ b/scripts/train_dic_creator.pl @@ -0,0 +1,118 @@ +#!/usr/bin/env perl +# +# +#Copyright © 2019 Yiğit Sever +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +# Get source language code and target language code +# optionally give cutoff, cutoff/2 pairs will be prepared for train/test +# optionally give a different dictionary directory name +# +# USAGE: +# $ perl train_dic_creator.pl (cutoff) (dictionary_dir) + +use strict; +use warnings; +use List::Util qw(shuffle); + +my ($source_lang, $target_lang, $dict_dir, $cutoff) = @ARGV; + +if (not defined $source_lang or not defined $target_lang) { + die "usage: ./train_dic_creator.pl (cutoff)"; +} + +if (not defined $cutoff && $cutoff ne '') { + $cutoff = 20000; +} + +if (not defined $dict_dir && $dict_dir ne '') { + $dict_dir = './dictionaries/'; +} + +my $flipped = 0; +my $file_name; + +if (-e "$dict_dir/$target_lang-$source_lang.dic") { + warn "Dictionary is formatted as $target_lang $source_lang, still creating $source_lang $target_lang"; + $file_name = "$target_lang-$source_lang.dic"; + $flipped = 1; +} elsif (-e "$dict_dir/$source_lang-$target_lang.dic") { + $file_name = "$source_lang-$target_lang.dic"; +} + +my $file_path = $dict_dir . $file_name; + +local @ARGV = $file_path; +local $^I = '.bak'; + +while (<>) { # remove empty lines + print if ! /^$/; +} + +my @lines = `sort -rn $file_path`; # better translations swim to top + +my @result; +my $c = 0; + +foreach my $line (@lines) { + chomp($line); + if ($line !~ m/^\d+\s+[0-9.]+\s+(\S+)\s+(\S+)\s+[0-9.]+\s+[0-9.]+$/) { + # line has multiple tokens + next; + } else { + my ($source, $target) = $line =~ m/^\d+\s+[0-9.]+\s+(\S+)\s+(\S+)\s+[0-9.]+\s+[0-9.]+$/; + + if ($flipped) { # The file name and given parameters mismatch, correcting + push @result, "$target $source"; + } else { + push @result, "$source $target"; + } + $c++; + + if ($c >= $cutoff) { + last; + } + } +} + +my $test = scalar @result; + +if ($cutoff > scalar @result) { + $cutoff = scalar @result; +} + +@result = shuffle @result; + +my $size = $cutoff / 2; + +my @head = @result[0..$size - 1]; +my @tail = @result[-$size..-1]; + +my $train_file_name = $source_lang . '_' . $target_lang . '.train'; +my $test_file_name = $source_lang . '_' . $target_lang . '.test'; + +open my $train_fh, '>', $dict_dir . $train_file_name; +open my $test_fh, '>', $dict_dir . $test_file_name; + +print $train_fh join("\n", @head); +print $test_fh join("\n", @tail); + +unlink "$file_path$^I"; diff --git a/tab_creator.pl b/tab_creator.pl deleted file mode 100755 index 6efce46..0000000 --- a/tab_creator.pl +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env perl -# -# -# Copyright © 2019 Yiğit Sever -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE -# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -use strict; -use warnings; -use File::Basename; - -my %language_codes = ( - als => "sq", - bul => "bg", - ell => "el", - ita => "it", - ron => "ro", - slv => "sl", -); - -my ($tab_file, $tab_dir) = @ARGV; - -if (not defined $tab_file or not defined $tab_file) { - die "usage: ./tab_creator.pl "; -} - -if (not -e $tab_file) { - die "'$tab_file' does not exist"; -} - -if (not defined $tab_dir && $tab_dir ne '') { - $tab_dir = './wordnets/tab_files'; -} - -open (my $fh, '<', $tab_file) or die "Could not open '$tab_file' $!"; - -my $filename = basename($tab_file); - -my $lang_code; -if ($filename =~ m/wn-data-(\w{3})\.tab/) { - $lang_code = $1; -} - - -my $short_lang_code = $language_codes{$lang_code}; - -my $outfilename = $tab_dir . '/' . $short_lang_code . '.tab'; -open (my $out_fh, '>', $outfilename) or die "Could not open '$outfilename', $!"; - -while (my $row = <$fh>) { - chomp $row; - if ($row =~ m/$lang_code:def/) { - if ($row =~ m/^(\d+)-(\w)\s+$lang_code:def\s*\d\s+(.*)$/) { - my $offset = $1; - my $pos = $2; - my $def = $3; - print $out_fh "$pos $offset $def\n"; - } - } -} diff --git a/train_dic_creator.pl b/train_dic_creator.pl deleted file mode 100755 index a228044..0000000 --- a/train_dic_creator.pl +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env perl -# -# -#Copyright © 2019 Yiğit Sever -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE -# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -# Get source language code and target language code -# optionally give cutoff, cutoff/2 pairs will be prepared for train/test -# optionally give a different dictionary directory name -# -# USAGE: -# $ perl train_dic_creator.pl (cutoff) (dictionary_dir) - -use strict; -use warnings; -use List::Util qw(shuffle); - -my ($source_lang, $target_lang, $cutoff, $dict_dir) = @ARGV; - -if (not defined $source_lang or not defined $target_lang) { - die "usage: ./train_dic_creator.pl (cutoff)"; -} - -if (not defined $cutoff && $cutoff ne '') { - $cutoff = 20000; -} - -if (not defined $dict_dir && $dict_dir ne '') { - $dict_dir = './dictionaries/'; -} - -my $flipped = 0; -my $file_name; - -if (-e "$dict_dir/$target_lang-$source_lang.dic") { - warn "Dictionary is formatted as $target_lang $source_lang, still creating $source_lang $target_lang"; - $file_name = "$target_lang-$source_lang.dic"; - $flipped = 1; -} elsif (-e "$dict_dir/$source_lang-$target_lang.dic") { - $file_name = "$source_lang-$target_lang.dic"; -} - -my $file_path = $dict_dir . $file_name; - -local @ARGV = $file_path; -local $^I = '.bak'; - -while (<>) { # remove empty lines - print if ! /^$/; -} - -my @lines = `sort -rn $file_path`; # better translations swim to top - -my @result; -my $c = 0; - -foreach my $line (@lines) { - chomp($line); - if ($line !~ m/^\d+\s+[0-9.]+\s+(\S+)\s+(\S+)\s+[0-9.]+\s+[0-9.]+$/) { - # line has multiple tokens - next; - } else { - my ($source, $target) = $line =~ m/^\d+\s+[0-9.]+\s+(\S+)\s+(\S+)\s+[0-9.]+\s+[0-9.]+$/; - - if ($flipped) { # The file name and given parameters mismatch, correcting - push @result, "$target $source"; - } else { - push @result, "$source $target"; - } - $c++; - - if ($c >= $cutoff) { - last; - } - } -} - -my $test = scalar @result; - -if ($cutoff > scalar @result) { - $cutoff = scalar @result; -} - -@result = shuffle @result; - -my $size = $cutoff / 2; - -my @head = @result[0..$size - 1]; -my @tail = @result[-$size..-1]; - -my $train_file_name = $source_lang . '_' . $target_lang . '.train'; -my $test_file_name = $source_lang . '_' . $target_lang . '.test'; - -open my $train_fh, '>', $dict_dir . $train_file_name; -open my $test_fh, '>', $dict_dir . $test_file_name; - -print $train_fh join("\n", @head); -print $test_fh join("\n", @tail); - -unlink "$file_path$^I"; -- cgit v1.2.3-70-g09d2