aboutsummaryrefslogtreecommitdiffstats
path: root/get_data.sh
diff options
context:
space:
mode:
authorYigit Sever2019-09-17 21:36:20 +0300
committerYigit Sever2019-09-17 21:42:03 +0300
commit2ba704a6fced5722a1f1e3da464185e3724a57d2 (patch)
tree6e240fb60e7e97cad5252a18a3454897347a1586 /get_data.sh
parent2c763b88c5c376c8dc39f1bef058449a2865d1c6 (diff)
downloadEvaluating-Dictionary-Alignment-2ba704a6fced5722a1f1e3da464185e3724a57d2.tar.gz
Evaluating-Dictionary-Alignment-2ba704a6fced5722a1f1e3da464185e3724a57d2.tar.bz2
Evaluating-Dictionary-Alignment-2ba704a6fced5722a1f1e3da464185e3724a57d2.zip
Hide away the scripts
Diffstat (limited to 'get_data.sh')
-rwxr-xr-xget_data.sh16
1 files changed, 8 insertions, 8 deletions
diff --git a/get_data.sh b/get_data.sh
index f943722..068c776 100755
--- a/get_data.sh
+++ b/get_data.sh
@@ -23,6 +23,7 @@
23# 23#
24 24
25ROOT="$(pwd)" 25ROOT="$(pwd)"
26SCRIPTS="${ROOT}/scripts"
26WNET="${ROOT}/wordnets" 27WNET="${ROOT}/wordnets"
27mkdir -p "${WNET}" 28mkdir -p "${WNET}"
28 29
@@ -45,24 +46,23 @@ done
45rm -rf "${WNET}/ita/" # comes alongside iwn, not useful for us 46rm -rf "${WNET}/ita/" # comes alongside iwn, not useful for us
46mv "${WNET}/iwn" "${WNET}/ita" 47mv "${WNET}/iwn" "${WNET}/ita"
47 48
48
49echo "Creating .def files" 49echo "Creating .def files"
50 50
51TAB_DIR="${WNET}/tab_files" 51TAB_DIR="${WNET}/tab_files"
52mkdir -p "${TAB_DIR}" 52mkdir -p "${TAB_DIR}"
53 53
54for filename in ${WNET}/*/wn-data*.tab; do 54for filename in "${WNET}"/*/wn-data*.tab; do
55 ${ROOT}/tab_creator.pl $filename 55 "${SCRIPTS}/tab_creator.pl" "${filename}" "${TAB_DIR}"
56done 56done
57 57
58for PAIR in en,bg en,el en,it, en,ro, en,sl en,sq, bg,el bg,it bg,ro el,it el,ro el,sq it,ro ro,sl ro,sq; do 58for PAIR in en,bg en,el en,it, en,ro, en,sl en,sq, bg,el bg,it bg,ro el,it el,ro el,sq it,ro ro,sl ro,sq; do
59 IFS=',' read -r source_lang target_lang <<< "${PAIR}" 59 IFS=',' read -r source_lang target_lang <<< "${PAIR}"
60 python ${ROOT}/prep_lookup.py -s "${source_lang}" -t "${target_lang}" 60 python "${SCRIPTS}/prep_lookup.py" -s "${source_lang}" -t "${target_lang}"
61done 61done
62 62
63READY="${WNET}/ready" 63READY="${WNET}/ready"
64mkdir -p "${READY}" 64mkdir -p "${READY}"
65mv ${ROOT}/*.def "${READY}" 65mv "${ROOT}"/*.def "${READY}"
66 66
67echo "Downloading dictionaries" 67echo "Downloading dictionaries"
68 68
@@ -84,8 +84,8 @@ wget -nc -q https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/dic/it-ro.dic.g
84wget -nc -q https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/dic/ro-sl.dic.gz -P "${DICT}" # Romanian - Albanian 84wget -nc -q https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/dic/ro-sl.dic.gz -P "${DICT}" # Romanian - Albanian
85wget -nc -q https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/dic/ro-sq.dic.gz -P "${DICT}" # Romanian - Albanian 85wget -nc -q https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/dic/ro-sq.dic.gz -P "${DICT}" # Romanian - Albanian
86 86
87for FILE in ${DICT}/*; do 87for file in "${DICT}"/*; do
88 gunzip -q "${FILE}" 88 gunzip -q "${file}"
89done 89done
90 90
91export LC_CTYPE=en_US.UTF-8 91export LC_CTYPE=en_US.UTF-8
@@ -95,7 +95,7 @@ echo "Creating dictionaries"
95 95
96for PAIR in en,bg en,el en,it, en,ro, en,sl en,sq, bg,el bg,it bg,ro el,it el,ro el,sq it,ro ro,sl ro,sq; do 96for PAIR in en,bg en,el en,it, en,ro, en,sl en,sq, bg,el bg,it bg,ro el,it el,ro el,sq it,ro ro,sl ro,sq; do
97 IFS=',' read -r source_lang target_lang <<< "${PAIR}" 97 IFS=',' read -r source_lang target_lang <<< "${PAIR}"
98 perl "${ROOT}/train_dic_creator.pl" "${source_lang}" "${target_lang}" 98 perl "${SCRIPTS}/train_dic_creator.pl" "${source_lang}" "${target_lang}" "${DICT}"
99done 99done
100 100
101TRAIN_DIR="${DICT}/train" 101TRAIN_DIR="${DICT}/train"