Clean up sentence embeddings

3 files changed, 179 insertions, 304 deletions

diff --git a/sentence_emb_matching.py b/sentence_emb_matching.py
deleted file mode 100644
index 38812d7..0000000
--- a/sentence_emb_matching.py
+++ /dev/null
@@ -1,153 +0,0 @@
-import argparse
-
-parser = argparse.ArgumentParser(description='run matching using sentence embeddings and cosine similarity')
-parser.add_argument('source_lang', help='source language short name')
-parser.add_argument('target_lang', help='target language short name')
-parser.add_argument('source_vector', help='path of the source vector')
-parser.add_argument('target_vector', help='path of the target vector')
-parser.add_argument('source_defs', help='path of the source definitions')
-parser.add_argument('target_defs', help='path of the target definitions')
-parser.add_argument('-n', '--instances', help='number of instances in each language to retrieve', default=2000, type=int)
-
-args = parser.parse_args()
-
-source_lang = args.source_lang
-target_lang = args.target_lang
-
-def load_embeddings(path, dimension = 300):
-    """
-    Loads the embeddings from a word2vec formatted file.
-    The first line may or may not include the word count and dimension
-    """
-    vectors = {}
-    with open(path, mode='r', encoding='utf8') as fp:
-        first_line = fp.readline().rstrip('\n')
-        if first_line.count(' ') == 1: # includes the "word_count dimension" information
-            (word_count, dimension) = map(int, first_line.split())
-        else: # assume the file only contains vectors
-            fp.seek(0)
-        for line in fp:
-            elems = line.split()
-            vectors[" ".join(elems[:-dimension])] = " ".join(elems[-dimension:])
-    return vectors
-
-source_vectors_filename = args.source_vector
-target_vectors_filename = args.target_vector
-vectors_source = load_embeddings(source_vectors_filename)
-vectors_target = load_embeddings(target_vectors_filename)
-
-source_defs_filename = args.source_defs
-target_defs_filename = args.target_defs
-defs_source = [line.rstrip('\n') for line in open(source_defs_filename, encoding='utf8')]
-defs_target = [line.rstrip('\n') for line in open(target_defs_filename, encoding='utf8')]
-
-import numpy as np
-from mosestokenizer import *
-
-def clean_corpus_using_embeddings_vocabulary(
-        embeddings_dictionary,
-        corpus,
-        vectors,
-        language,
-        ):
-    '''
-    Cleans corpus using the dictionary of embeddings.
-    Any word without an associated embedding in the dictionary is ignored.
-    '''
-    clean_corpus, clean_vectors, keys = [], {}, []
-    words_we_want = set(embeddings_dictionary)
-    tokenize = MosesTokenizer(language)
-    for key, doc in enumerate(corpus):
-        clean_doc = []
-        words = tokenize(doc)
-        for word in words:
-            if word in words_we_want:
-                clean_doc.append(word)
-                clean_vectors[word] = np.array(vectors[word].split()).astype(np.float)
-        if len(clean_doc) > 3 and len(clean_doc) < 25:
-            keys.append(key)
-        clean_corpus.append(' '.join(clean_doc))
-    tokenize.close()
-    return np.array(clean_corpus), clean_vectors, keys
-
-clean_src_corpus, clean_src_vectors, src_keys = clean_corpus_using_embeddings_vocabulary(
-        set(vectors_source.keys()),
-        defs_source,
-        vectors_source,
-        source_lang,
-        )
-
-clean_target_corpus, clean_target_vectors, target_keys = clean_corpus_using_embeddings_vocabulary(
-        set(vectors_target.keys()),
-        defs_target,
-        vectors_target,
-        target_lang,
-        )
-
-import random
-take = args.instances
-
-common_keys = set(src_keys).intersection(set(target_keys))
-take = min(len(common_keys), take) # you can't sample more than length
-experiment_keys = random.sample(common_keys, take)
-
-instances = len(experiment_keys)
-
-clean_src_corpus = list(clean_src_corpus[experiment_keys])
-clean_target_corpus = list(clean_target_corpus[experiment_keys])
-
-print(f'{source_lang} - {target_lang} : document sizes: {len(clean_src_corpus)}, {len(clean_target_corpus)}')
-
-del vectors_source, vectors_target, defs_source, defs_target
-
-from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
-
-vocab_counter = CountVectorizer().fit(clean_src_corpus + clean_target_corpus)
-common = [w for w in vocab_counter.get_feature_names() if w in clean_src_vectors or w in clean_target_vectors]
-W_common = []
-
-for w in common:
-    if w in clean_src_vectors:
-        W_common.append(np.array(clean_src_vectors[w]))
-    else:
-        W_common.append(np.array(clean_target_vectors[w]))
-
-print(f'{source_lang} - {target_lang}: the vocabulary size is {len(W_common)}')
-
-from sklearn.preprocessing import normalize
-W_common = np.array(W_common)
-W_common = normalize(W_common) # default is l2
-
-vect_tfidf = TfidfVectorizer(vocabulary=common, dtype=np.double, norm='l2')
-vect_tfidf.fit(clean_src_corpus + clean_target_corpus)
-X_idf_source = vect_tfidf.transform(clean_src_corpus)
-X_idf_target = vect_tfidf.transform(clean_target_corpus)
-
-print(f'Matrices are {X_idf_source.shape} and {W_common.shape}')
-print(f'The dimensions are {X_idf_source.ndim} and {W_common.ndim}')
-
-X_idf_source_array = X_idf_source.toarray()
-X_idf_target_array = X_idf_target.toarray()
-S_emb_source = np.matmul(X_idf_source_array, W_common)
-S_emb_target = np.matmul(X_idf_target_array, W_common)
-
-S_emb_target_transpose = np.transpose(S_emb_target)
-
-cost_matrix = np.matmul(S_emb_source, S_emb_target_transpose)
-
-from lapjv import lapjv
-cost_matrix = cost_matrix * -1000
-row_ind, col_ind, a = lapjv(cost_matrix, verbose=False)
-
-result = zip(row_ind, col_ind)
-hit_one = len([x for x,y in result if x == y])
-print(f'{hit_one} definitions have been mapped correctly, shape of cost matrix: {str(cost_matrix.shape)}')
-
-import csv
-percentage = hit_one / instances * 100
-fields = [f'{source_lang}', f'{target_lang}', f'{instances}', f'{hit_one}', f'{percentage}']
-
-with open('semb_matcing.csv', 'a') as f:
-    writer = csv.writer(f)
-    writer.writerow(fields)
-
diff --git a/sentence_emb_retrieval.py b/sentence_emb_retrieval.py
deleted file mode 100644
index 63ebcdc..0000000
--- a/sentence_emb_retrieval.py
+++ /dev/null
@@ -1,151 +0,0 @@
-import argparse
-
-parser = argparse.ArgumentParser(description='Run Retrieval using Sentence Embedding + Cosine')
-parser.add_argument('source_lang', help='source language short name')
-parser.add_argument('target_lang', help='target language short name')
-parser.add_argument('source_vector', help='path of the source vector')
-parser.add_argument('target_vector', help='path of the target vector')
-parser.add_argument('source_defs', help='path of the source definitions')
-parser.add_argument('target_defs', help='path of the target definitions')
-parser.add_argument('-n', '--instances', help='number of instances in each language to retrieve', default=1000, type=int)
-args = parser.parse_args()
-
-source_lang = args.source_lang
-target_lang = args.target_lang
-
-def load_embeddings(path, dimension = 300):
-    """
-    Loads the embeddings from a word2vec formatted file.
-    The first line may or may not include the word count and dimension
-    """
-    vectors = {}
-    with open(path, mode='r', encoding='utf8') as fp:
-        first_line = fp.readline().rstrip('\n')
-        if first_line.count(' ') == 1: # includes the "word_count dimension" information
-            (word_count, dimension) = map(int, first_line.split())
-        else: # assume the file only contains vectors
-            fp.seek(0)
-        for line in fp:
-            elems = line.split()
-            vectors[" ".join(elems[:-dimension])] = " ".join(elems[-dimension:])
-    return vectors
-
-lang_source = args.source_lang
-lang_target = args.target_lang
-
-vectors_filename_source = args.source_vector
-vectors_filename_target = args.target_vector
-
-vectors_source = load_embeddings(vectors_filename_source)
-vectors_target = load_embeddings(vectors_filename_target)
-
-defs_filename_source = args.source_defs
-defs_filename_target = args.target_defs
-defs_source = [line.rstrip('\n') for line in open(defs_filename_source, encoding='utf8')]
-defs_target = [line.rstrip('\n') for line in open(defs_filename_target, encoding='utf8')]
-
-print('Read {} {} documents and {} {} documents'.format(len(defs_source), lang_source, len(defs_target), lang_target))
-
-import numpy as np
-from mosestokenizer import *
-
-def clean_corpus_using_embeddings_vocabulary(
-        embeddings_dictionary,
-        corpus,
-        vectors,
-        language,
-        ):
-    '''
-    Cleans corpus using the dictionary of embeddings.
-    Any word without an associated embedding in the dictionary is ignored.
-    Adds '__target-language' and '__source-language' at the end of the words according to their language.
-    '''
-    clean_corpus, clean_vectors, keys = [], {}, []
-    words_we_want = set(embeddings_dictionary)
-    tokenize = MosesTokenizer(language)
-    for key, doc in enumerate(corpus):
-        clean_doc = []
-        words = tokenize(doc)
-        for word in words:
-            if word in words_we_want:
-                clean_doc.append(word + '__%s' % language)
-                clean_vectors[word + '__%s' % language] = np.array(vectors[word].split()).astype(np.float)
-        if len(clean_doc) > 3 and len(clean_doc) < 25:
-            keys.append(key)
-        clean_corpus.append(' '.join(clean_doc))
-    tokenize.close()
-    return np.array(clean_corpus), clean_vectors, keys
-
-clean_corpus_source, clean_vectors_source, keys_source = clean_corpus_using_embeddings_vocabulary(
-        set(vectors_source.keys()),
-        defs_source,
-        vectors_source,
-        lang_source,
-        )
-
-clean_corpus_target, clean_vectors_target, keys_target = clean_corpus_using_embeddings_vocabulary(
-        set(vectors_target.keys()),
-        defs_target,
-        vectors_target,
-        lang_target,
-        )
-
-import random
-take = args.instances
-
-common_keys = set(keys_source).intersection(set(keys_target)) # definitions that fit the above requirements
-take = min(len(common_keys), take) # you can't sample more than length
-experiment_keys = random.sample(common_keys, take)
-
-instances = len(experiment_keys)
-
-clean_corpus_source = list(clean_corpus_source[experiment_keys])
-clean_corpus_target = list(clean_corpus_target[experiment_keys])
-print(f'{source_lang} - {target_lang} : document sizes: {len(clean_corpus_source)}, {len(clean_corpus_target)}')
-
-del vectors_source, vectors_target, defs_source, defs_target
-
-from sklearn.feature_extraction.text import CountVectorizer
-from sklearn.feature_extraction.text import TfidfVectorizer
-
-vocab_counter = CountVectorizer().fit(clean_corpus_source + clean_corpus_target)
-common = [w for w in vocab_counter.get_feature_names() if w in clean_vectors_source or w in clean_vectors_target]
-
-W_common = []
-for w in common:
-    if w in clean_vectors_source:
-        W_common.append(np.array(clean_vectors_source[w]))
-    else:
-        W_common.append(np.array(clean_vectors_target[w]))
-
-print('The vocabulary size is %d' % (len(W_common)))
-
-from sklearn.preprocessing import normalize
-W_common = np.array(W_common)
-W_common = normalize(W_common) # default is l2
-
-vect_tfidf = TfidfVectorizer(vocabulary=common, dtype=np.double, norm='l2')
-vect_tfidf.fit(clean_corpus_source + clean_corpus_target)
-X_idf_source = vect_tfidf.transform(clean_corpus_source)
-X_idf_target = vect_tfidf.transform(clean_corpus_target)
-
-print(f'Matrices are {X_idf_source.shape} and {W_common.shape}')
-print(f'The dimensions are {X_idf_source.ndim} and {W_common.ndim}')
-
-X_idf_source_array = X_idf_source.toarray()
-X_idf_target_array = X_idf_target.toarray()
-S_emb_source = np.matmul(X_idf_source_array, W_common)
-S_emb_target = np.matmul(X_idf_target_array, W_common)
-
-S_emb_target_transpose = np.transpose(S_emb_target)
-
-cost_matrix = np.matmul(S_emb_source, S_emb_target_transpose)
-
-hit_at_one = len([x for x,y in enumerate(cost_matrix.argmax(axis=1)) if x == y])
-
-import csv
-percentage = hit_at_one / instances * 100
-fields = [f'{source_lang}', f'{target_lang}', f'{instances}', f'{hit_at_one}', f'{percentage}']
-with open('/home/syigit/multilang_results/sentence_emb_retrieval_axis_1.csv', 'a') as f:
-    writer = csv.writer(f)
-    writer.writerow(fields)
diff --git a/sentence_embedding.py b/sentence_embedding.py
new file mode 100644
index 0000000..0cd5361
--- /dev/null
+++ b/sentence_embedding.py
@@ -0,0 +1,179 @@
+import argparse
+import csv
+import random
+
+import numpy as np
+from lapjv import lapjv
+from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
+from sklearn.preprocessing import normalize
+
+from Wasserstein_Distance import load_embeddings, process_corpus
+
+
+def main(args):
+
+    run_method = list()
+
+    if input_paradigm == "all":
+        run_paradigm.extend("matching", "retrieval")
+    else:
+        run_paradigm.append(input_paradigm)
+
+    source_lang = args.source_lang
+    target_lang = args.target_lang
+    batch = args.batch
+
+    source_vectors_filename = args.source_vector
+    target_vectors_filename = args.target_vector
+
+    vectors_source = load_embeddings(source_vectors_filename)
+    vectors_target = load_embeddings(target_vectors_filename)
+
+    source_defs_filename = args.source_defs
+    target_defs_filename = args.target_defs
+    defs_source = [
+        line.rstrip("\n") for line in open(source_defs_filename, encoding="utf8")
+    ]
+    defs_target = [
+        line.rstrip("\n") for line in open(target_defs_filename, encoding="utf8")
+    ]
+
+    clean_source_corpus, clean_source_vectors, source_keys = process_corpus(
+        set(vectors_source.keys()), defs_source, vectors_source, source_lang
+    )
+
+    clean_target_corpus, clean_target_vectors, target_keys = process_corpus(
+        set(vectors_target.keys()), defs_target, vectors_target, target_lang
+    )
+
+    take = args.instances
+    common_keys = set(source_keys).intersection(set(target_keys))
+    take = min(len(common_keys), take)  # you can't sample more than length
+    experiment_keys = random.sample(common_keys, take)
+
+    instances = len(experiment_keys)
+
+    clean_source_corpus = list(clean_source_corpus[experiment_keys])
+    clean_target_corpus = list(clean_target_corpus[experiment_keys])
+
+    if not batch:
+        print(
+            f"{source_lang} - {target_lang} "
+            + f" document sizes: {len(clean_source_corpus)}, {len(clean_target_corpus)}"
+        )
+
+    del vectors_source, vectors_target, defs_source, defs_target
+
+    vocab_counter = CountVectorizer().fit(clean_source_corpus + clean_target_corpus)
+    common = [
+        w
+        for w in vocab_counter.get_feature_names()
+        if w in clean_source_vectors or w in clean_target_vectors
+    ]
+    W_common = []
+
+    for w in common:
+        if w in clean_source_vectors:
+            W_common.append(np.array(clean_source_vectors[w]))
+        else:
+            W_common.append(np.array(clean_target_vectors[w]))
+
+    W_common = np.array(W_common)
+    W_common = normalize(W_common)  # default is l2
+
+    vect_tfidf = TfidfVectorizer(vocabulary=common, dtype=np.double, norm="l2")
+    vect_tfidf.fit(clean_source_corpus + clean_target_corpus)
+    X_idf_source = vect_tfidf.transform(clean_source_corpus)
+    X_idf_target = vect_tfidf.transform(clean_target_corpus)
+
+    X_idf_source_array = X_idf_source.toarray()
+    X_idf_target_array = X_idf_target.toarray()
+    S_emb_source = np.matmul(X_idf_source_array, W_common)
+    S_emb_target = np.matmul(X_idf_target_array, W_common)
+
+    S_emb_target_transpose = np.transpose(S_emb_target)
+
+    cost_matrix = np.matmul(S_emb_source, S_emb_target_transpose)
+
+    for paradigm in run_paradigm:
+        if paradigm == 'matching':
+
+            cost_matrix = cost_matrix * -1000
+            row_ind, col_ind, a = lapjv(cost_matrix, verbose=False)
+
+            result = zip(row_ind, col_ind)
+            hit_at_one = len([x for x, y in result if x == y])
+            percentage = hit_at_one / instances * 100
+
+            if not batch:
+                print(f"{hit_at_one} definitions have been matched correctly")
+
+            if batch:
+                fields = [
+                    f"{source_lang}",
+                    f"{target_lang}",
+                    f"{instances}",
+                    f"{hit_at_one}",
+                    f"{percentage}",
+                ]
+
+            with open("semb_matcing_results.csv", "a") as f:
+                writer = csv.writer(f)
+                writer.writerow(fields)
+
+        if paradigm == 'retrieval':
+
+            hit_at_one = len([x for x, y in enumerate(cost_matrix.argmax(axis=1)) if x == y])
+            percentage = hit_at_one / instances * 100
+
+            if not batch:
+                print(f"{hit_at_one} definitions have retrieved correctly")
+
+            if batch:
+                fields = [
+                    f"{source_lang}",
+                    f"{target_lang}",
+                    f"{instances}",
+                    f"{hit_at_one}",
+                    f"{percentage}",
+                ]
+
+                with open("semb_retrieval_results.csv", "a") as f:
+                    writer = csv.writer(f)
+                    writer.writerow(fields)
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(
+        description="align dictionaries using sentence embedding representation"
+    )
+    parser.add_argument("source_lang", help="source language short name")
+    parser.add_argument("target_lang", help="target language short name")
+    parser.add_argument("source_vector", help="path of the source vector")
+    parser.add_argument("target_vector", help="path of the target vector")
+    parser.add_argument("source_defs", help="path of the source definitions")
+    parser.add_argument("target_defs", help="path of the target definitions")
+    parser.add_argument(
+        "-n",
+        "--instances",
+        help="number of instances in each language to retrieve",
+        default=1000,
+        type=int,
+    )
+    parser.add_argument(
+        "-b",
+        "--batch",
+        action="store_true",
+        help="running in batch (store results in csv) or"
+        + "running a single instance (output the results)",
+    )
+    parser.add_argument(
+        "paradigm",
+        choices=["all", "retrieval", "matching"],
+        default="all",
+        help="which paradigms to align with",
+    )
+
+    args = parser.parse_args()
+    main(args)