Fix failing on some models (#20)

Author: evgenyrp

3rd_party/crow

@@ -24,6 +24,11 @@ RUN sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.
     apt-get update && \
     apt-get install -y intel-mkl-64bit-2020.0-088
 
+# attempt to use openblas
+#RUN apt search openblas
+#RUN apt install -y libopenblas-dev
+#RUN update-alternatives --config libblas.so.3
+
 # Bergamot
 # pcre2 is requried to build berrgamot-translator with -DUSE_WASM_COMPATIBLE_SOURCES=off
 RUN apt-get install -y libpcre2-dev

3rd_party/moz-bergamot-translator

@@ -16,7 +16,7 @@ run:
 	docker run --name translation-service -it --rm -v $$(pwd)/models:/models -p 8080:8080 -e PORT=8080 translation-service
 
 debug:
-	docker run --name translation-service -it --rm -v $$(pwd):/app -v $$(pwd)/models:/models -p 8080:8080 translation-service bash
+	docker run --name translation-service -it --rm -v $$(pwd):/app -v $$(pwd)/models:/models -e PORT=8080 -p 8080:8080 translation-service bash
 
 call:
 	curl --header "Content-Type: application/json" \

Dockerfile

@@ -1,24 +1,32 @@
 #!/bin/bash
 
+version=0.3.0
+
 mkdir models
 cd models
 
 mkdir esen
 mkdir enet
 mkdir enru
+mkdir enbg
 
 cd esen
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/esen/lex.50.50.esen.s2t.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/esen/model.esen.intgemm.alphas.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/esen/vocab.esen.spm
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/esen/lex.50.50.esen.s2t.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/esen/model.esen.intgemm.alphas.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/esen/vocab.esen.spm
 
 cd ../enet
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enet/lex.50.50.enet.s2t.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enet/model.enet.intgemm.alphas.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enet/vocab.eten.spm
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enet/lex.50.50.enet.s2t.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enet/model.enet.intgemm.alphas.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enet/vocab.eten.spm
 
 
 cd ../enru
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enru/lex.50.50.enru.s2t.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enru/model.enru.intgemm.alphas.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enru/vocab.enru.spm
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enru/lex.50.50.enru.s2t.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enru/model.enru.intgemm.alphas.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enru/vocab.enru.spm
+
+cd ../enbg
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enbg/lex.50.50.enbg.s2t.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enbg/model.enbg.intgemm.alphas.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enbg/vocab.bgen.spm

Makefile

@@ -107,13 +107,14 @@ namespace marian {
                         "\nmini-batch-words: 1024"
                         "\nworkspace: 128"
                         "\nmax-length-factor: 2.0"
-                        "\nskip-costls: True"
+                        "\nskip-cost: True"
                         "\nquiet: True"
                         "\nquiet-translation: True"
-                        "\ngemm-precision: int8shift";
+                        "\ngemm-precision: int8shift"
+                        "\nalignment: soft";
 
                 options = options + "\nmodels: [" + modelPath + "]\nvocabs: [" + vocabPath + ", " + vocabPath +
-                          "]\nshortlist: [" + shortlistPath + ", 50, 50]";
+                          "]\nshortlist: [" + shortlistPath + ", false]";
 
                 return options;
             }

scripts/download-test-models.sh

@@ -44,6 +44,14 @@ def test_non_latin():
     assert translation == 'Как дела?' or translation == 'Как у тебя дела?'
 
 
+def test_different_shortlist():
+    query = {"from": "en", "to": "bg", "text": 'Hello world'}
+
+    resp = requests.post(url + '/v1/translate', json=query)
+
+    assert resp.status_code == 200
+    assert resp.json()['result'] == 'Здравей свят'
+
 def test_json_escaping():
     query = {"from": "es", "to": "en", "text": '"Hola Mundo"'}