Merge branch '2.2-dev'

# Conflicts:
#	requirements.txt
#	setup.py
#	tutorials/wikiqa/dssm.ipynb

Author: uduse

.gitattributes

@@ -0,0 +1 @@
+tutorials/* linguist-vendored

.travis.yml

@@ -5,24 +5,22 @@ cache: pip
 sudo: true
 
 env:
-    global:
-        - PYTHONPATH=$PYTHONPATH:$TRAVIS_BUILD_DIR/tests:$TRAVIS_BUILD_DIR/matchzoo
+  global:
+    - PYTHONPATH=$PYTHONPATH:$TRAVIS_BUILD_DIR/tests:$TRAVIS_BUILD_DIR/matchzoo
 
 matrix:
-    allow_failures:
-      - os: osx
-    include:
-      - os: linux
-        dist: trusty
-        python: 3.6
-      - os: osx
-        language: generic
-        env: PYTHON_VERSION=3.6
-      - os: osx
-        language: generic
-        env: PYTHON_VERSION=3.7
+  allow_failures:
+    - os: osx
+  include:
+    - os: linux
+      dist: trusty
+      python: 3.6
+    - os: osx
+      osx_image: xcode10.2
+      language: shell
 
 install:
+  - pip3 install -U pip
   - pip3 install -r requirements.txt
   - python3 -m nltk.downloader punkt
   - python3 -m nltk.downloader wordnet
@@ -31,7 +29,10 @@ install:
 script:
   - stty cols 80
   - export COLUMNS=80
-  - make test
+  - if [ "$TRAVIS_EVENT_TYPE" == "pull_request" ]; then make push; fi
+  - if [ "$TRAVIS_EVENT_TYPE" == "push" ]; then make push; fi
+  - if [ "$TRAVIS_EVENT_TYPE" == "cron" ]; then make cron; fi
+
 
 after_success:
-      - codecov
+  - codecov

Makefile

@@ -1,18 +1,70 @@
+# Usages:
+#
+# to install matchzoo dependencies:
+# $ make init
+#
+# to run all matchzoo tests, recommended for big PRs and new versions:
+# $	make test
+#
+# there are three kinds of tests:
+#
+# 1. "quick" tests
+# 	- run in seconds
+#   - include all unit tests without marks and all doctests
+#   - for rapid prototyping
+#   - CI run this for all PRs
+#
+# 2. "slow" tests
+#   - run in minutes
+#   - include all unit tests marked "slow"
+#   - CI run this for all PRs
+#
+# 3. "cron" tests
+#   - run in minutes
+#   - involves underministic behavoirs (e.g. network connection)
+#   - include all unit tests marked "cron"
+#   - CI run this on a daily basis
+#
+# to run quick tests, excluding time consuming tests and crons:
+# $ make quick
+#
+# to run slow tests, excluding normal tests and crons:
+# $ make slow
+#
+# to run crons:
+# $ make cron
+#
+# to run all tests:
+# $ make test
+#
+# to run CI push/PR tests:
+# $ make push
+#
+# to run docstring style check:
+# $ make flake
+
 init:
 	pip install -r requirements.txt
 
-TEST_ARGS = --doctest-modules --doctest-continue-on-failure --cov matchzoo/ --cov-report term-missing --cov-report html --cov-config .coveragerc matchzoo/ tests/ -W ignore::DeprecationWarning
+TEST_ARGS = -v --full-trace -l --doctest-modules --doctest-continue-on-failure --cov matchzoo/ --cov-report term-missing --cov-report html --cov-config .coveragerc matchzoo/ tests/ -W ignore::DeprecationWarning --ignore=matchzoo/contrib
 FLAKE_ARGS = ./matchzoo --exclude=__init__.py,matchzoo/contrib
 
 test:
 	pytest $(TEST_ARGS)
 	flake8 $(FLAKE_ARGS)
 
+push:
+	pytest -m 'not cron' $(TEST_ARGS) ${ARGS}
+	flake8 $(FLAKE_ARGS)
+
 quick:
-	pytest -m 'not slow' $(TEST_ARGS)
+	pytest -m 'not slow and not cron' $(TEST_ARGS) ${ARGS}
 
 slow:
-	pytest -m 'slow' $(TEST_ARGS)
+	pytest -m 'slow and not cron' $(TEST_ARGS) ${ARGS}
+
+cron:
+	pytest -m 'cron' $(TEST_ARGS) ${ARGS}
 
 flake:
-	flake8 $(FLAKE_ARGS)
+	flake8 $(FLAKE_ARGS) ${ARGS}

README.md

@@ -8,7 +8,6 @@
 > MatchZoo 是一个通用的文本匹配工具包，它旨在方便大家快速的实现、比较、以及分享最新的深度文本匹配模型。
 
 [![Python 3.6](https://img.shields.io/badge/python-3.6%20%7C%203.7-blue.svg)](https://www.python.org/downloads/release/python-360/)
-[![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/NTMC-Community/community)
 [![Pypi Downloads](https://img.shields.io/pypi/dm/matchzoo.svg?label=pypi)](https://pypi.org/project/MatchZoo/)
 [![Documentation Status](https://readthedocs.org/projects/matchzoo/badge/?version=master)](https://matchzoo.readthedocs.io/en/master/?badge=master)
 [![Build Status](https://travis-ci.org/NTMC-Community/MatchZoo.svg?branch=master)](https://travis-ci.org/NTMC-Community/MatchZoo/)
@@ -68,7 +67,6 @@ import matchzoo as mz
 
 train_pack = mz.datasets.wiki_qa.load_data('train', task='ranking')
 valid_pack = mz.datasets.wiki_qa.load_data('dev', task='ranking')
-predict_pack = mz.datasets.wiki_qa.load_data('test', task='ranking')
 ```
 
 Preprocess your input data in three lines of code, keep track parameters to be passed into the model.
@@ -85,7 +83,6 @@ Make use of MatchZoo customized loss functions and evaluation metrics:
 ranking_task = mz.tasks.Ranking(loss=mz.losses.RankCrossEntropyLoss(num_neg=4))
 ranking_task.metrics = [
     mz.metrics.NormalizedDiscountedCumulativeGain(k=3),
-    mz.metrics.NormalizedDiscountedCumulativeGain(k=5),
     mz.metrics.MeanAveragePrecision()
 ]
 ```
@@ -96,10 +93,6 @@ Initialize the model, fine-tune the hyper-parameters.
 model = mz.models.DSSM()
 model.params['input_shapes'] = preprocessor.context['input_shapes']
 model.params['task'] = ranking_task
-model.params['mlp_num_layers'] = 3
-model.params['mlp_num_units'] = 300
-model.params['mlp_num_fan_out'] = 128
-model.params['mlp_activation_func'] = 'relu'
 model.guess_and_fill_missing_params()
 model.build()
 model.compile()
@@ -109,10 +102,8 @@ Generate pair-wise training data on-the-fly, evaluate model performance using cu
 
 ```python
 train_generator = mz.PairDataGenerator(train_processed, num_dup=1, num_neg=4, batch_size=64, shuffle=True)
-
 valid_x, valid_y = valid_processed.unpack()
-evaluate = mz.callbacks.EvaluateAllMetrics(model, x=valid_x, y=valid_y, batch_size=len(pred_x))
-
+evaluate = mz.callbacks.EvaluateAllMetrics(model, x=valid_x, y=valid_y, batch_size=len(valid_x))
 history = model.fit_generator(train_generator, epochs=20, callbacks=[evaluate], workers=5, use_multiprocessing=False)
 ```
 
@@ -127,7 +118,7 @@ If you're interested in the cutting-edge research progress, please take a look a
 
 ## Install
 
-MatchZoo is dependent on [Keras](https://github.com/keras-team/keras), please install one of its backend engines: TensorFlow, Theano, or CNTK. We recommend the TensorFlow backend. Two ways to install MatchZoo:
+MatchZoo is dependent on [Keras](https://github.com/keras-team/keras) and [Tensorflow](https://github.com/tensorflow/tensorflow). Two ways to install MatchZoo:
 
 **Install MatchZoo from Pypi:**
 
@@ -144,7 +135,7 @@ python setup.py install
 ```
 
 
-## Models:
+## Models
 
 1. [DRMM](https://github.com/NTMC-Community/MatchZoo/tree/master/matchzoo/models/drmm.py): this model is an implementation of <a href="http://www.bigdatalab.ac.cn/~gjf/papers/2016/CIKM2016a_guo.pdf">A Deep Relevance Matching Model for Ad-hoc Retrieval</a>.
 

docs/Readme.md

@@ -16,7 +16,8 @@ pip install -r requirements.txt
 # Enter docs folder.
 cd docs
 # Use sphinx autodoc to generate rst.
-sphinx-apidoc -o source/ ../matchzoo/
+# usage: sphinx-apidoc [OPTIONS] -o <OUTPUT_PATH> <MODULE_PATH> [EXCLUDE_PATTERN,...]
+sphinx-apidoc -o source/ ../matchzoo/ ../matchzoo/contrib
 # Generate html from rst
 make clean
 make html

docs/source/conf.py

@@ -20,15 +20,16 @@
 sys.path.insert(0, os.path.abspath('../../matchzoo/data_generator'))
 sys.path.insert(0, os.path.abspath('../../matchzoo/data_pack'))
 sys.path.insert(0, os.path.abspath('../../matchzoo/datasets'))
+sys.path.insert(0, os.path.abspath('../../matchzoo/embedding'))
 sys.path.insert(0, os.path.abspath('../../matchzoo/engine'))
 sys.path.insert(0, os.path.abspath('../../matchzoo/layers'))
 sys.path.insert(0, os.path.abspath('../../matchzoo/losses'))
-sys.path.insert(0, os.path.abspath('../../matchzoo/models'))
 sys.path.insert(0, os.path.abspath('../../matchzoo/metrics'))
+sys.path.insert(0, os.path.abspath('../../matchzoo/models'))
 sys.path.insert(0, os.path.abspath('../../matchzoo/preprocessors'))
-sys.path.insert(0, os.path.abspath('../../matchzoo/processor_units'))
-sys.path.insert(0, os.path.abspath('../../matchzoo/utils'))
 sys.path.insert(0, os.path.abspath('../../matchzoo/tasks'))
+sys.path.insert(0, os.path.abspath('../../matchzoo/utils'))
+
 
 # -- Project information -----------------------------------------------------
 
@@ -39,7 +40,7 @@
 # The short X.Y version
 version = ''
 # The full version, including alpha/beta/rc tags
-release = '2.0'
+release = '2.1'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/matchzoo.auto.preparer.rst

@@ -0,0 +1,30 @@
+matchzoo.auto.preparer package
+==============================
+
+Submodules
+----------
+
+matchzoo.auto.preparer.prepare module
+-------------------------------------
+
+.. automodule:: matchzoo.auto.preparer.prepare
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+matchzoo.auto.preparer.preparer module
+--------------------------------------
+
+.. automodule:: matchzoo.auto.preparer.preparer
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: matchzoo.auto.preparer
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/matchzoo.auto.rst

@@ -1,25 +1,13 @@
 matchzoo.auto package
 =====================
 
-Submodules
-----------
+Subpackages
+-----------
 
-matchzoo.auto.prepare module
-----------------------------
-
-.. automodule:: matchzoo.auto.prepare
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-matchzoo.auto.tune module
--------------------------
-
-.. automodule:: matchzoo.auto.tune
-    :members:
-    :undoc-members:
-    :show-inheritance:
+.. toctree::
 
+    matchzoo.auto.preparer
+    matchzoo.auto.tuner
 
 Module contents
 ---------------

docs/source/matchzoo.auto.tuner.callbacks.rst

@@ -0,0 +1,46 @@
+matchzoo.auto.tuner.callbacks package
+=====================================
+
+Submodules
+----------
+
+matchzoo.auto.tuner.callbacks.callback module
+---------------------------------------------
+
+.. automodule:: matchzoo.auto.tuner.callbacks.callback
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+matchzoo.auto.tuner.callbacks.lambda\_callback module
+-----------------------------------------------------
+
+.. automodule:: matchzoo.auto.tuner.callbacks.lambda_callback
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+matchzoo.auto.tuner.callbacks.load\_embedding\_matrix module
+------------------------------------------------------------
+
+.. automodule:: matchzoo.auto.tuner.callbacks.load_embedding_matrix
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+matchzoo.auto.tuner.callbacks.save\_model module
+------------------------------------------------
+
+.. automodule:: matchzoo.auto.tuner.callbacks.save_model
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: matchzoo.auto.tuner.callbacks
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/matchzoo.auto.tuner.rst

@@ -0,0 +1,37 @@
+matchzoo.auto.tuner package
+===========================
+
+Subpackages
+-----------
+
+.. toctree::
+
+    matchzoo.auto.tuner.callbacks
+
+Submodules
+----------
+
+matchzoo.auto.tuner.tune module
+-------------------------------
+
+.. automodule:: matchzoo.auto.tuner.tune
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+matchzoo.auto.tuner.tuner module
+--------------------------------
+
+.. automodule:: matchzoo.auto.tuner.tuner
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: matchzoo.auto.tuner
+    :members:
+    :undoc-members:
+    :show-inheritance: