Publish site and docs to namecheap (#54)

.github/scripts/setup-env.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+mkdir -p $HOME/.gnupg $HOME/.gradle $HOME/.ssh
+
+echo $GPG_SECRET_B64 | base64 --decode > $HOME/.gnupg/secring.gpg
+echo $GRADLE_PROPERTIES_B64 | base64 --decode > $HOME/.gradle/gradle.properties
+echo $PYPIRC_B64 | base64 --decode > $HOME/.pypirc
+echo $SSH_CONFIG_B64 | base64 --decode > $HOME/.ssh/config
+echo $SSH_IDRSA_B64 | base64 --decode > $HOME/.ssh/elastiknn-site
+chmod 400 $HOME/.ssh/elastiknn-site

.github/workflows/ci.yml

@@ -21,14 +21,6 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    # Secrets used for publishing.
-    - name: Setup Secrets
-      env:
-        GPG_SECRET_B64: ${{ secrets.GPG_SECRET_B64 }}
-        GRADLE_PROPERTIES_B64: ${{ secrets.GRADLE_PROPERTIES_B64 }}
-        PYPIRC_B64: ${{ secrets.PYPIRC_B64 }}
-      run: ./.github/scripts/setup-secrets.sh
-
     # Language Setup
     - name: Setup Python
       run: |
@@ -40,6 +32,22 @@ jobs:
       with:
         java-version: 12.0.2
 
+    - name: Setup Ruby
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: '2.6'
+
+    # Needs to run after language setup.
+    - name: Setup Environment
+      env:
+        GPG_SECRET_B64: ${{ secrets.GPG_SECRET_B64 }}
+        GRADLE_PROPERTIES_B64: ${{ secrets.GRADLE_PROPERTIES_B64 }}
+        PYPIRC_B64: ${{ secrets.PYPIRC_B64 }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        SSH_CONFIG_B64: ${{ secrets.SSH_CONFIG_B64 }}
+        SSH_IDRSA_B64: ${{ secrets.SSH_IDRSA_B64 }}
+      run: ./.github/scripts/setup-env.sh
+
     # Caching Setup
     - name: Cache Gradle
       uses: actions/cache@v1
@@ -78,6 +86,12 @@ jobs:
     - name: Run Examples
       run: make examples
 
+    # Site and Docs
+    - name: Compile Site and Docs
+      run: |
+        gem install bundler
+        make compile/site compile/docs
+
     # Snapshot Releases
     - name: Set Version (PR)
       if: github.event_name == 'pull_request'
@@ -101,8 +115,8 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
+        sudo snap install hub --classic  
         cp version-sonatype version
-        sudo snap install hub --classic
         make publish/snapshot/plugin
 
     # Cleanup

.github/workflows/release.yml

@@ -4,9 +4,7 @@ on:
   push:
     branches: 
       - master
-  pull_request:
-    branches:
-      - master
+
 jobs:
   release:
     runs-on: ubuntu-18.04
@@ -23,42 +21,48 @@ jobs:
 
     # Language Setup
     - name: Setup Python
-      if: github.event_name == 'push' && env.GET_RELEASE == '404'
+      if: env.GET_RELEASE == '404'
       run: |
         python3 --version
         sudo apt-get install -y python3-setuptools
 
     - name: Setup Java
       uses: actions/setup-java@v1
-      if: github.event_name == 'push' && env.GET_RELEASE == '404'
+      if: env.GET_RELEASE == '404'
       with:
         java-version: 12.0.2
 
-#    # Release docs
-#    - name: Setup Docs
-#      if: github.event_name == 'push' && env.GET_RELEASE == '404'
-#      run: |
-#        cp docs/Gemfile .
-#
-#    # Specify the Jekyll source location as a parameter
-#    - uses: helaili/jekyll-action@2.0.0
-#      if: github.event_name == 'push' && env.GET_RELEASE == '404'
-#      env:
-#        JEKYLL_PAT: ${{ secrets.JEKYLL_PAT }}
-#      with:
-#        jekyll_src: 'docs'
+    - name: Setup Ruby
+      uses: actions/setup-ruby@v1
+      if: env.GET_RELEASE == '404'
+      with:
+        ruby-version: '2.6'
 
-    # Release software
-    - name: Release software artifacts
+    # Has to run after installing languages.
+    - name: Setup Environment
+      if: env.GET_RELEASE == '404'
       env:
         GPG_SECRET_B64: ${{ secrets.GPG_SECRET_B64 }}
         GRADLE_PROPERTIES_B64: ${{ secrets.GRADLE_PROPERTIES_B64 }}
         PYPIRC_B64: ${{ secrets.PYPIRC_B64 }}
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      if: github.event_name == 'push' && env.GET_RELEASE == '404'
+        SSH_CONFIG_B64: ${{ secrets.SSH_CONFIG_B64 }}
+        SSH_IDRSA_B64: ${{ secrets.SSH_IDRSA_B64 }}
+      run: ./.github/scripts/setup-env.sh
+
+    - name: Publish Site
+      run: |
+        gem install bundler
+        make publish/site
+
+    - name: Publish Docs
+      if: env.GET_RELEASE == '404'
+      run: make publish/docs
+
+    - name: Release Artifacts
+      if: env.GET_RELEASE == '404'
       run: |
         sudo snap install hub --classic
-        ./.github/scripts/setup-secrets.sh
         make publish/release/python
         make publish/release/sonatype
         make publish/release/plugin

Makefile

@@ -10,6 +10,9 @@ build_bucket = s3://com-klibisz-elastiknn-builds/
 dc = docker-compose
 version = $(shell cat version)
 src_all = $(shell git diff --name-only --diff-filter=ACMR)
+site_srvr = elastiknn-site
+site_main = elastiknn.klibisz.com
+site_arch = archive.elastiknn.klibisz.com
 
 clean:
 	./gradlew clean
@@ -29,10 +32,10 @@ clean:
 	cd client-python && python3 -m virtualenv venv
 	touch $@
 
-.mk/client-python-install: .mk/client-python-venv
+.mk/client-python-install: .mk/client-python-venv client-python/requirements*.txt
 	cd client-python \
 		&& $(vpip) install -q -r requirements.txt \
-		&& $(vpip) install -q pytest twine
+		&& $(vpip) install -q -r requirements-build.txt
 	touch $@
 
 .mk/gradle-compile: $(src_all)
@@ -114,3 +117,32 @@ publish/snapshot/python: .mk/client-python-publish-local
 publish/release/python: .mk/client-python-publish-local
 	cd client-python \
 	&& $(vpy) -m twine upload -r pypi dist/*
+
+.mk/gradle-docs: $(src_all)
+	$(gradle) unifiedScaladocs
+	touch $@
+
+.mk/client-python-docs: $(src_all) .mk/client-python-install
+	cd client-python \
+	&& rm -rf pdoc \
+	&& venv/bin/pdoc3 --html elastiknn -c show_type_annotations=True -o pdoc
+	touch $@
+
+.mk/jekyll-site-build: docs/**/*
+	cd docs \
+	&& bundle install \
+	&& bundle exec jekyll build
+	touch $@
+
+compile/docs: .mk/gradle-docs .mk/client-python-docs
+
+compile/site: .mk/jekyll-site-build
+
+publish/docs: .mk/gradle-docs .mk/client-python-docs
+	ssh elastiknn-site mkdir -p $(site_arch)/$(version)
+	rsync -av --delete build/docs/scaladoc $(site_srvr):$(site_arch)/$(version)
+	rsync -av --delete client-python/pdoc/elastiknn/ $(site_srvr):$(site_arch)/$(version)/pdoc
+
+publish/site: .mk/jekyll-site-build
+	rsync -av --delete docs/_site/ $(site_srvr):$(site_main)
+

build.gradle

@@ -22,5 +22,22 @@ allprojects {
     ext."sonatypeUrl" = System.getenv().getOrDefault("SONATYPE_URL", "https://oss.sonatype.org/content/repositories/snapshots/")
     ext."sonatypeUsername" = project.hasProperty("sonatypeUsername") ? project.getProperty("sonatypeUsername") : ""
     ext."sonatypePassword" = project.hasProperty("sonatypePassword") ? project.getProperty("sonatypePassword") : ""
+}
 
+task unifiedScaladocs(type: ScalaDoc, description: 'Generate unified scaladocs', group: 'Documentation') {
+    Set docProjects = ["core", "client-elastic4s"]
+    destinationDir = file("$buildDir/docs/scaladoc")
+    title = "$project.name $version"
+    scalaDocOptions = new ScalaDocOptions()
+    subprojects.each { proj ->
+        if (docProjects.contains(proj.name)) {
+            proj.tasks.withType(ScalaDoc).each {
+                source += proj.sourceSets.main.allJava
+                source += proj.sourceSets.main.allScala
+                classpath += proj.sourceSets.main.compileClasspath
+                excludes += scaladoc.excludes
+                includes += scaladoc.includes
+            }
+        }
+    }
 }

client-python/.gitignore

@@ -133,3 +133,4 @@ elastiknn/elastiknn_pb2.py
 elastiknn/elastiknn.proto
 scalapb/scalapb_pb2.py
 *.pyi
+pdoc

client-python/elastiknn/api.py

@@ -1,3 +1,5 @@
+"""Python dataclasses representing the Elastiknn Json API."""
+
 from abc import ABC
 from enum import Enum
 from random import Random

client-python/elastiknn/client.py

@@ -1,4 +1,6 @@
-from typing import Iterable
+"""Client for using Elastiknn."""
+
+from typing import Iterable, Tuple, Dict
 
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import bulk
@@ -8,21 +10,73 @@
 
 class ElastiKnnClient(object):
 
-    def __init__(self, hosts: List[str] = None):
-        if hosts is None:
-            hosts = ["http://localhost:9200"]
-        self.hosts = hosts
-        self.es = Elasticsearch(self.hosts)
+    def __init__(self, es: Elasticsearch = None):
+        """Wrapper on the official `elasticsearch.Elasticsearch` client for making Elastiknn requests.
+
+        The client is fairly primitive in that it assumes the vector is basically the only thing stored in each doc.
+        For more complex use-cases you should use this client as an example for constructing the mapping, indexing,
+        and search requests in your own code.
+
+        Parameters
+        ----------
+        es : `elasticsearch.Elasticsearch` client.
+            This client is used internally to make all requests.
+            Defaults to a client pointing at http://localhost:9200.
+        """
+        if es is None:
+            self.es = Elasticsearch(["http://localhost:9200"])
+        else:
+            self.es = es
 
     def put_mapping(self, index: str, field: str, mapping: Mapping.Base):
+        """
+        Update the mapping at the given index and field to store an Elastiknn vector.
+
+        Parameters
+        ----------
+        index : string
+            Index containing the given field. Should already exist.
+        field : string
+            Field that should use the given mapping.
+        mapping : instance of `Mapping.Base`
+            Mapping object defining the vector's storage properties.
+
+        Returns
+        -------
+        Dict
+            Json response as a dict. Successful request returns `{"acknowledged": true}`.
+        """
         body = {
             "properties": {
                 field: mapping.to_dict()
             }
         }
         return self.es.transport.perform_request("PUT", f"/{index}/_mapping", body=body)
 
-    def index(self, index: str, field: str, vecs: Iterable[Vec.Base], ids: List[str] = None, refresh: bool = False) -> (int, List):
+    def index(self, index: str, field: str, vecs: Iterable[Vec.Base], ids: List[str] = None, refresh: bool = False) -> Tuple[int, List[Dict]]:
+        """Index (i.e. store) the given vectors at the given index and field with the optional ids.
+
+        Parameters
+        ----------
+        index : string
+            Index where the vectors are stored.
+        field : string
+            Field containing the vector in each document.
+        vecs : List of `Vec.Base`
+            Vectors that should be indexed.
+        ids : List of strings
+            Optional list of ids associated with the given vectors.
+            If not given, the ids are generated by Elasticsearch.
+        refresh : bool
+            Whether to refresh before returning. Set to true if you want to immediately run queries after indexing.
+
+        Returns
+        -------
+        Int
+            Number of vectors successfully indexed.
+        List of Dicts
+            Error responses for the vectors that were not indexed.
+        """
         if ids is None or ids == []:
             ids = [None for _ in vecs]
 
@@ -40,7 +94,25 @@ def gen():
             self.es.indices.refresh(index=index)
         return res
 
-    def nearest_neighbors(self, index: str, query: NearestNeighborsQuery.Base, k: int = 10, fetch_source: bool = False):
+    def nearest_neighbors(self, index: str, query: NearestNeighborsQuery.Base, k: int = 10, fetch_source: bool = False) -> Dict:
+        """Build and execute a nearest neighbors query against the given index.
+
+        Parameters
+        ----------
+        index : string
+            Index to run the search against.
+        query : NearestNeighborsQuery.Base
+            Query object defining the query properties.
+        k: int
+            Number of hits to return.
+        fetch_source : bool
+            Whether to return the `_source` of the document. It's generally faster to _not_ return the `_source`.
+
+        Returns
+        -------
+        Dict
+            Standard Elasticsearch search response parsed as a dict.
+        """
         body = {
             "query": {
                 "elastiknn_nearest_neighbors": query.to_dict()

client-python/elastiknn/codec.py

@@ -1,8 +1,12 @@
+"""Mixin for API objects that can be converted to a dict compatible with the Elastiknn Json API."""
+
 from typing import Dict
 
 
 class Codec(object):
+    """Mixin for API objects that can be converted to a dict compatible with the Elastiknn Json API."""
 
-    def to_dict(self):
+    def to_dict(self) -> Dict:
+        """Return a dictionary compatible with the Elastiknn Json API."""
         raise NotImplementedError