From 4814eaf938f912ad08debc55db649fb2f671e645 Mon Sep 17 00:00:00 2001
From: Todd Bilsborrow <todd@synapse-ai.com>
Date: Wed, 30 Mar 2016 11:16:05 -0700
Subject: [PATCH] Estimate confidence numbers for hypotheses, based on Kaldi's
likelihood numbers. Confidence algorithm borrowed from the
sample_full_post_processor commits 2d1be9d and ea87b6a, and implemented in
the main master/worker code.
---
kaldigstserver/master_server.py | 14 +++++++++-----
kaldigstserver/worker.py | 18 +++++++++++++++++-
2 files changed, 26 insertions(+), 6 deletions(-)
diff --git a/kaldigstserver/master_server.py b/kaldigstserver/master_server.py
index d7883c9..b634703 100644
--- a/kaldigstserver/master_server.py
+++ b/kaldigstserver/master_server.py
@@ -114,7 +114,7 @@ class HttpChunkedRecognizeHandler(tornado.web.RequestHandler):
def prepare(self):
self.id = str(uuid.uuid4())
- self.final_hyp = ""
+ self.final_hyp = {"transcript":"", "num_segments":0, "total_confidence":0.0}
self.final_result_queue = Queue()
self.user_id = self.request.headers.get("device-id", "none")
self.content_id = self.request.headers.get("content-id", "none")
@@ -165,7 +165,9 @@ def end_request(self, *args, **kwargs):
hyp = yield tornado.gen.Task(self.get_final_hyp)
if self.error_status == 0:
logging.info("%s: Final hyp: %s" % (self.id, hyp))
- response = {"status" : 0, "id": self.id, "hypotheses": [{"utterance" : hyp}]}
+ # overall confidence is the average of all segments' confidences
+ hyp_confidence = 0 if hyp["num_segments"] == 0 else hyp["total_confidence"] / hyp["num_segments"]
+ response = {"status" : 0, "id": self.id, "hypotheses": [{"utterance" : hyp["transcript"], ":confidence": hyp_confidence}]}
self.write(response)
else:
logging.info("%s: Error (status=%d) processing HTTP request: %s" % (self.id, self.error_status, self.error_message))
@@ -186,9 +188,11 @@ def send_event(self, event):
if event["status"] == 0 and ("result" in event):
try:
if len(event["result"]["hypotheses"]) > 0 and event["result"]["final"]:
- if len(self.final_hyp) > 0:
- self.final_hyp += " "
- self.final_hyp += event["result"]["hypotheses"][0]["transcript"]
+ if len(self.final_hyp["transcript"]) != 0:
+ self.final_hyp["transcript"] += " "
+ self.final_hyp["transcript"] += event["result"]["hypotheses"][0]["transcript"]
+ self.final_hyp["num_segments"] += 1
+ self.final_hyp["total_confidence"] += event["result"]["hypotheses"][0]["confidence"]
except:
e = sys.exc_info()[0]
logging.warn("Failed to extract hypothesis from recognition result:" + e)
diff --git a/kaldigstserver/worker.py b/kaldigstserver/worker.py
index 6dc79a5..c9a4005 100644
--- a/kaldigstserver/worker.py
+++ b/kaldigstserver/worker.py
@@ -15,6 +15,7 @@
import zlib
import base64
import time
+from math import exp
from ws4py.client.threadedclient import WebSocketClient
@@ -189,13 +190,28 @@ def _on_full_result(self, full_result_json):
full_result = json.loads(full_result_json)
full_result['segment'] = self.num_segments
if full_result.get("status", -1) == common.STATUS_SUCCESS:
+
+ # confidence estimation (requires hypotheses in descending likelihood order)
+ hypotheses = full_result["result"]["hypotheses"]
+ last_confidence = 1
+ for h in [hypotheses[i:i+2] for i in xrange(len(hypotheses))]:
+ # iterate over sliding window of size 2, including a 1-element slice at the end
+ if len(h) == 1:
+ # this is the last hypothesis (also could be the only one)
+ h[0]["confidence"] = last_confidence
+ else:
+ # otherwise confidence is based on how far away the following likelihood is
+ # but no larger than the previous confidence
+ curr_hyp, next_hyp = h[0], h[1]
+ last_confidence = min(last_confidence, 1 - exp(next_hyp["likelihood"] - curr_hyp["likelihood"]))
+ curr_hyp["confidence"] = last_confidence
+
#logger.info("%s: Postprocessing (final=%s) result.." % (self.request_id, final))
logger.debug("%s: Before postprocessing: %s" % (self.request_id, full_result))
full_result = self.post_process_full(full_result)
logger.info("%s: Postprocessing done." % self.request_id)
logger.debug("%s: After postprocessing: %s" % (self.request_id, full_result))
-
try:
self.send(json.dumps(full_result))
except: