Skip to content

Commit cbbde0b

Browse files
solderzzcsolderzzc
committed
disable realtime message, switched to aws s3 as storage
1 parent e16d7e5 commit cbbde0b

File tree

5 files changed

+216
-105
lines changed

5 files changed

+216
-105
lines changed

‎src/camera/detector/deepeye.js

+62-12
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,11 @@
11
var fs = require('fs');
2-
var http = require('http');
3-
var qs=require('querystring');
4-
var path = require('path');
52
var request = require('request');
63
var celery = require('node-celery');
74
var requestretry = require('requestretry')
5+
const https = require('https');
6+
const url = require('url');
87

98
var async = require('async')
10-
var face_motion = require('./face_motions')
11-
//var maintainer = require('./maintainer')
12-
var mqtt_2_group = require('./mqttgif')
13-
var timeline = require('./timeline')
149
var upload = require('./upload')
1510
var device_SN = null
1611

@@ -32,6 +27,12 @@ var REDIS_PORT = process.env.REDIS_PORT || 6379
3227
var CLUSTER_REDIS_ADDRESS = process.env.CLUSTER_REDIS_ADDRESS || "redis"
3328
var CLUSTER_REDIS_PORT = process.env.CLUSTER_REDIS_PORT || 6379
3429

30+
const SHARP_AI_TOKEN=process.env.SHARP_AI_TOKEN
31+
const SHARP_AI_USER_ID=process.env.SHARP_AI_USER_ID
32+
const SHARP_AI_USERNAME=process.env.SHARP_AI_USERNAME
33+
const API_SERVER_ADDRESS=process.env.API_SERVER_ADDRESS
34+
const API_SERVER_PORT=process.env.API_SERVER_PORT
35+
3536
function GetEnvironmentVarInt(varname, defaultvalue)
3637
{
3738
var result = process.env[varname];
@@ -461,6 +462,22 @@ function post_recognition_result_to_api_server(json,accessUrl){
461462
}
462463
});
463464
}
465+
466+
function check_quote_and_sign_url(key,cb){
467+
const options = {
468+
method: 'POST',
469+
url: 'http://'+API_SERVER_ADDRESS+':'+API_SERVER_PORT+'/api/v1/quote/upload',
470+
headers: {
471+
'X-Auth-Token': SHARP_AI_TOKEN,
472+
'X-User-Id':SHARP_AI_USER_ID,
473+
'Content-type': 'application/json'
474+
},
475+
json: true,
476+
body:{key:key}
477+
};
478+
request(options, cb);
479+
}
480+
464481
function classify_task(task_info, cb) {
465482
if(connected_to_celery_broker){
466483
client.call('classify.classify',
@@ -483,12 +500,45 @@ function classify_task(task_info, cb) {
483500
json.result['url'] = task_info.url
484501
post_recognition_result_to_api_server(json,task_info.url)
485502
} else {
486-
json.result['url'] = upload.getAccessUrl(json.result.key)+'.png'
487-
upload.putFile(json.result.key+'.png',task_info.path,function(error,accessUrl){
488-
ON_DEBUG && console.log('error=',error,'accessUrl=',accessUrl)
489-
if(!error){
490-
post_recognition_result_to_api_server(json,accessUrl)
503+
check_quote_and_sign_url(json.result.key+'.png',function(error, response, body){
504+
if (!error && response.statusCode == 200) {
505+
//const info = JSON.parse(body);
506+
ON_DEBUG && console.log('server response: '+ body);
507+
ON_DEBUG && console.log(body);
508+
const info = JSON.parse(body);
509+
ON_DEBUG && console.log(info);
510+
511+
var presigned=info.presigned;
512+
var filename = task_info.path;
513+
var stats = fs.statSync(filename);
514+
var file_size = stats.size;
515+
var dataStream = fs.createReadStream(filename);
516+
var options = url.parse(presigned);
517+
options.method = 'PUT';
518+
options.headers = {
519+
'Content-Length' : file_size,
520+
'Content-Type' : 'image/png'
521+
}
522+
523+
console.log(options)
524+
const stream = https.request(options);
525+
stream.on('response', (res) => {
526+
console.log(res.statusCode);
527+
res.pipe(process.stdout);
528+
529+
console.log('uploaded to ' + info.url)
530+
post_recognition_result_to_api_server(json,info.url)
531+
});
532+
533+
dataStream.pipe(stream);
491534
}
535+
/*json.result['url'] = upload.getAccessUrl(json.result.key)+'.png'
536+
upload.putFile(json.result.key+'.png',task_info.path,function(error,accessUrl){
537+
ON_DEBUG && console.log('error=',error,'accessUrl=',accessUrl)
538+
if(!error){
539+
post_recognition_result_to_api_server(json,accessUrl)
540+
}
541+
})*/
492542
})
493543
}
494544
delete json.result.key

‎src/camera/detector/face_motions.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -392,7 +392,7 @@ function insert2timelineDB(files, tracker_id, uuid, group_id, totalpeople, recog
392392
if(err)
393393
console.log(err)
394394

395-
timeline.send_realtime_msg(function(err){})
395+
//timeline.send_realtime_msg(function(err){})
396396
})
397397

398398
return cb && cb(null, null, 0);

‎src/camera/detector/package-lock.json

+11-6
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/camera/detector/upload.js

+3-86
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
1-
//const qiniu = require("qiniu");
21
const proc = require("process");
32
var fs = require('fs');
4-
const Minio=require('minio')
53

64
function GetEnvironmentVarInt(varname, defaultvalue)
75
{
@@ -12,63 +10,7 @@ function GetEnvironmentVarInt(varname, defaultvalue)
1210
return defaultvalue;
1311
}
1412

15-
const AWS_END_POINT = process.env.AWS_END_POINT
16-
const AWS_PORT = GetEnvironmentVarInt('AWS_PORT',80)
17-
const AWS_USE_SSL = (process.env.AWS_USE_SSL === "true")
18-
const AWS_ACCESS_KEY = process.env.AWS_ACCESS_KEY
19-
const AWS_SECRET_KEY = process.env.AWS_SECRET_KEY
20-
const AWS_BUCKET = process.env.AWS_BUCKET
21-
const AWS_READABLE_PREFIX=process.env.AWS_READABLE_PREFIX
2213

23-
var mc = new Minio.Client({
24-
endPoint: AWS_END_POINT,
25-
port: AWS_PORT,
26-
useSSL: AWS_USE_SSL,
27-
accessKey: AWS_ACCESS_KEY,
28-
secretKey: AWS_SECRET_KEY
29-
});
30-
31-
function ensureSharpAIBucketPublicReadalbe(cb){
32-
const BucketReadablePolicy={
33-
"Version":"2012-10-17",
34-
"Statement":[
35-
{
36-
"Sid":"PublicRead",
37-
"Effect":"Allow",
38-
"Principal": "*",
39-
"Action":["s3:GetObject","s3:GetObjectVersion"],
40-
"Resource":["arn:aws:s3:::"+AWS_BUCKET+"/*"]
41-
}
42-
]
43-
}
44-
mc.setBucketPolicy(AWS_BUCKET, JSON.stringify(BucketReadablePolicy), function(err) {
45-
if (err) throw err
46-
console.log('Bucket policy set')
47-
})
48-
}
49-
function ensureSharpAIBucket(cb){
50-
mc.bucketExists(AWS_BUCKET, function(err, exists) {
51-
if (err || !exists) {
52-
console.log('storage bucket does not exist')
53-
console.log(err)
54-
mc.makeBucket(AWS_BUCKET, function(err) {
55-
if (err) {
56-
cb && cb(err)
57-
return console.log('Error creating bucket.', err)
58-
}
59-
console.log('Bucket created successfully')
60-
cb && cb(null)
61-
ensureSharpAIBucketPublicReadalbe()
62-
})
63-
return
64-
}
65-
if (exists) {
66-
cb && cb(null)
67-
ensureSharpAIBucketPublicReadalbe()
68-
return console.log('Bucket exists.')
69-
}
70-
})
71-
}
7214

7315
function getAccessUrl(file_key){
7416
return AWS_READABLE_PREFIX+file_key;
@@ -78,20 +20,20 @@ function putFileAWS(file_key,localFile,cb){
7820
if(!err && url){
7921
cb && cb(null,url)
8022
} else {
81-
console.log('1st upload err, retry after 5s');
23+
console.log('1st upload failed, retry after 5s');
8224
setTimeout(function(){
8325
_putFileAWS(file_key,localFile,function(err, url){
8426
if(!err && url){
8527
cb && cb(null,url)
8628
} else {
87-
console.log('2nd upload err, retry after 15s');
29+
console.log('2nd upload failed, retry after 15s');
8830
setTimeout(function(){
8931
_putFileAWS(file_key,localFile,function(err, url){
9032
if(!err && url){
9133
cb && cb(null,url)
9234
} else {
9335

94-
console.log('3rd upload err, gave up');
36+
console.log('3rd upload failed, no more retry');
9537
cb && cb('error',null)
9638
}
9739
})
@@ -105,32 +47,7 @@ function putFileAWS(file_key,localFile,cb){
10547

10648
//var obj = {counter:0}
10749
function _putFileAWS(file_key,localFile,cb){
108-
//if(obj.counter++ %2 === 0){ return cb('error',null)} test code for failed upload
109-
var read = fs.createReadStream(localFile);
11050

111-
try{
112-
mc.putObject(AWS_BUCKET,file_key,read,function(err, etag){
113-
if(err){
114-
console.log('upload errr, need retry')
115-
if(cb){
116-
cb(err,null)
117-
}
118-
return
119-
}
120-
var access_url = getAccessUrl(file_key);
121-
console.log('upload succ to: '+access_url);
122-
if(cb){
123-
cb(null,access_url)
124-
}
125-
})
126-
} catch(e){
127-
console.log(e)
128-
console.log('exception when put object to aws')
129-
ensureSharpAIBucket(function(result){
130-
console.log('after ensureSharpAIBucket')
131-
console.log(result)
132-
})
133-
}
13451
}
13552

13653
module.exports = {

0 commit comments

Comments
 (0)