Skip to content

Commit

Permalink
add validate api
Browse files Browse the repository at this point in the history
  • Loading branch information
@sharkpc138
sharkpc138 committed Jan 14, 2025
1 parent f0b5ca1 commit 71c9861
Show file tree
Hide file tree
Showing 20 changed files with 649 additions and 88 deletions.
95 changes: 93 additions & 2 deletions pkg/docs/operator/docs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ const docTemplate = `{
"required": true
},
{
"description": "sink contentd; Each content in array must be unique",
"description": "sink contents; All contents in the array must be unique",
"name": "sink",
"in": "body",
"required": true,
Expand All @@ -119,7 +119,10 @@ const docTemplate = `{
"400": {
"description": "Invalid parameters",
"schema": {
"type": "string"
"type": "array",
"items": {
"$ref": "#/definitions/v1.ValidationError"
}
}
},
"405": {
Expand Down Expand Up @@ -205,6 +208,83 @@ const docTemplate = `{
}
}
}
},
"/api/v1/namespaces/{namespace}/sinks/{name}/validate": {
"post": {
"consumes": [
"application/json"
],
"tags": [
"Post"
],
"summary": "Validate log sink",
"parameters": [
{
"type": "string",
"description": "namespace name",
"name": "namespace",
"in": "path",
"required": true
},
{
"type": "string",
"description": "sink name",
"name": "name",
"in": "path",
"required": true
},
{
"description": "sink contents; All contents in the array must be unique",
"name": "sink",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/v1.Sink"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"201": {
"description": "Created successfully",
"schema": {
"type": "string"
}
},
"400": {
"description": "Invalid parameters",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/v1.ValidationError"
}
}
},
"405": {
"description": "Method not allowed",
"schema": {
"type": "string"
}
},
"422": {
"description": "Restricted by limits",
"schema": {
"type": "string"
}
},
"500": {
"description": "Failed to get sink content",
"schema": {
"type": "string"
}
}
}
}
}
},
"definitions": {
Expand Down Expand Up @@ -579,6 +659,17 @@ const docTemplate = `{
"additionalProperties": {
"type": "string"
}
},
"v1.ValidationError": {
"type": "object",
"properties": {
"field": {
"type": "string"
},
"message": {
"type": "string"
}
}
}
}
}`
Expand Down
95 changes: 93 additions & 2 deletions pkg/docs/operator/swagger.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
"required": true
},
{
"description": "sink contentd; Each content in array must be unique",
"description": "sink contents; All contents in the array must be unique",
"name": "sink",
"in": "body",
"required": true,
Expand All @@ -111,7 +111,10 @@
"400": {
"description": "Invalid parameters",
"schema": {
"type": "string"
"type": "array",
"items": {
"$ref": "#/definitions/v1.ValidationError"
}
}
},
"405": {
Expand Down Expand Up @@ -197,6 +200,83 @@
}
}
}
},
"/api/v1/namespaces/{namespace}/sinks/{name}/validate": {
"post": {
"consumes": [
"application/json"
],
"tags": [
"Post"
],
"summary": "Validate log sink",
"parameters": [
{
"type": "string",
"description": "namespace name",
"name": "namespace",
"in": "path",
"required": true
},
{
"type": "string",
"description": "sink name",
"name": "name",
"in": "path",
"required": true
},
{
"description": "sink contents; All contents in the array must be unique",
"name": "sink",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/v1.Sink"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "string"
}
},
"201": {
"description": "Created successfully",
"schema": {
"type": "string"
}
},
"400": {
"description": "Invalid parameters",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/v1.ValidationError"
}
}
},
"405": {
"description": "Method not allowed",
"schema": {
"type": "string"
}
},
"422": {
"description": "Restricted by limits",
"schema": {
"type": "string"
}
},
"500": {
"description": "Failed to get sink content",
"schema": {
"type": "string"
}
}
}
}
}
},
"definitions": {
Expand Down Expand Up @@ -571,6 +651,17 @@
"additionalProperties": {
"type": "string"
}
},
"v1.ValidationError": {
"type": "object",
"properties": {
"field": {
"type": "string"
},
"message": {
"type": "string"
}
}
}
}
}
64 changes: 62 additions & 2 deletions pkg/docs/operator/swagger.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,13 @@ definitions:
additionalProperties:
type: string
type: object
v1.ValidationError:
properties:
field:
type: string
message:
type: string
type: object
info:
contact: {}
description: Descriptions of Lobster log-sink management APIs
Expand Down Expand Up @@ -311,7 +318,7 @@ paths:
name: name
required: true
type: string
- description: sink contentd; Each content in array must be unique
- description: sink contents; All contents in the array must be unique
in: body
name: sink
required: true
Expand All @@ -329,7 +336,9 @@ paths:
"400":
description: Invalid parameters
schema:
type: string
items:
$ref: '#/definitions/v1.ValidationError'
type: array
"405":
description: Method not allowed
schema:
Expand Down Expand Up @@ -387,4 +396,55 @@ paths:
summary: Delete sink
tags:
- Delete
/api/v1/namespaces/{namespace}/sinks/{name}/validate:
post:
consumes:
- application/json
parameters:
- description: namespace name
in: path
name: namespace
required: true
type: string
- description: sink name
in: path
name: name
required: true
type: string
- description: sink contents; All contents in the array must be unique
in: body
name: sink
required: true
schema:
$ref: '#/definitions/v1.Sink'
responses:
"200":
description: OK
schema:
type: string
"201":
description: Created successfully
schema:
type: string
"400":
description: Invalid parameters
schema:
items:
$ref: '#/definitions/v1.ValidationError'
type: array
"405":
description: Method not allowed
schema:
type: string
"422":
description: Restricted by limits
schema:
type: string
"500":
description: Failed to get sink content
schema:
type: string
summary: Validate log sink
tags:
- Post
swagger: "2.0"
4 changes: 2 additions & 2 deletions pkg/lobster/sink/exporter/exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,8 @@ func (e *LogExporter) Run(stopChan chan struct{}) {
continue
}

if err := uploader.Validate(); err != nil {
glog.Error(err)
if errList := uploader.Validate(); !errList.IsEmpty() {
glog.Error(errList.String())
metrics.AddSinkFailure(order.Request, order.SinkNamespace, order.SinkName, uploader.Type(), uploader.Name())
continue
}
Expand Down
9 changes: 7 additions & 2 deletions pkg/lobster/sink/exporter/uploader/basic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/golang/glog"
"github.com/naver/lobster/pkg/lobster/model"
"github.com/naver/lobster/pkg/lobster/sink/order"
v1 "github.com/naver/lobster/pkg/operator/api/v1"
"github.com/naver/lobster/pkg/operator/api/v1/template"
)

Expand Down Expand Up @@ -95,7 +96,7 @@ func (b BasicUploader) FileName(start, end time.Time) string {
return fileName
}

func (b BasicUploader) Validate() error {
func (b BasicUploader) Validate() v1.ValidationErrors {
return b.Order.LogExportRule.BasicBucket.Validate()
}

Expand Down Expand Up @@ -155,16 +156,20 @@ func (b BasicUploader) Upload(data []byte, dir, fileName string) error {
func (b BasicUploader) defaultDir(chunk model.Chunk, date time.Time) string {
dirPath := b.Order.Path()
layout := b.Order.LogExportRule.BasicBucket.TimeLayoutOfSubDirectory
rootPath := b.Order.LogExportRule.BasicBucket.RootPath

if len(chunk.Source.Path) > 0 {
dirPath = fmt.Sprintf("%s/%s", dirPath, chunk.Source.Path)
}
if len(layout) == 0 {
layout = defaultLayout
}
if len(rootPath) == 0 {
rootPath = "/"
}

return fmt.Sprintf("%s/%s/%s",
b.Order.LogExportRule.BasicBucket.RootPath,
rootPath,
date.Format(layout),
dirPath)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/lobster/sink/exporter/uploader/kafka.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func (k KafkaUploader) FileName(start, end time.Time) string {
return ""
}

func (k KafkaUploader) Validate() error {
func (k KafkaUploader) Validate() v1.ValidationErrors {
return k.Order.LogExportRule.Kafka.Validate()
}

Expand Down
Loading

0 comments on commit 71c9861

Please sign in to comment.