Merge pull request #11 from krillinai/feat-aitool

add aliyun support

Author: maranello-o

config/config-example.toml

@@ -1,7 +1,9 @@
 [app]
     segment_duration = 5 # 音频分割间隔，单位：分钟
     translate_parallel_num = 5 # 并发进行大模型翻译的数量上限
-    proxy = "" # 网络代理
+    proxy = "" # 网络代理地址，格式如http://127.0.0.1:7890
+    transcribe_provider = "openai" # 语音识别供应商，当前可选值：openai,aliyun。强烈建议openai，选择后需要在下方配置供应商密钥等信息，下同
+    llm_provider = "openai" # LLM供应商，当前可选值：openai
 
 [server]
     host = "127.0.0.1"
@@ -10,8 +12,10 @@
 [openai]
     api_key = "sk-XXX" # OpenAI API密钥
 
-[aliyun] # 此处配置可以先忽略
-    access_key_id = ""
-    access_key_secret = ""
-    app_key = ""
-    cosy_voice_ws_addr = ""
+[aliyun]
+    [aliyun.tts] # tts配置可以先忽略
+        access_key_id = ""
+        access_key_secret = ""
+        app_key= ""
+    [aliyun.asr]
+        api_key = ""

config/config.go

@@ -10,6 +10,8 @@ type App struct {
 	TranslateParallelNum int    `toml:"translate_parallel_num"`
 	Proxy                string `toml:"proxy"`
 	ParsedProxy          *url.URL
+	TranscribeProvider   string `toml:"transcribe_provider"`
+	LlmProvider          string `toml:"llm_provider"`
 }
 
 type Server struct {
@@ -21,11 +23,19 @@ type Openai struct {
 	ApiKey string `toml:"api_key"`
 }
 
-type Aliyun struct {
+type AliyunTts struct {
 	AccessKeyId     string `toml:"access_key_id"`
 	AccessKeySecret string `toml:"access_key_secret"`
 	AppKey          string `toml:"app_key"`
-	CosyVoiceWsAddr string `toml:"cosy_voice_ws_addr"`
+}
+
+type AliyunBailian struct {
+	ApiKey string `toml:"api_key"`
+}
+
+type Aliyun struct {
+	Tts     AliyunTts     `toml:"tts"`
+	Bailian AliyunBailian `toml:"bailian"`
 }
 
 type Config struct {

go.mod

@@ -7,9 +7,10 @@ toolchain go1.22.10
 require (
 	github.com/BurntSushi/toml v1.3.2
 	github.com/aliyun/alibaba-cloud-sdk-go v1.63.72
+	github.com/aliyun/alibabacloud-bailian-go-sdk v1.2.0
 	github.com/gin-gonic/gin v1.10.0
 	github.com/go-resty/resty/v2 v2.7.0
-	github.com/google/uuid v1.3.0
+	github.com/google/uuid v1.4.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/samber/lo v1.38.1
 	github.com/sashabaranov/go-openai v1.36.0
@@ -18,8 +19,20 @@ require (
 )
 
 require (
+	github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 // indirect
+	github.com/alibabacloud-go/bailian-20230601 v1.1.0 // indirect
+	github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.2 // indirect
+	github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 // indirect
+	github.com/alibabacloud-go/endpoint-util v1.1.0 // indirect
+	github.com/alibabacloud-go/openapi-util v0.1.0 // indirect
+	github.com/alibabacloud-go/tea v1.1.19 // indirect
+	github.com/alibabacloud-go/tea-utils v1.3.1 // indirect
+	github.com/alibabacloud-go/tea-utils/v2 v2.0.4 // indirect
+	github.com/alibabacloud-go/tea-xml v1.1.2 // indirect
+	github.com/aliyun/credentials-go v1.1.2 // indirect
 	github.com/bytedance/sonic v1.11.6 // indirect
 	github.com/bytedance/sonic/loader v0.1.1 // indirect
+	github.com/clbanning/mxj/v2 v2.5.5 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
@@ -39,6 +52,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/tjfoc/gmsm v1.3.2 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	go.uber.org/atomic v1.10.0 // indirect

go.sum

@@ -1,19 +1,47 @@
 package service
 
 import (
+	"go.uber.org/zap"
 	"krillin-ai/config"
+	"krillin-ai/internal/types"
+	"krillin-ai/log"
 	"krillin-ai/pkg/aliyun"
 	"krillin-ai/pkg/openai"
 )
 
 type Service struct {
 	OpenaiClient    *openai.Client
-	CosyCloneClient *aliyun.CosyCloneClient
+	CosyCloneClient *aliyun.Client
+
+	Transcriber   types.Transcriber
+	ChatCompleter types.ChatCompleter
 }
 
 func NewService() *Service {
+	var transcriber types.Transcriber
+	var chatCompleter types.ChatCompleter
+
+	switch config.Conf.App.TranscribeProvider {
+	case "openai":
+		transcriber = openai.NewClient(config.Conf.Openai.ApiKey, config.Conf.App.ParsedProxy)
+	case "aliyun":
+		transcriber = aliyun.NewClient()
+	}
+	log.GetLogger().Info("当前选择的转录源： ", zap.String("transcriber", config.Conf.App.TranscribeProvider))
+
+	switch config.Conf.App.LlmProvider {
+	case "openai":
+		chatCompleter = openai.NewClient(config.Conf.Openai.ApiKey, config.Conf.App.ParsedProxy)
+	case "aliyun":
+		chatCompleter = aliyun.NewChatClient(config.Conf.Aliyun.Bailian.ApiKey)
+	}
+	log.GetLogger().Info("当前选择的LLM源： ", zap.String("llm", config.Conf.App.LlmProvider))
+
 	return &Service{
 		OpenaiClient:    openai.NewClient(config.Conf.Openai.ApiKey, config.Conf.App.ParsedProxy),
-		CosyCloneClient: aliyun.NewCosyCloneClient(),
+		CosyCloneClient: aliyun.NewClient(),
+
+		Transcriber:   transcriber,
+		ChatCompleter: chatCompleter,
 	}
 }

internal/service/init.go

@@ -14,7 +14,6 @@ import (
 	"krillin-ai/internal/storage"
 	"krillin-ai/internal/types"
 	"krillin-ai/log"
-	"krillin-ai/pkg/openai"
 	"krillin-ai/pkg/util"
 	"os"
 	"os/exec"
@@ -188,7 +187,7 @@ func (s Service) GetTaskStatus(req dto.GetVideoSubtitleTaskReq) (*dto.GetVideoSu
 	}, nil
 }
 
-// 新版流程：链接->本地音频文件->扣费->视频信息获取（若有）->本地字幕文件->cos上的字幕信息
+// 新版流程：链接->本地音频文件->视频信息获取（若有）->本地字幕文件->cos上的字幕信息
 
 func (s Service) linkToAudioFile(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {
 	var (
@@ -439,13 +438,13 @@ func (s Service) audioToSrt(ctx context.Context, stepParam *types.SubtitleTaskSt
 				<-parallelControlChan
 			}()
 			// 语音转文字
-			var transcriptionData *openai.TranscriptionData
+			var transcriptionData *types.TranscriptionData
 			for i := 0; i < 3; i++ {
 				language := string(stepParam.OriginLanguage)
 				if language == "zh_cn" {
 					language = "zh" // 切换一下
 				}
-				transcriptionData, err = s.OpenaiClient.Transcription(audioFile.AudioFile, language)
+				transcriptionData, err = s.Transcriber.Transcription(audioFile.AudioFile, language)
 				if err == nil {
 					break
 				}
@@ -629,7 +628,7 @@ func (s Service) splitSrt(ctx context.Context, stepParam *types.SubtitleTaskStep
 	return nil
 }
 
-func getSentenceTimestamps(words []openai.Word, sentence string, lastTs float64, language types.StandardLanguageName) (types.SrtSentence, float64, error) {
+func getSentenceTimestamps(words []types.Word, sentence string, lastTs float64, language types.StandardLanguageName) (types.SrtSentence, float64, error) {
 	var srtSt types.SrtSentence
 	var sentenceWordList []string
 	if language == types.LanguageNameEnglish || language == types.LanguageNameGerman { // 处理方式不同
@@ -638,7 +637,7 @@ func getSentenceTimestamps(words []openai.Word, sentence string, lastTs float64,
 			return srtSt, 0, fmt.Errorf("sentence is empty")
 		}
 
-		sentenceWords := make([]openai.Word, 0)
+		sentenceWords := make([]types.Word, 0)
 
 		thisLastTs := lastTs
 		sentenceWordIndex := 0
@@ -663,7 +662,7 @@ func getSentenceTimestamps(words []openai.Word, sentence string, lastTs float64,
 			}
 
 			if sentenceWordIndex >= len(words) {
-				sentenceWords = append(sentenceWords, openai.Word{
+				sentenceWords = append(sentenceWords, types.Word{
 					Text: sentenceWord,
 				})
 				sentenceWordIndex = 0
@@ -726,7 +725,7 @@ func getSentenceTimestamps(words []openai.Word, sentence string, lastTs float64,
 			return srtSt, 0, fmt.Errorf("sentence is empty")
 		}
 
-		sentenceWords := make([]openai.Word, 0)
+		sentenceWords := make([]types.Word, 0)
 
 		thisLastTs := lastTs
 		sentenceWordIndex := 0
@@ -768,7 +767,7 @@ func getSentenceTimestamps(words []openai.Word, sentence string, lastTs float64,
 }
 
 // 找到 Num 值递增的最大连续子数组
-func findMaxIncreasingSubArray(words []openai.Word) (int, int) {
+func findMaxIncreasingSubArray(words []types.Word) (int, int) {
 	if len(words) == 0 {
 		return 0, 0
 	}
@@ -805,7 +804,7 @@ func findMaxIncreasingSubArray(words []openai.Word) (int, int) {
 }
 
 // 跳跃（非连续）找到 Num 值递增的最大子数组
-func jumpFindMaxIncreasingSubArray(words []openai.Word) (int, int) {
+func jumpFindMaxIncreasingSubArray(words []types.Word) (int, int) {
 	if len(words) == 0 {
 		return -1, -1
 	}
@@ -923,7 +922,7 @@ func (s Service) splitTextAndTranslate(taskId, baseTaskPath string, targetLangua
 		splitContent = ""
 	} else {
 		for i := 0; i < 3; i++ {
-			splitContent, err = s.OpenaiClient.ChatCompletion(splitPrompt + audioFile.TranscriptionData.Text)
+			splitContent, err = s.ChatCompleter.ChatCompletion(splitPrompt + audioFile.TranscriptionData.Text)
 			if err == nil {
 				break
 			}

internal/service/subtitle_service.go

@@ -0,0 +1,9 @@
+package types
+
+type ChatCompleter interface {
+	ChatCompletion(query string) (string, error)
+}
+
+type Transcriber interface {
+	Transcription(audioFile, language string) (*TranscriptionData, error)
+}

internal/types/interface.go

@@ -1,9 +1,5 @@
 package types
 
-import (
-	"krillin-ai/pkg/openai"
-)
-
 var SplitTextPrompt = `你是一个语言处理专家，擅长翻译和处理文本，按下面的要求，根据句意和标点对给出的内容进行拆分并翻译：
 
 - 将原句子翻译成 %s
@@ -76,7 +72,7 @@ var TranslateVideoTitleAndDescriptionPrompt = `你是一个专业的翻译专家
 type SmallAudio struct {
 	AudioFile         string
 	Num               int
-	TranscriptionData *openai.TranscriptionData
+	TranscriptionData *TranscriptionData
 	SrtNoTsFile       string
 }
 
@@ -140,6 +136,10 @@ const (
 	TtsResultAudioFileName          = "tts_final_audio.wav"
 )
 
+const (
+	AsrMono16kAudioFileName = "mono_16k_audio.mp3"
+)
+
 type StandardLanguageName string
 
 const (
@@ -269,8 +269,20 @@ type SubtitleTask struct {
 	SrtNum                int            `json:"srt_num" gorm:"column:srt_num"`                               // 字幕数量
 	SubtitleInfos         []SubtitleInfo `gorm:"foreignKey:TaskId;references:TaskId"`
 	Cover                 string         `json:"cover" gorm:"column:cover"`                             // 封面
-	Cost                  uint32         `json:"cost" gorm:"column:cost"`                               // 消耗的额度
 	SpeechDownloadUrl     string         `json:"speech_download_url" gorm:"column:speech_download_url"` // 语音文件下载地址
 	CreateTime            int64          `json:"create_time" gorm:"column:create_time;autoCreateTime"`  // 创建时间
 	UpdateTime            int64          `json:"update_time" gorm:"column:update_time;autoUpdateTime"`  // 更新时间
 }
+
+type Word struct {
+	Num   int
+	Text  string
+	Start float64
+	End   float64
+}
+
+type TranscriptionData struct {
+	Language string
+	Text     string
+	Words    []Word
+}

internal/types/subtitle_task.go

@@ -35,6 +35,7 @@ func main() {
 	err = util.CheckAndDownloadFfprobe()
 	if err != nil {
 		log.GetLogger().Error("ffprobe环境准备失败", zap.Error(err))
+		return
 	}
 	err = util.CheckAndDownloadYtDlp()
 	if err != nil {

main.go

@@ -0,0 +1,46 @@
+package aliyun
+
+import (
+	"context"
+	goopenai "github.com/sashabaranov/go-openai"
+	"go.uber.org/zap"
+	"krillin-ai/log"
+)
+
+type ChatClient struct {
+	*goopenai.Client
+}
+
+func NewChatClient(apiKey string) *ChatClient {
+	cfg := goopenai.DefaultConfig(apiKey)
+	cfg.BaseURL = "https://dashscope.aliyuncs.com/compatible-mode/v1" // 使用阿里云的openai兼容模式调用
+	return &ChatClient{
+		Client: goopenai.NewClientWithConfig(cfg),
+	}
+}
+
+func (c ChatClient) ChatCompletion(query string) (string, error) {
+	req := goopenai.ChatCompletionRequest{
+		Model: "qwen-plus",
+		Messages: []goopenai.ChatCompletionMessage{
+			{
+				Role:    goopenai.ChatMessageRoleSystem,
+				Content: "You are an assistant that helps with subtitle translation.",
+			},
+			{
+				Role:    goopenai.ChatMessageRoleUser,
+				Content: query,
+			},
+		},
+	}
+
+	resp, err := c.CreateChatCompletion(context.Background(), req)
+	if err != nil {
+		log.GetLogger().Error("aliyun openai create chat completion failed", zap.Error(err))
+		return "", err
+	}
+
+	resContent := resp.Choices[0].Message.Content
+
+	return resContent, nil
+}