chore: Audio Handle

kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/controller/ChatGptController.java

@@ -46,8 +46,25 @@ public class ChatGptController {
 
 	@Inner(false)
 	@PostMapping("/chat/completions")
-	public R completions(@Valid @ModelAttribute ImContent imContent, @RequestPart(value = "file", required = false) MultipartFile file) {
+	public R chatCompletions(@Valid @RequestBody ImContent imContent) {
-        return R.ok(iImContentService.completions(imContent, file));
+		SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
+        return R.ok(iImContentService.chatCompletions(imContent));
+	}
+
+
+	@Inner(false)
+	@PostMapping("/audio/transcriptions")
+	public R audioTranscriptions(@Valid @ModelAttribute ImContent imContent, @RequestPart(value = "file", required = false) MultipartFile file) {
+		SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
+		return R.ok(iImContentService.audioTranscriptions(imContent, file));
+	}
+
+
+	@Inner(false)
+	@PostMapping("/audio/speech")
+	public R audioSpeech(@Valid @RequestBody ImContent imContent) {
+		SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
+		return R.ok(iImContentService.audioSpeech(imContent));
 	}
 
 }

kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/service/IImContentService.java

@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.baomidou.mybatisplus.extension.service.IService;
 import com.cloud.kicc.system.api.entity.ImContent;
+import com.cloud.kicc.system.api.entity.OssFile;
 import org.springframework.web.multipart.MultipartFile;
 
 import java.util.Map;
@@ -18,8 +19,32 @@ import java.util.Map;
  */
 public interface IImContentService extends IService<ImContent> {
 
+    /**
+     * 聊天记录
+     * @param
+     * @return
+     */
     IPage<Map<String, Object>> listHistoryMessage(Page page, ImContent imContent);
 
-    Map<String, Object> completions(ImContent imContent, MultipartFile file);
+    /**
+     * 聊天
+     * @param
+     * @return
+     */
+    ImContent chatCompletions(ImContent imContent);
+
+    /**
+     * 语音转文字
+     * @param imContent
+     * @return OssFile
+     */
+    ImContent audioTranscriptions(ImContent imContent, MultipartFile file);
+
+    /**
+     * 文字转语音
+     * @param imContent
+     * @return OssFile
+     */
+    OssFile audioSpeech(ImContent imContent);
 
 }

kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/service/impl/ImContentServiceImpl.java

@@ -72,15 +72,71 @@ public class ImContentServiceImpl extends ServiceImpl<ImContentMapper, ImContent
 
     @Override
     @Transactional(rollbackFor = Exception.class)
+    public ImContent chatCompletions(ImContent imContent) {
+        if (StrUtil.isBlank(imContent.getContent())) {
+            throw new CheckedException("提问内容不能为空请检查!");
+        }
+        // 保存用户发送聊天记录
+        imContent.setSendUserId(SecurityUtils.getCasUser().getId());
+        imContent.setSendTime(LocalDateTime.now());
+        imContent.setReceiveUserId(imContent.getReceiveUserId());
+        imContent.setReceiveTime(LocalDateTime.now());
+        super.save(imContent);
+
+        // 查询当前历史消息,进行AI上下文消息合并
+        List<ImContent> imContents = super.list(Wrappers.<ImContent>lambdaQuery().and(i ->
+                i.eq(ImContent::getSendUserId, imContent.getSendUserId())
+                 .eq(ImContent::getReceiveUserId, imContent.getReceiveUserId())
+        ).or(i ->
+                i.eq(ImContent::getSendUserId, imContent.getReceiveUserId())
+                 .eq(ImContent::getReceiveUserId, imContent.getSendUserId())
+        ) .orderByAsc(ImContent::getSendTime));
+
+        // 历史上下文消息
+        List<ChatMessage> historyMessages = imContents.stream().map(item -> {
+            ChatMessage chatMessage = new ChatMessage();
+            chatMessage.setRole(StrUtil.equals(item.getSendUserId(), imContent.getSendUserId()) ? ChatMessageRole.USER.value() : ChatMessageRole.ASSISTANT.value());
+            chatMessage.setContent(item.getContent());
+            return chatMessage;
+        }).collect(Collectors.toList());
+
+        // Completion消息
+        ChatMessage completionMessage = new ChatMessage(ChatMessageRole.USER.value(), imContent.getContent());
+        historyMessages.add(completionMessage);
+
+        ChatCompletionResult chatCompletionResult = AiUtil.getChatCompletion(historyMessages);
+        List<ChatCompletionChoice> choiceList = chatCompletionResult.getChoices();
+
+        if (choiceList.isEmpty()) {
+            throw new CheckedException("Ai生成内容解析失败!");
+        }
+
+        // ai chat message build
+        ImContent receiveContent = new ImContent()
+                .setSendUserId(imContent.getReceiveUserId())
+                .setSendTime(LocalDateTime.now())
+                .setReceiveUserId(SecurityUtils.getCasUser().getId())
+                .setReceiveTime(LocalDateTime.now())
+                .setContent(choiceList.get(0).getMessage().getContent())
+                .setContentType(imContent.getContentType());
+
+        super.save(receiveContent);
+        return receiveContent;
+    }
+
+    @Override
     @SneakyThrows
-    public Map<String, Object> completions(ImContent imContent, MultipartFile file) {
+    @Transactional(rollbackFor = Exception.class)
-        SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
+    public ImContent audioTranscriptions(ImContent imContent, MultipartFile file) {
-        if (imContent.getContentType().equals(ImMessageTypeEnum.AUDIO.getValue())) {
         Optional.ofNullable(file).orElseThrow(() -> new CheckedException("当前语音文件为空,请检查后重试!"));
         OpenAiService openAiService = AiUtil.getOpenAiService();
-            String speechName = "speech.mp3";
         String sendFileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(file.getOriginalFilename());
-            String receiveFileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(speechName);
+        File inputVoiceFile = File.createTempFile(FileUtil.getPrefix(file.getOriginalFilename()), StrUtil.DOT + FileUtil.extName(file.getOriginalFilename()));
+        try {
+            // 获取输入音频时长
+            FileUtil.writeBytes(file.getBytes(), inputVoiceFile);
+            AudioFile inputAudioFile = AudioFileIO.read(inputVoiceFile);
+            long inputVoiceDuration = inputAudioFile.getAudioHeader().getTrackLength();
 
             // 构建发送文件信息进行OSS存储
             OssFile sendOssOssFile = new OssFile()
@@ -88,37 +144,46 @@ public class ImContentServiceImpl extends ServiceImpl<ImContentMapper, ImContent
                     .setBucketName(ossProperties.getBucketName())
                     .setOriginal(file.getOriginalFilename())
                     .setType(FileUtil.extName(file.getOriginalFilename()))
+                    .setDuration(inputVoiceDuration)
                     .setFileSize(file.getSize())
                     .setMimeType(file.getContentType());
             ossTemplate.putObject(ossProperties.getBucketName(), sendFileName, file.getContentType(), file.getInputStream());
             String sendVoiceUrl = ossTemplate.getObjectURL(ossProperties.getBucketName(), sendFileName);
             sendOssOssFile.setAvailablePath(sendVoiceUrl);
             fileService.save(sendOssOssFile);
-            imContent.setFiles(sendOssOssFile.getId());
-
-            File inputVoiceFile = File.createTempFile(FileUtil.getPrefix(file.getOriginalFilename()), StrUtil.DOT + FileUtil.extName(file.getOriginalFilename()));
-            File outputVoiceFile = File.createTempFile(FileUtil.getPrefix(speechName), StrUtil.DOT + FileUtil.extName(speechName));
-            ImContent receiveContent;
-            OssFile receiveOssOssFile;
-            try {
 
             // 语音转文字
             CreateTranscriptionRequest request = CreateTranscriptionRequest.builder()
                     .model("whisper-1")
                     .language("zh")
                     .build();
-                FileUtil.writeBytes(file.getBytes(), inputVoiceFile);
             TranscriptionResult transcriptionResult = openAiService.createTranscription(request, inputVoiceFile);
             imContent.setContent(transcriptionResult.getText());
+            imContent.setFiles(sendOssOssFile.getId());
+        } catch (Exception e) {
+            throw new CheckedException(e.getLocalizedMessage());
+        } finally {
+            if (inputVoiceFile.delete())
+                System.out.println("已成功删除临时输入语音文件!");
+        }
+        return imContent;
+    }
 
-                // ai涡轮增压
+    @Override
-                receiveContent = askChatCompletion(imContent);
+    @SneakyThrows
-                receiveContent.setContentType(ImMessageTypeEnum.AUDIO.getValue());
+    @Transactional(rollbackFor = Exception.class)
-
+    public OssFile audioSpeech(ImContent imContent) {
+        Optional.ofNullable(imContent.getId()).orElseThrow(() -> new CheckedException("当前聊天内容ID必传,需要进行文件绑定请检查后重试!"));
+        OpenAiService openAiService = AiUtil.getOpenAiService();
+        String speechName = "speech.mp3";
+        String receiveFileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(speechName);
+        File outputVoiceFile = File.createTempFile(FileUtil.getPrefix(speechName), StrUtil.DOT + FileUtil.extName(speechName));
+        OssFile receiveOssOssFile;
+        try {
             // 文字转语音
             CreateSpeechRequest createSpeechRequest = CreateSpeechRequest.builder()
                     .model("tts-1")
-                        .input(receiveContent.getContent())
+                    .input(imContent.getContent())
                     .voice(openAiConfigProperties.getVoice())
                     .responseFormat(FileUtil.extName(speechName))
                     .speed(openAiConfigProperties.getSpeed())
@@ -143,77 +208,16 @@ public class ImContentServiceImpl extends ServiceImpl<ImContentMapper, ImContent
             String receiveVoiceUrl = ossTemplate.getObjectURL(ossProperties.getBucketName(), receiveFileName);
             receiveOssOssFile.setAvailablePath(receiveVoiceUrl);
             fileService.save(receiveOssOssFile);
+
+            // 给当前音频聊天内容绑定音频文件
+            super.update(Wrappers.<ImContent>lambdaUpdate().eq(ImContent::getId, imContent.getId()).set(ImContent::getFiles, receiveOssOssFile.getId()));
         } catch (Exception e) {
             throw new CheckedException(e.getLocalizedMessage());
         } finally {
-                if (inputVoiceFile.delete())
-                    System.out.println("已成功删除临时输入语音文件!");
             if (outputVoiceFile.delete())
-                    System.out.println("已成功删除临时输输出语音文件!");
+                System.out.println("已成功删除临时输出语音文件!");
-            }
-
-            // ai chat message build
-            receiveContent.setFiles(receiveOssOssFile.getId());
-            super.save(receiveContent);
-            Map<String, Object> result = Convert.toMap(String.class, Object.class, receiveContent);
-            result.putAll(Convert.toMap(String.class, Object.class, receiveOssOssFile));
-            return result;
-        } else {
-
-            // ai chat message build
-            ImContent receiveContent = askChatCompletion(imContent);
-            super.save(receiveContent);
-            return Convert.toMap(String.class, Object.class, receiveContent);
-        }
-    }
-
-    private ImContent askChatCompletion(ImContent imContent) {
-        if (StrUtil.isBlank(imContent.getContent())) {
-            throw new CheckedException("提问内容不能为空请检查!");
         }
-        // 保存用户发送聊天记录
+        return receiveOssOssFile;
-        imContent.setSendUserId(SecurityUtils.getCasUser().getId());
-        imContent.setSendTime(LocalDateTime.now());
-        imContent.setReceiveUserId(imContent.getReceiveUserId());
-        imContent.setReceiveTime(LocalDateTime.now());
-        super.save(imContent);
-
-        // 查询当前历史消息,进行AI上下文消息合并
-        List<ImContent> imContents = super.list(Wrappers.<ImContent>lambdaQuery().and(i ->
-                i.eq(ImContent::getSendUserId, imContent.getSendUserId())
-                 .eq(ImContent::getReceiveUserId, imContent.getReceiveUserId())
-        ).or(i ->
-                i.eq(ImContent::getSendUserId, imContent.getReceiveUserId())
-                 .eq(ImContent::getReceiveUserId, imContent.getSendUserId())
-        ) .orderByAsc(ImContent::getSendTime));
-
-        // 历史上下文消息
-        List<ChatMessage> historyMessages = imContents.stream().map(item -> {
-            ChatMessage chatMessage = new ChatMessage();
-            chatMessage.setRole(StrUtil.equals(item.getSendUserId(), imContent.getSendUserId()) ? ChatMessageRole.USER.value() : ChatMessageRole.ASSISTANT.value());
-            chatMessage.setContent(item.getContent());
-            return chatMessage;
-        }).collect(Collectors.toList());
-
-        // 提问消息
-        ChatMessage askMessage = new ChatMessage(ChatMessageRole.USER.value(), imContent.getContent());
-        historyMessages.add(askMessage);
-
-        ChatCompletionResult chatCompletionResult = AiUtil.getChatCompletion(historyMessages);
-        List<ChatCompletionChoice> choiceList = chatCompletionResult.getChoices();
-
-        if (choiceList.isEmpty()) {
-            throw new CheckedException("Ai生成解析失败!");
-        }
-
-        // 构建AI生成消息
-        return new ImContent()
-               .setSendUserId(imContent.getReceiveUserId())
-               .setSendTime(LocalDateTime.now())
-               .setReceiveUserId(SecurityUtils.getCasUser().getId())
-               .setReceiveTime(LocalDateTime.now())
-               .setContent(choiceList.get(0).getMessage().getContent())
-               .setContentType(ImMessageTypeEnum.TEXT.getValue());
     }
 
 }

kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/util/AiUtil.java

@@ -1,11 +1,13 @@
 package com.cloud.kicc.system.util;
 
-import cn.hutool.core.map.MapUtil;
 import com.cloud.kicc.common.core.util.SpringContextHolderUtil;
 import com.cloud.kicc.common.security.util.SecurityUtils;
 import com.cloud.kicc.system.config.OpenAiConfigProperties;
 import com.theokanning.openai.client.OpenAiApi;
-import com.theokanning.openai.completion.chat.*;
+import com.theokanning.openai.completion.chat.ChatCompletionRequest;
+import com.theokanning.openai.completion.chat.ChatCompletionResult;
+import com.theokanning.openai.completion.chat.ChatMessage;
+import com.theokanning.openai.completion.chat.ChatMessageRole;
 import com.theokanning.openai.service.OpenAiService;
 import lombok.experimental.UtilityClass;
 import okhttp3.OkHttpClient;
@@ -15,7 +17,6 @@ import retrofit2.converter.jackson.JacksonConverterFactory;
 
 import java.time.Duration;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 
 /**
@@ -51,7 +52,7 @@ public class AiUtil {
      * <a href="https://platform.openai.com/docs/api-reference/chat/create">...</a>
      */
     public ChatCompletionResult getChatCompletion(List<ChatMessage> historyMessages) {
-        String rule = "是与AI助手的对话。助手乐于助人，富有创造力，聪明限制每次回答在30字以内不得超过30字。";
+        String rule = "是与AI助手的对话。助手乐于助人，富有创造力，聪明限制每次回答在120字以内不得超过120字。";
         List<ChatMessage> messages = new ArrayList<>();
         ChatMessage systemMessage = new ChatMessage(ChatMessageRole.SYSTEM.value(), rule);
         messages.add(systemMessage);