kanglai
/
kicc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

chore: Audio Handle

master
wangxiang 2 years ago
parent
b5d866ed76
commit
d43a40902f
No known key found for this signature in database
GPG Key ID: 1BA7946AB6B232E4
4 changed files with 162 additions and 115 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 21
      kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/controller/ChatGptController.java
  2. 27
      kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/service/IImContentService.java
  3. 172
      kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/service/impl/ImContentServiceImpl.java
  4. 9
      kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/util/AiUtil.java

21
kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/controller/ChatGptController.java
Unescape View File

@ -46,8 +46,25 @@ public class ChatGptController { @@ -46,8 +46,25 @@ public class ChatGptController {
@Inner(false)
@PostMapping("/chat/completions")
public R completions(@Valid @ModelAttribute ImContent imContent, @RequestPart(value = "file", required = false) MultipartFile file) {
return R.ok(iImContentService.completions(imContent, file));
public R chatCompletions(@Valid @RequestBody ImContent imContent) {
SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
return R.ok(iImContentService.chatCompletions(imContent));
}
@Inner(false)
@PostMapping("/audio/transcriptions")
public R audioTranscriptions(@Valid @ModelAttribute ImContent imContent, @RequestPart(value = "file", required = false) MultipartFile file) {
SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
return R.ok(iImContentService.audioTranscriptions(imContent, file));
}
@Inner(false)
@PostMapping("/audio/speech")
public R audioSpeech(@Valid @RequestBody ImContent imContent) {
SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
return R.ok(iImContentService.audioSpeech(imContent));
}
}

27
kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/service/IImContentService.java
Unescape View File

@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage; @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.IService;
import com.cloud.kicc.system.api.entity.ImContent;
import com.cloud.kicc.system.api.entity.OssFile;
import org.springframework.web.multipart.MultipartFile;
import java.util.Map;
@ -18,8 +19,32 @@ import java.util.Map; @@ -18,8 +19,32 @@ import java.util.Map;
*/
public interface IImContentService extends IService<ImContent> {
/**
* 聊天记录
* @param
* @return
*/
IPage<Map<String, Object>> listHistoryMessage(Page page, ImContent imContent);
Map<String, Object> completions(ImContent imContent, MultipartFile file);
/**
* 聊天
* @param
* @return
*/
ImContent chatCompletions(ImContent imContent);
/**
* 语音转文字
* @param imContent
* @return OssFile
*/
ImContent audioTranscriptions(ImContent imContent, MultipartFile file);
/**
* 文字转语音
* @param imContent
* @return OssFile
*/
OssFile audioSpeech(ImContent imContent);
}

172
kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/service/impl/ImContentServiceImpl.java
Unescape View File

@ -72,15 +72,71 @@ public class ImContentServiceImpl extends ServiceImpl<ImContentMapper, ImContent @@ -72,15 +72,71 @@ public class ImContentServiceImpl extends ServiceImpl<ImContentMapper, ImContent
@Override
@Transactional(rollbackFor = Exception.class)
public ImContent chatCompletions(ImContent imContent) {
if (StrUtil.isBlank(imContent.getContent())) {
throw new CheckedException("提问内容不能为空请检查!");
}
// 保存用户发送聊天记录
imContent.setSendUserId(SecurityUtils.getCasUser().getId());
imContent.setSendTime(LocalDateTime.now());
imContent.setReceiveUserId(imContent.getReceiveUserId());
imContent.setReceiveTime(LocalDateTime.now());
super.save(imContent);
// 查询当前历史消息,进行AI上下文消息合并
List<ImContent> imContents = super.list(Wrappers.<ImContent>lambdaQuery().and(i ->
i.eq(ImContent::getSendUserId, imContent.getSendUserId())
.eq(ImContent::getReceiveUserId, imContent.getReceiveUserId())
).or(i ->
i.eq(ImContent::getSendUserId, imContent.getReceiveUserId())
.eq(ImContent::getReceiveUserId, imContent.getSendUserId())
) .orderByAsc(ImContent::getSendTime));
// 历史上下文消息
List<ChatMessage> historyMessages = imContents.stream().map(item -> {
ChatMessage chatMessage = new ChatMessage();
chatMessage.setRole(StrUtil.equals(item.getSendUserId(), imContent.getSendUserId()) ? ChatMessageRole.USER.value() : ChatMessageRole.ASSISTANT.value());
chatMessage.setContent(item.getContent());
return chatMessage;
}).collect(Collectors.toList());
// Completion消息
ChatMessage completionMessage = new ChatMessage(ChatMessageRole.USER.value(), imContent.getContent());
historyMessages.add(completionMessage);
ChatCompletionResult chatCompletionResult = AiUtil.getChatCompletion(historyMessages);
List<ChatCompletionChoice> choiceList = chatCompletionResult.getChoices();
if (choiceList.isEmpty()) {
throw new CheckedException("Ai生成内容解析失败!");
}
// ai chat message build
ImContent receiveContent = new ImContent()
.setSendUserId(imContent.getReceiveUserId())
.setSendTime(LocalDateTime.now())
.setReceiveUserId(SecurityUtils.getCasUser().getId())
.setReceiveTime(LocalDateTime.now())
.setContent(choiceList.get(0).getMessage().getContent())
.setContentType(imContent.getContentType());
super.save(receiveContent);
return receiveContent;
}
@Override
@SneakyThrows
public Map<String, Object> completions(ImContent imContent, MultipartFile file) {
SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
if (imContent.getContentType().equals(ImMessageTypeEnum.AUDIO.getValue())) {
@Transactional(rollbackFor = Exception.class)
public ImContent audioTranscriptions(ImContent imContent, MultipartFile file) {
Optional.ofNullable(file).orElseThrow(() -> new CheckedException("当前语音文件为空,请检查后重试!"));
OpenAiService openAiService = AiUtil.getOpenAiService();
String speechName = "speech.mp3";
String sendFileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(file.getOriginalFilename());
String receiveFileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(speechName);
File inputVoiceFile = File.createTempFile(FileUtil.getPrefix(file.getOriginalFilename()), StrUtil.DOT + FileUtil.extName(file.getOriginalFilename()));
try {
// 获取输入音频时长
FileUtil.writeBytes(file.getBytes(), inputVoiceFile);
AudioFile inputAudioFile = AudioFileIO.read(inputVoiceFile);
long inputVoiceDuration = inputAudioFile.getAudioHeader().getTrackLength();
// 构建发送文件信息进行OSS存储
OssFile sendOssOssFile = new OssFile()
@ -88,37 +144,46 @@ public class ImContentServiceImpl extends ServiceImpl<ImContentMapper, ImContent @@ -88,37 +144,46 @@ public class ImContentServiceImpl extends ServiceImpl<ImContentMapper, ImContent
.setBucketName(ossProperties.getBucketName())
.setOriginal(file.getOriginalFilename())
.setType(FileUtil.extName(file.getOriginalFilename()))
.setDuration(inputVoiceDuration)
.setFileSize(file.getSize())
.setMimeType(file.getContentType());
ossTemplate.putObject(ossProperties.getBucketName(), sendFileName, file.getContentType(), file.getInputStream());
String sendVoiceUrl = ossTemplate.getObjectURL(ossProperties.getBucketName(), sendFileName);
sendOssOssFile.setAvailablePath(sendVoiceUrl);
fileService.save(sendOssOssFile);
imContent.setFiles(sendOssOssFile.getId());
File inputVoiceFile = File.createTempFile(FileUtil.getPrefix(file.getOriginalFilename()), StrUtil.DOT + FileUtil.extName(file.getOriginalFilename()));
File outputVoiceFile = File.createTempFile(FileUtil.getPrefix(speechName), StrUtil.DOT + FileUtil.extName(speechName));
ImContent receiveContent;
OssFile receiveOssOssFile;
try {
// 语音转文字
CreateTranscriptionRequest request = CreateTranscriptionRequest.builder()
.model("whisper-1")
.language("zh")
.build();
FileUtil.writeBytes(file.getBytes(), inputVoiceFile);
TranscriptionResult transcriptionResult = openAiService.createTranscription(request, inputVoiceFile);
imContent.setContent(transcriptionResult.getText());
imContent.setFiles(sendOssOssFile.getId());
} catch (Exception e) {
throw new CheckedException(e.getLocalizedMessage());
} finally {
if (inputVoiceFile.delete())
System.out.println("已成功删除临时输入语音文件!");
}
return imContent;
}
// ai涡轮增压
receiveContent = askChatCompletion(imContent);
receiveContent.setContentType(ImMessageTypeEnum.AUDIO.getValue());
@Override
@SneakyThrows
@Transactional(rollbackFor = Exception.class)
public OssFile audioSpeech(ImContent imContent) {
Optional.ofNullable(imContent.getId()).orElseThrow(() -> new CheckedException("当前聊天内容ID必传,需要进行文件绑定请检查后重试!"));
OpenAiService openAiService = AiUtil.getOpenAiService();
String speechName = "speech.mp3";
String receiveFileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(speechName);
File outputVoiceFile = File.createTempFile(FileUtil.getPrefix(speechName), StrUtil.DOT + FileUtil.extName(speechName));
OssFile receiveOssOssFile;
try {
// 文字转语音
CreateSpeechRequest createSpeechRequest = CreateSpeechRequest.builder()
.model("tts-1")
.input(receiveContent.getContent())
.input(imContent.getContent())
.voice(openAiConfigProperties.getVoice())
.responseFormat(FileUtil.extName(speechName))
.speed(openAiConfigProperties.getSpeed())
@ -143,77 +208,16 @@ public class ImContentServiceImpl extends ServiceImpl<ImContentMapper, ImContent @@ -143,77 +208,16 @@ public class ImContentServiceImpl extends ServiceImpl<ImContentMapper, ImContent
String receiveVoiceUrl = ossTemplate.getObjectURL(ossProperties.getBucketName(), receiveFileName);
receiveOssOssFile.setAvailablePath(receiveVoiceUrl);
fileService.save(receiveOssOssFile);
// 给当前音频聊天内容绑定音频文件
super.update(Wrappers.<ImContent>lambdaUpdate().eq(ImContent::getId, imContent.getId()).set(ImContent::getFiles, receiveOssOssFile.getId()));
} catch (Exception e) {
throw new CheckedException(e.getLocalizedMessage());
} finally {
if (inputVoiceFile.delete())
System.out.println("已成功删除临时输入语音文件!");
if (outputVoiceFile.delete())
System.out.println("已成功删除临时输输出语音文件!");
}
// ai chat message build
receiveContent.setFiles(receiveOssOssFile.getId());
super.save(receiveContent);
Map<String, Object> result = Convert.toMap(String.class, Object.class, receiveContent);
result.putAll(Convert.toMap(String.class, Object.class, receiveOssOssFile));
return result;
} else {
// ai chat message build
ImContent receiveContent = askChatCompletion(imContent);
super.save(receiveContent);
return Convert.toMap(String.class, Object.class, receiveContent);
}
}
private ImContent askChatCompletion(ImContent imContent) {
if (StrUtil.isBlank(imContent.getContent())) {
throw new CheckedException("提问内容不能为空请检查!");
System.out.println("已成功删除临时输出语音文件!");
}
// 保存用户发送聊天记录
imContent.setSendUserId(SecurityUtils.getCasUser().getId());
imContent.setSendTime(LocalDateTime.now());
imContent.setReceiveUserId(imContent.getReceiveUserId());
imContent.setReceiveTime(LocalDateTime.now());
super.save(imContent);
// 查询当前历史消息,进行AI上下文消息合并
List<ImContent> imContents = super.list(Wrappers.<ImContent>lambdaQuery().and(i ->
i.eq(ImContent::getSendUserId, imContent.getSendUserId())
.eq(ImContent::getReceiveUserId, imContent.getReceiveUserId())
).or(i ->
i.eq(ImContent::getSendUserId, imContent.getReceiveUserId())
.eq(ImContent::getReceiveUserId, imContent.getSendUserId())
) .orderByAsc(ImContent::getSendTime));
// 历史上下文消息
List<ChatMessage> historyMessages = imContents.stream().map(item -> {
ChatMessage chatMessage = new ChatMessage();
chatMessage.setRole(StrUtil.equals(item.getSendUserId(), imContent.getSendUserId()) ? ChatMessageRole.USER.value() : ChatMessageRole.ASSISTANT.value());
chatMessage.setContent(item.getContent());
return chatMessage;
}).collect(Collectors.toList());
// 提问消息
ChatMessage askMessage = new ChatMessage(ChatMessageRole.USER.value(), imContent.getContent());
historyMessages.add(askMessage);
ChatCompletionResult chatCompletionResult = AiUtil.getChatCompletion(historyMessages);
List<ChatCompletionChoice> choiceList = chatCompletionResult.getChoices();
if (choiceList.isEmpty()) {
throw new CheckedException("Ai生成解析失败!");
}
// 构建AI生成消息
return new ImContent()
.setSendUserId(imContent.getReceiveUserId())
.setSendTime(LocalDateTime.now())
.setReceiveUserId(SecurityUtils.getCasUser().getId())
.setReceiveTime(LocalDateTime.now())
.setContent(choiceList.get(0).getMessage().getContent())
.setContentType(ImMessageTypeEnum.TEXT.getValue());
return receiveOssOssFile;
}
}

9
kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/util/AiUtil.java
Unescape View File

@ -1,11 +1,13 @@ @@ -1,11 +1,13 @@
package com.cloud.kicc.system.util;
import cn.hutool.core.map.MapUtil;
import com.cloud.kicc.common.core.util.SpringContextHolderUtil;
import com.cloud.kicc.common.security.util.SecurityUtils;
import com.cloud.kicc.system.config.OpenAiConfigProperties;
import com.theokanning.openai.client.OpenAiApi;
import com.theokanning.openai.completion.chat.*;
import com.theokanning.openai.completion.chat.ChatCompletionRequest;
import com.theokanning.openai.completion.chat.ChatCompletionResult;
import com.theokanning.openai.completion.chat.ChatMessage;
import com.theokanning.openai.completion.chat.ChatMessageRole;
import com.theokanning.openai.service.OpenAiService;
import lombok.experimental.UtilityClass;
import okhttp3.OkHttpClient;
@ -15,7 +17,6 @@ import retrofit2.converter.jackson.JacksonConverterFactory; @@ -15,7 +17,6 @@ import retrofit2.converter.jackson.JacksonConverterFactory;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
/**
@ -51,7 +52,7 @@ public class AiUtil { @@ -51,7 +52,7 @@ public class AiUtil {
* <a href="https://platform.openai.com/docs/api-reference/chat/create">...</a>
*/
public ChatCompletionResult getChatCompletion(List<ChatMessage> historyMessages) {
String rule = "是与AI助手的对话。助手乐于助人，富有创造力，聪明限制每次回答在30字以内不得超过30字。";
String rule = "是与AI助手的对话。助手乐于助人，富有创造力，聪明限制每次回答在120字以内不得超过120字。";
List<ChatMessage> messages = new ArrayList<>();
ChatMessage systemMessage = new ChatMessage(ChatMessageRole.SYSTEM.value(), rule);
messages.add(systemMessage);

Write Preview
Loading…
Cancel
Save