kanglai
/
kicc
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

chore: Audio Handle

master
wangxiang 2 years ago
parent
b5d866ed76
commit
d43a40902f
No known key found for this signature in database
GPG Key ID: 1BA7946AB6B232E4
4 changed files with 162 additions and 115 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 21
      kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/controller/ChatGptController.java
  2. 27
      kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/service/IImContentService.java
  3. 220
      kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/service/impl/ImContentServiceImpl.java
  4. 9
      kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/util/AiUtil.java

21
kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/controller/ChatGptController.java
Unescape View File

@ -46,8 +46,25 @@ public class ChatGptController {
@Inner(false) @Inner(false)
@PostMapping("/chat/completions") @PostMapping("/chat/completions")
public R completions(@Valid @ModelAttribute ImContent imContent, @RequestPart(value = "file", required = false) MultipartFile file) { public R chatCompletions(@Valid @RequestBody ImContent imContent) {
return R.ok(iImContentService.completions(imContent, file)); SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
return R.ok(iImContentService.chatCompletions(imContent));
}
@Inner(false)
@PostMapping("/audio/transcriptions")
public R audioTranscriptions(@Valid @ModelAttribute ImContent imContent, @RequestPart(value = "file", required = false) MultipartFile file) {
SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
return R.ok(iImContentService.audioTranscriptions(imContent, file));
}
@Inner(false)
@PostMapping("/audio/speech")
public R audioSpeech(@Valid @RequestBody ImContent imContent) {
SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
return R.ok(iImContentService.audioSpeech(imContent));
} }
} }

27
kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/service/IImContentService.java
Unescape View File

@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.IService; import com.baomidou.mybatisplus.extension.service.IService;
import com.cloud.kicc.system.api.entity.ImContent; import com.cloud.kicc.system.api.entity.ImContent;
import com.cloud.kicc.system.api.entity.OssFile;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import java.util.Map; import java.util.Map;
@ -18,8 +19,32 @@ import java.util.Map;
*/ */
public interface IImContentService extends IService<ImContent> { public interface IImContentService extends IService<ImContent> {
/**
* 聊天记录
* @param
* @return
*/
IPage<Map<String, Object>> listHistoryMessage(Page page, ImContent imContent); IPage<Map<String, Object>> listHistoryMessage(Page page, ImContent imContent);
Map<String, Object> completions(ImContent imContent, MultipartFile file); /**
* 聊天
* @param
* @return
*/
ImContent chatCompletions(ImContent imContent);
/**
* 语音转文字
* @param imContent
* @return OssFile
*/
ImContent audioTranscriptions(ImContent imContent, MultipartFile file);
/**
* 文字转语音
* @param imContent
* @return OssFile
*/
OssFile audioSpeech(ImContent imContent);
} }

220
kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/service/impl/ImContentServiceImpl.java
Unescape View File

@ -72,102 +72,7 @@ public class ImContentServiceImpl extends ServiceImpl<ImContentMapper, ImContent
@Override @Override
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
@SneakyThrows public ImContent chatCompletions(ImContent imContent) {
public Map<String, Object> completions(ImContent imContent, MultipartFile file) {
SecurityUtils.openInterfaceTemporaryLoginSession(imContent.getSendUserId());
if (imContent.getContentType().equals(ImMessageTypeEnum.AUDIO.getValue())) {
Optional.ofNullable(file).orElseThrow(() -> new CheckedException("当前语音文件为空,请检查后重试!"));
OpenAiService openAiService = AiUtil.getOpenAiService();
String speechName = "speech.mp3";
String sendFileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(file.getOriginalFilename());
String receiveFileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(speechName);
// 构建发送文件信息进行OSS存储
OssFile sendOssOssFile = new OssFile()
.setFileName(sendFileName)
.setBucketName(ossProperties.getBucketName())
.setOriginal(file.getOriginalFilename())
.setType(FileUtil.extName(file.getOriginalFilename()))
.setFileSize(file.getSize())
.setMimeType(file.getContentType());
ossTemplate.putObject(ossProperties.getBucketName(), sendFileName, file.getContentType(), file.getInputStream());
String sendVoiceUrl = ossTemplate.getObjectURL(ossProperties.getBucketName(), sendFileName);
sendOssOssFile.setAvailablePath(sendVoiceUrl);
fileService.save(sendOssOssFile);
imContent.setFiles(sendOssOssFile.getId());
File inputVoiceFile = File.createTempFile(FileUtil.getPrefix(file.getOriginalFilename()), StrUtil.DOT + FileUtil.extName(file.getOriginalFilename()));
File outputVoiceFile = File.createTempFile(FileUtil.getPrefix(speechName), StrUtil.DOT + FileUtil.extName(speechName));
ImContent receiveContent;
OssFile receiveOssOssFile;
try {
// 语音转文字
CreateTranscriptionRequest request = CreateTranscriptionRequest.builder()
.model("whisper-1")
.language("zh")
.build();
FileUtil.writeBytes(file.getBytes(), inputVoiceFile);
TranscriptionResult transcriptionResult = openAiService.createTranscription(request, inputVoiceFile);
imContent.setContent(transcriptionResult.getText());
// ai涡轮增压
receiveContent = askChatCompletion(imContent);
receiveContent.setContentType(ImMessageTypeEnum.AUDIO.getValue());
// 文字转语音
CreateSpeechRequest createSpeechRequest = CreateSpeechRequest.builder()
.model("tts-1")
.input(receiveContent.getContent())
.voice(openAiConfigProperties.getVoice())
.responseFormat(FileUtil.extName(speechName))
.speed(openAiConfigProperties.getSpeed())
.build();
ResponseBody responseBody = openAiService.createSpeech(createSpeechRequest);
FileUtil.writeBytes(responseBody.bytes(), outputVoiceFile);
// 获取输出音频时长
AudioFile outputAudioFile = AudioFileIO.read(outputVoiceFile);
long outputVoiceDuration = outputAudioFile.getAudioHeader().getTrackLength();
// 构建发送文件信息进行OSS存储
receiveOssOssFile = new OssFile()
.setFileName(receiveFileName)
.setBucketName(ossProperties.getBucketName())
.setOriginal(speechName)
.setType(FileUtil.extName(speechName))
.setFileSize(responseBody.contentLength())
.setDuration(outputVoiceDuration)
.setMimeType(Objects.requireNonNull(responseBody.contentType()).toString());
ossTemplate.putObject(ossProperties.getBucketName(), receiveFileName, Objects.requireNonNull(responseBody.contentType()).toString(), FileUtil.getInputStream(outputVoiceFile));
String receiveVoiceUrl = ossTemplate.getObjectURL(ossProperties.getBucketName(), receiveFileName);
receiveOssOssFile.setAvailablePath(receiveVoiceUrl);
fileService.save(receiveOssOssFile);
} catch (Exception e) {
throw new CheckedException(e.getLocalizedMessage());
} finally {
if (inputVoiceFile.delete())
System.out.println("已成功删除临时输入语音文件!");
if (outputVoiceFile.delete())
System.out.println("已成功删除临时输输出语音文件!");
}
// ai chat message build
receiveContent.setFiles(receiveOssOssFile.getId());
super.save(receiveContent);
Map<String, Object> result = Convert.toMap(String.class, Object.class, receiveContent);
result.putAll(Convert.toMap(String.class, Object.class, receiveOssOssFile));
return result;
} else {
// ai chat message build
ImContent receiveContent = askChatCompletion(imContent);
super.save(receiveContent);
return Convert.toMap(String.class, Object.class, receiveContent);
}
}
private ImContent askChatCompletion(ImContent imContent) {
if (StrUtil.isBlank(imContent.getContent())) { if (StrUtil.isBlank(imContent.getContent())) {
throw new CheckedException("提问内容不能为空请检查!"); throw new CheckedException("提问内容不能为空请检查!");
} }
@ -195,25 +100,124 @@ public class ImContentServiceImpl extends ServiceImpl<ImContentMapper, ImContent
return chatMessage; return chatMessage;
}).collect(Collectors.toList()); }).collect(Collectors.toList());
// 提问消息 // Completion消息
ChatMessage askMessage = new ChatMessage(ChatMessageRole.USER.value(), imContent.getContent()); ChatMessage completionMessage = new ChatMessage(ChatMessageRole.USER.value(), imContent.getContent());
historyMessages.add(askMessage); historyMessages.add(completionMessage);
ChatCompletionResult chatCompletionResult = AiUtil.getChatCompletion(historyMessages); ChatCompletionResult chatCompletionResult = AiUtil.getChatCompletion(historyMessages);
List<ChatCompletionChoice> choiceList = chatCompletionResult.getChoices(); List<ChatCompletionChoice> choiceList = chatCompletionResult.getChoices();
if (choiceList.isEmpty()) { if (choiceList.isEmpty()) {
throw new CheckedException("Ai生成解析失败!"); throw new CheckedException("Ai生成内容解析失败!");
} }
// 构建AI生成消息 // ai chat message build
return new ImContent() ImContent receiveContent = new ImContent()
.setSendUserId(imContent.getReceiveUserId()) .setSendUserId(imContent.getReceiveUserId())
.setSendTime(LocalDateTime.now()) .setSendTime(LocalDateTime.now())
.setReceiveUserId(SecurityUtils.getCasUser().getId()) .setReceiveUserId(SecurityUtils.getCasUser().getId())
.setReceiveTime(LocalDateTime.now()) .setReceiveTime(LocalDateTime.now())
.setContent(choiceList.get(0).getMessage().getContent()) .setContent(choiceList.get(0).getMessage().getContent())
.setContentType(ImMessageTypeEnum.TEXT.getValue()); .setContentType(imContent.getContentType());
super.save(receiveContent);
return receiveContent;
}
@Override
@SneakyThrows
@Transactional(rollbackFor = Exception.class)
public ImContent audioTranscriptions(ImContent imContent, MultipartFile file) {
Optional.ofNullable(file).orElseThrow(() -> new CheckedException("当前语音文件为空,请检查后重试!"));
OpenAiService openAiService = AiUtil.getOpenAiService();
String sendFileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(file.getOriginalFilename());
File inputVoiceFile = File.createTempFile(FileUtil.getPrefix(file.getOriginalFilename()), StrUtil.DOT + FileUtil.extName(file.getOriginalFilename()));
try {
// 获取输入音频时长
FileUtil.writeBytes(file.getBytes(), inputVoiceFile);
AudioFile inputAudioFile = AudioFileIO.read(inputVoiceFile);
long inputVoiceDuration = inputAudioFile.getAudioHeader().getTrackLength();
// 构建发送文件信息进行OSS存储
OssFile sendOssOssFile = new OssFile()
.setFileName(sendFileName)
.setBucketName(ossProperties.getBucketName())
.setOriginal(file.getOriginalFilename())
.setType(FileUtil.extName(file.getOriginalFilename()))
.setDuration(inputVoiceDuration)
.setFileSize(file.getSize())
.setMimeType(file.getContentType());
ossTemplate.putObject(ossProperties.getBucketName(), sendFileName, file.getContentType(), file.getInputStream());
String sendVoiceUrl = ossTemplate.getObjectURL(ossProperties.getBucketName(), sendFileName);
sendOssOssFile.setAvailablePath(sendVoiceUrl);
fileService.save(sendOssOssFile);
// 语音转文字
CreateTranscriptionRequest request = CreateTranscriptionRequest.builder()
.model("whisper-1")
.language("zh")
.build();
TranscriptionResult transcriptionResult = openAiService.createTranscription(request, inputVoiceFile);
imContent.setContent(transcriptionResult.getText());
imContent.setFiles(sendOssOssFile.getId());
} catch (Exception e) {
throw new CheckedException(e.getLocalizedMessage());
} finally {
if (inputVoiceFile.delete())
System.out.println("已成功删除临时输入语音文件!");
}
return imContent;
}
@Override
@SneakyThrows
@Transactional(rollbackFor = Exception.class)
public OssFile audioSpeech(ImContent imContent) {
Optional.ofNullable(imContent.getId()).orElseThrow(() -> new CheckedException("当前聊天内容ID必传,需要进行文件绑定请检查后重试!"));
OpenAiService openAiService = AiUtil.getOpenAiService();
String speechName = "speech.mp3";
String receiveFileName = IdUtil.simpleUUID() + StrUtil.DOT + FileUtil.extName(speechName);
File outputVoiceFile = File.createTempFile(FileUtil.getPrefix(speechName), StrUtil.DOT + FileUtil.extName(speechName));
OssFile receiveOssOssFile;
try {
// 文字转语音
CreateSpeechRequest createSpeechRequest = CreateSpeechRequest.builder()
.model("tts-1")
.input(imContent.getContent())
.voice(openAiConfigProperties.getVoice())
.responseFormat(FileUtil.extName(speechName))
.speed(openAiConfigProperties.getSpeed())
.build();
ResponseBody responseBody = openAiService.createSpeech(createSpeechRequest);
FileUtil.writeBytes(responseBody.bytes(), outputVoiceFile);
// 获取输出音频时长
AudioFile outputAudioFile = AudioFileIO.read(outputVoiceFile);
long outputVoiceDuration = outputAudioFile.getAudioHeader().getTrackLength();
// 构建发送文件信息进行OSS存储
receiveOssOssFile = new OssFile()
.setFileName(receiveFileName)
.setBucketName(ossProperties.getBucketName())
.setOriginal(speechName)
.setType(FileUtil.extName(speechName))
.setFileSize(responseBody.contentLength())
.setDuration(outputVoiceDuration)
.setMimeType(Objects.requireNonNull(responseBody.contentType()).toString());
ossTemplate.putObject(ossProperties.getBucketName(), receiveFileName, Objects.requireNonNull(responseBody.contentType()).toString(), FileUtil.getInputStream(outputVoiceFile));
String receiveVoiceUrl = ossTemplate.getObjectURL(ossProperties.getBucketName(), receiveFileName);
receiveOssOssFile.setAvailablePath(receiveVoiceUrl);
fileService.save(receiveOssOssFile);
// 给当前音频聊天内容绑定音频文件
super.update(Wrappers.<ImContent>lambdaUpdate().eq(ImContent::getId, imContent.getId()).set(ImContent::getFiles, receiveOssOssFile.getId()));
} catch (Exception e) {
throw new CheckedException(e.getLocalizedMessage());
} finally {
if (outputVoiceFile.delete())
System.out.println("已成功删除临时输出语音文件!");
}
return receiveOssOssFile;
} }
} }

9
kicc-platform/kicc-platform-biz/kicc-system-biz/src/main/java/com/cloud/kicc/system/util/AiUtil.java
Unescape View File

@ -1,11 +1,13 @@
package com.cloud.kicc.system.util; package com.cloud.kicc.system.util;
import cn.hutool.core.map.MapUtil;
import com.cloud.kicc.common.core.util.SpringContextHolderUtil; import com.cloud.kicc.common.core.util.SpringContextHolderUtil;
import com.cloud.kicc.common.security.util.SecurityUtils; import com.cloud.kicc.common.security.util.SecurityUtils;
import com.cloud.kicc.system.config.OpenAiConfigProperties; import com.cloud.kicc.system.config.OpenAiConfigProperties;
import com.theokanning.openai.client.OpenAiApi; import com.theokanning.openai.client.OpenAiApi;
import com.theokanning.openai.completion.chat.*; import com.theokanning.openai.completion.chat.ChatCompletionRequest;
import com.theokanning.openai.completion.chat.ChatCompletionResult;
import com.theokanning.openai.completion.chat.ChatMessage;
import com.theokanning.openai.completion.chat.ChatMessageRole;
import com.theokanning.openai.service.OpenAiService; import com.theokanning.openai.service.OpenAiService;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
import okhttp3.OkHttpClient; import okhttp3.OkHttpClient;
@ -15,7 +17,6 @@ import retrofit2.converter.jackson.JacksonConverterFactory;
import java.time.Duration; import java.time.Duration;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
/** /**
@ -51,7 +52,7 @@ public class AiUtil {
* <a href="https://platform.openai.com/docs/api-reference/chat/create">...</a> * <a href="https://platform.openai.com/docs/api-reference/chat/create">...</a>
*/ */
public ChatCompletionResult getChatCompletion(List<ChatMessage> historyMessages) { public ChatCompletionResult getChatCompletion(List<ChatMessage> historyMessages) {
String rule = "是与AI助手的对话。助手乐于助人，富有创造力，聪明限制每次回答在30字以内不得超过30字。"; String rule = "是与AI助手的对话。助手乐于助人，富有创造力，聪明限制每次回答在120字以内不得超过120字。";
List<ChatMessage> messages = new ArrayList<>(); List<ChatMessage> messages = new ArrayList<>();
ChatMessage systemMessage = new ChatMessage(ChatMessageRole.SYSTEM.value(), rule); ChatMessage systemMessage = new ChatMessage(ChatMessageRole.SYSTEM.value(), rule);
messages.add(systemMessage); messages.add(systemMessage);

Write Preview
Loading…
Cancel
Save