|
|
@@ -0,0 +1,174 @@
|
|
|
+import concurrent.futures
|
|
|
+import copy
|
|
|
+from functools import reduce
|
|
|
+from io import BytesIO
|
|
|
+from typing import Optional
|
|
|
+
|
|
|
+from flask import Response, stream_with_context
|
|
|
+from openai import AzureOpenAI
|
|
|
+from pydub import AudioSegment
|
|
|
+
|
|
|
+from core.model_runtime.entities.model_entities import AIModelEntity
|
|
|
+from core.model_runtime.errors.invoke import InvokeBadRequestError
|
|
|
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
|
|
|
+from core.model_runtime.model_providers.__base.tts_model import TTSModel
|
|
|
+from core.model_runtime.model_providers.azure_openai._common import _CommonAzureOpenAI
|
|
|
+from core.model_runtime.model_providers.azure_openai._constant import TTS_BASE_MODELS, AzureBaseModel
|
|
|
+from extensions.ext_storage import storage
|
|
|
+
|
|
|
+
|
|
|
+class AzureOpenAIText2SpeechModel(_CommonAzureOpenAI, TTSModel):
|
|
|
+ """
|
|
|
+ Model class for OpenAI Speech to text model.
|
|
|
+ """
|
|
|
+
|
|
|
+ def _invoke(self, model: str, tenant_id: str, credentials: dict,
|
|
|
+ content_text: str, voice: str, streaming: bool, user: Optional[str] = None) -> any:
|
|
|
+ """
|
|
|
+ _invoke text2speech model
|
|
|
+
|
|
|
+ :param model: model name
|
|
|
+ :param tenant_id: user tenant id
|
|
|
+ :param credentials: model credentials
|
|
|
+ :param content_text: text content to be translated
|
|
|
+ :param voice: model timbre
|
|
|
+ :param streaming: output is streaming
|
|
|
+ :param user: unique user id
|
|
|
+ :return: text translated to audio file
|
|
|
+ """
|
|
|
+ audio_type = self._get_model_audio_type(model, credentials)
|
|
|
+ if not voice or voice not in [d['value'] for d in self.get_tts_model_voices(model=model, credentials=credentials)]:
|
|
|
+ voice = self._get_model_default_voice(model, credentials)
|
|
|
+ if streaming:
|
|
|
+ return Response(stream_with_context(self._tts_invoke_streaming(model=model,
|
|
|
+ credentials=credentials,
|
|
|
+ content_text=content_text,
|
|
|
+ tenant_id=tenant_id,
|
|
|
+ voice=voice)),
|
|
|
+ status=200, mimetype=f'audio/{audio_type}')
|
|
|
+ else:
|
|
|
+ return self._tts_invoke(model=model, credentials=credentials, content_text=content_text, voice=voice)
|
|
|
+
|
|
|
+ def validate_credentials(self, model: str, credentials: dict, user: Optional[str] = None) -> None:
|
|
|
+ """
|
|
|
+ validate credentials text2speech model
|
|
|
+
|
|
|
+ :param model: model name
|
|
|
+ :param credentials: model credentials
|
|
|
+ :param user: unique user id
|
|
|
+ :return: text translated to audio file
|
|
|
+ """
|
|
|
+ try:
|
|
|
+ self._tts_invoke(
|
|
|
+ model=model,
|
|
|
+ credentials=credentials,
|
|
|
+ content_text='Hello Dify!',
|
|
|
+ voice=self._get_model_default_voice(model, credentials),
|
|
|
+ )
|
|
|
+ except Exception as ex:
|
|
|
+ raise CredentialsValidateFailedError(str(ex))
|
|
|
+
|
|
|
+ def _tts_invoke(self, model: str, credentials: dict, content_text: str, voice: str) -> Response:
|
|
|
+ """
|
|
|
+ _tts_invoke text2speech model
|
|
|
+
|
|
|
+ :param model: model name
|
|
|
+ :param credentials: model credentials
|
|
|
+ :param content_text: text content to be translated
|
|
|
+ :param voice: model timbre
|
|
|
+ :return: text translated to audio file
|
|
|
+ """
|
|
|
+ audio_type = self._get_model_audio_type(model, credentials)
|
|
|
+ word_limit = self._get_model_word_limit(model, credentials)
|
|
|
+ max_workers = self._get_model_workers_limit(model, credentials)
|
|
|
+ try:
|
|
|
+ sentences = list(self._split_text_into_sentences(text=content_text, limit=word_limit))
|
|
|
+ audio_bytes_list = list()
|
|
|
+
|
|
|
+ # Create a thread pool and map the function to the list of sentences
|
|
|
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
|
+ futures = [executor.submit(self._process_sentence, sentence=sentence, model=model, voice=voice,
|
|
|
+ credentials=credentials) for sentence in sentences]
|
|
|
+ for future in futures:
|
|
|
+ try:
|
|
|
+ if future.result():
|
|
|
+ audio_bytes_list.append(future.result())
|
|
|
+ except Exception as ex:
|
|
|
+ raise InvokeBadRequestError(str(ex))
|
|
|
+
|
|
|
+ if len(audio_bytes_list) > 0:
|
|
|
+ audio_segments = [AudioSegment.from_file(BytesIO(audio_bytes), format=audio_type) for audio_bytes in
|
|
|
+ audio_bytes_list if audio_bytes]
|
|
|
+ combined_segment = reduce(lambda x, y: x + y, audio_segments)
|
|
|
+ buffer: BytesIO = BytesIO()
|
|
|
+ combined_segment.export(buffer, format=audio_type)
|
|
|
+ buffer.seek(0)
|
|
|
+ return Response(buffer.read(), status=200, mimetype=f"audio/{audio_type}")
|
|
|
+ except Exception as ex:
|
|
|
+ raise InvokeBadRequestError(str(ex))
|
|
|
+
|
|
|
+ # Todo: To improve the streaming function
|
|
|
+ def _tts_invoke_streaming(self, model: str, tenant_id: str, credentials: dict, content_text: str,
|
|
|
+ voice: str) -> any:
|
|
|
+ """
|
|
|
+ _tts_invoke_streaming text2speech model
|
|
|
+
|
|
|
+ :param model: model name
|
|
|
+ :param tenant_id: user tenant id
|
|
|
+ :param credentials: model credentials
|
|
|
+ :param content_text: text content to be translated
|
|
|
+ :param voice: model timbre
|
|
|
+ :return: text translated to audio file
|
|
|
+ """
|
|
|
+ # transform credentials to kwargs for model instance
|
|
|
+ credentials_kwargs = self._to_credential_kwargs(credentials)
|
|
|
+ if not voice or voice not in self.get_tts_model_voices(model=model, credentials=credentials):
|
|
|
+ voice = self._get_model_default_voice(model, credentials)
|
|
|
+ word_limit = self._get_model_word_limit(model, credentials)
|
|
|
+ audio_type = self._get_model_audio_type(model, credentials)
|
|
|
+ tts_file_id = self._get_file_name(content_text)
|
|
|
+ file_path = f'generate_files/audio/{tenant_id}/{tts_file_id}.{audio_type}'
|
|
|
+ try:
|
|
|
+ client = AzureOpenAI(**credentials_kwargs)
|
|
|
+ sentences = list(self._split_text_into_sentences(text=content_text, limit=word_limit))
|
|
|
+ for sentence in sentences:
|
|
|
+ response = client.audio.speech.create(model=model, voice=voice, input=sentence.strip())
|
|
|
+ # response.stream_to_file(file_path)
|
|
|
+ storage.save(file_path, response.read())
|
|
|
+ except Exception as ex:
|
|
|
+ raise InvokeBadRequestError(str(ex))
|
|
|
+
|
|
|
+ def _process_sentence(self, sentence: str, model: str,
|
|
|
+ voice, credentials: dict):
|
|
|
+ """
|
|
|
+ _tts_invoke openai text2speech model api
|
|
|
+
|
|
|
+ :param model: model name
|
|
|
+ :param credentials: model credentials
|
|
|
+ :param voice: model timbre
|
|
|
+ :param sentence: text content to be translated
|
|
|
+ :return: text translated to audio file
|
|
|
+ """
|
|
|
+ # transform credentials to kwargs for model instance
|
|
|
+ credentials_kwargs = self._to_credential_kwargs(credentials)
|
|
|
+ client = AzureOpenAI(**credentials_kwargs)
|
|
|
+ response = client.audio.speech.create(model=model, voice=voice, input=sentence.strip())
|
|
|
+ if isinstance(response.read(), bytes):
|
|
|
+ return response.read()
|
|
|
+
|
|
|
+ def get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
|
|
|
+ ai_model_entity = self._get_ai_model_entity(credentials['base_model_name'], model)
|
|
|
+ return ai_model_entity.entity
|
|
|
+
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _get_ai_model_entity(base_model_name: str, model: str) -> AzureBaseModel:
|
|
|
+ for ai_model_entity in TTS_BASE_MODELS:
|
|
|
+ if ai_model_entity.base_model_name == base_model_name:
|
|
|
+ ai_model_entity_copy = copy.deepcopy(ai_model_entity)
|
|
|
+ ai_model_entity_copy.entity.model = model
|
|
|
+ ai_model_entity_copy.entity.label.en_US = model
|
|
|
+ ai_model_entity_copy.entity.label.zh_Hans = model
|
|
|
+ return ai_model_entity_copy
|
|
|
+
|
|
|
+ return None
|
呆萌闷油瓶