git.basschouten.com Git - openhab-addons.git/blob

   1 /**
   2  * Copyright (c) 2010-2022 Contributors to the openHAB project
   3  *
   4  * See the NOTICE file(s) distributed with this work for additional
   5  * information.
   6  *
   7  * This program and the accompanying materials are made available under the
   8  * terms of the Eclipse Public License 2.0 which is available at
   9  * http://www.eclipse.org/legal/epl-2.0
  10  *
  11  * SPDX-License-Identifier: EPL-2.0
  12  */
  13 package org.openhab.voice.googletts.internal;
  14
  15 import java.io.File;
  16 import java.io.FileNotFoundException;
  17 import java.io.FileOutputStream;
  18 import java.io.IOException;
  19 import java.math.BigInteger;
  20 import java.nio.charset.StandardCharsets;
  21 import java.nio.file.Files;
  22 import java.security.MessageDigest;
  23 import java.security.NoSuchAlgorithmException;
  24 import java.util.ArrayList;
  25 import java.util.Arrays;
  26 import java.util.Base64;
  27 import java.util.Dictionary;
  28 import java.util.HashMap;
  29 import java.util.HashSet;
  30 import java.util.List;
  31 import java.util.Locale;
  32 import java.util.Map;
  33 import java.util.Set;
  34
  35 import org.eclipse.jdt.annotation.Nullable;
  36 import org.eclipse.jetty.http.HttpHeader;
  37 import org.eclipse.jetty.http.MimeTypes;
  38 import org.openhab.core.audio.AudioFormat;
  39 import org.openhab.core.auth.AuthenticationException;
  40 import org.openhab.core.auth.client.oauth2.AccessTokenResponse;
  41 import org.openhab.core.auth.client.oauth2.OAuthClientService;
  42 import org.openhab.core.auth.client.oauth2.OAuthException;
  43 import org.openhab.core.auth.client.oauth2.OAuthFactory;
  44 import org.openhab.core.auth.client.oauth2.OAuthResponseException;
  45 import org.openhab.core.i18n.CommunicationException;
  46 import org.openhab.core.io.net.http.HttpRequestBuilder;
  47 import org.openhab.voice.googletts.internal.dto.AudioConfig;
  48 import org.openhab.voice.googletts.internal.dto.AudioEncoding;
  49 import org.openhab.voice.googletts.internal.dto.ListVoicesResponse;
  50 import org.openhab.voice.googletts.internal.dto.SsmlVoiceGender;
  51 import org.openhab.voice.googletts.internal.dto.SynthesisInput;
  52 import org.openhab.voice.googletts.internal.dto.SynthesizeSpeechRequest;
  53 import org.openhab.voice.googletts.internal.dto.SynthesizeSpeechResponse;
  54 import org.openhab.voice.googletts.internal.dto.Voice;
  55 import org.openhab.voice.googletts.internal.dto.VoiceSelectionParams;
  56 import org.osgi.service.cm.Configuration;
  57 import org.osgi.service.cm.ConfigurationAdmin;
  58 import org.slf4j.Logger;
  59 import org.slf4j.LoggerFactory;
  60
  61 import com.google.gson.Gson;
  62 import com.google.gson.GsonBuilder;
  63 import com.google.gson.JsonSyntaxException;
  64
  65 /**
  66  * Google Cloud TTS API call implementation.
  67  *
  68  * @author Gabor Bicskei - Initial contribution and API
  69  */
  70 class GoogleCloudAPI {
  71
  72     private static final char EXTENSION_SEPARATOR = '.';
  73     private static final char UNIX_SEPARATOR = '/';
  74     private static final char WINDOWS_SEPARATOR = '\\';
  75
  76     private static final String BEARER = "Bearer ";
  77
  78     private static final String GCP_AUTH_URI = "https://accounts.google.com/o/oauth2/auth";
  79     private static final String GCP_TOKEN_URI = "https://accounts.google.com/o/oauth2/token";
  80     private static final String GCP_REDIRECT_URI = "https://www.google.com";
  81     /**
  82      * Google Cloud Platform authorization scope
  83      */
  84     private static final String GCP_SCOPE = "https://www.googleapis.com/auth/cloud-platform";
  85
  86     /**
  87      * URL used for retrieving the list of available voices
  88      */
  89     private static final String LIST_VOICES_URL = "https://texttospeech.googleapis.com/v1/voices";
  90
  91     /**
  92      * URL used for synthesizing text to speech
  93      */
  94     private static final String SYTNHESIZE_SPEECH_URL = "https://texttospeech.googleapis.com/v1/text:synthesize";
  95
  96     /**
  97      * Logger
  98      */
  99     private final Logger logger = LoggerFactory.getLogger(GoogleCloudAPI.class);
 100
 101     /**
 102      * Supported voices and locales
 103      */
 104     private final Map<Locale, Set<GoogleTTSVoice>> voices = new HashMap<>();
 105
 106     /**
 107      * Cache folder
 108      */
 109     private File cacheFolder;
 110
 111     /**
 112      * Configuration
 113      */
 114     private @Nullable GoogleTTSConfig config;
 115
 116     /**
 117      * Status flag
 118      */
 119     private boolean initialized;
 120
 121     private final Gson gson = new GsonBuilder().create();
 122     private final ConfigurationAdmin configAdmin;
 123     private final OAuthFactory oAuthFactory;
 124
 125     private @Nullable OAuthClientService oAuthService;
 126
 127     /**
 128      * Constructor.
 129      *
 130      * @param cacheFolder Service cache folder
 131      */
 132     GoogleCloudAPI(ConfigurationAdmin configAdmin, OAuthFactory oAuthFactory, File cacheFolder) {
 133         this.configAdmin = configAdmin;
 134         this.oAuthFactory = oAuthFactory;
 135         this.cacheFolder = cacheFolder;
 136     }
 137
 138     /**
 139      * Configuration update.
 140      *
 141      * @param config New configuration.
 142      */
 143     void setConfig(GoogleTTSConfig config) {
 144         this.config = config;
 145
 146         String clientId = config.clientId;
 147         String clientSecret = config.clientSecret;
 148         if (clientId != null && !clientId.isEmpty() && clientSecret != null && !clientSecret.isEmpty()) {
 149             try {
 150                 final OAuthClientService oAuthService = oAuthFactory.createOAuthClientService(
 151                         GoogleTTSService.SERVICE_PID, GCP_TOKEN_URI, GCP_AUTH_URI, clientId, clientSecret, GCP_SCOPE,
 152                         false);
 153                 this.oAuthService = oAuthService;
 154                 getAccessToken();
 155                 initialized = true;
 156                 initVoices();
 157             } catch (AuthenticationException | CommunicationException e) {
 158                 logger.warn("Error initializing Google Cloud TTS service: {}", e.getMessage());
 159                 oAuthService = null;
 160                 initialized = false;
 161                 voices.clear();
 162             }
 163         } else {
 164             oAuthService = null;
 165             initialized = false;
 166             voices.clear();
 167         }
 168
 169         // maintain cache
 170         if (config.purgeCache) {
 171             File[] files = cacheFolder.listFiles();
 172             if (files != null && files.length > 0) {
 173                 Arrays.stream(files).forEach(File::delete);
 174             }
 175             logger.debug("Cache purged.");
 176         }
 177     }
 178
 179     /**
 180      * Fetches the OAuth2 tokens from Google Cloud Platform if the auth-code is set in the configuration. If successful
 181      * the auth-code will be removed from the configuration.
 182      *
 183      * @throws AuthenticationException
 184      * @throws CommunicationException
 185      */
 186     @SuppressWarnings("null")
 187     private void getAccessToken() throws AuthenticationException, CommunicationException {
 188         String authcode = config.authcode;
 189         if (authcode != null && !authcode.isEmpty()) {
 190             logger.debug("Trying to get access and refresh tokens.");
 191             try {
 192                 AccessTokenResponse response = oAuthService.getAccessTokenResponseByAuthorizationCode(authcode,
 193                         GCP_REDIRECT_URI);
 194                 if (response.getRefreshToken() == null || response.getRefreshToken().isEmpty()) {
 195                     throw new AuthenticationException("Error fetching refresh token. Please reauthorize");
 196                 }
 197             } catch (OAuthException | OAuthResponseException e) {
 198                 logger.debug("Error fetching access token: {}", e.getMessage(), e);
 199                 throw new AuthenticationException(
 200                         "Error fetching access token. Invalid authcode? Please generate a new one.");
 201             } catch (IOException e) {
 202                 throw new CommunicationException(
 203                         String.format("An unexpected IOException occurred: %s", e.getMessage()));
 204             }
 205
 206             config.authcode = null;
 207
 208             try {
 209                 Configuration serviceConfig = configAdmin.getConfiguration(GoogleTTSService.SERVICE_PID);
 210                 Dictionary<String, Object> configProperties = serviceConfig.getProperties();
 211                 if (configProperties != null) {
 212                     configProperties.put(GoogleTTSService.PARAM_AUTHCODE, "");
 213                     serviceConfig.update(configProperties);
 214                 }
 215             } catch (IOException e) {
 216                 // should not happen
 217                 logger.warn(
 218                         "Failed to update configuration for Google Cloud TTS service. Please clear the 'authcode' configuration parameter manualy.");
 219             }
 220         }
 221     }
 222
 223     @SuppressWarnings("null")
 224     private String getAuthorizationHeader() throws AuthenticationException, CommunicationException {
 225         final AccessTokenResponse accessTokenResponse;
 226         try {
 227             accessTokenResponse = oAuthService.getAccessTokenResponse();
 228         } catch (OAuthException | OAuthResponseException e) {
 229             logger.debug("Error fetching access token: {}", e.getMessage(), e);
 230             throw new AuthenticationException(
 231                     "Error fetching access token. Invalid authcode? Please generate a new one.");
 232         } catch (IOException e) {
 233             throw new CommunicationException(String.format("An unexpected IOException occurred: %s", e.getMessage()));
 234         }
 235         if (accessTokenResponse == null || accessTokenResponse.getAccessToken() == null
 236                 || accessTokenResponse.getAccessToken().isEmpty()) {
 237             throw new AuthenticationException("No access token. Is this thing authorized?");
 238         }
 239         if (accessTokenResponse.getRefreshToken() == null || accessTokenResponse.getRefreshToken().isEmpty()) {
 240             throw new AuthenticationException("No refresh token. Please reauthorize");
 241         }
 242         return BEARER + accessTokenResponse.getAccessToken();
 243     }
 244
 245     /**
 246      * Loads supported audio formats
 247      *
 248      * @return Set of audio formats
 249      */
 250     Set<String> getSupportedAudioFormats() {
 251         Set<String> formats = new HashSet<>();
 252         for (AudioEncoding audioEncoding : AudioEncoding.values()) {
 253             if (audioEncoding != AudioEncoding.AUDIO_ENCODING_UNSPECIFIED) {
 254                 formats.add(audioEncoding.toString());
 255             }
 256         }
 257         return formats;
 258     }
 259
 260     /**
 261      * Supported locales.
 262      *
 263      * @return Set of locales
 264      */
 265     Set<Locale> getSupportedLocales() {
 266         return voices.keySet();
 267     }
 268
 269     /**
 270      * Supported voices for locale.
 271      *
 272      * @param locale Locale
 273      * @return Set of voices
 274      */
 275     Set<GoogleTTSVoice> getVoicesForLocale(Locale locale) {
 276         Set<GoogleTTSVoice> localeVoices = voices.get(locale);
 277         return localeVoices != null ? localeVoices : Set.of();
 278     }
 279
 280     /**
 281      * Google API call to load locales and voices.
 282      *
 283      * @throws AuthenticationException
 284      * @throws CommunicationException
 285      */
 286     private void initVoices() throws AuthenticationException, CommunicationException {
 287         if (oAuthService != null) {
 288             voices.clear();
 289             for (GoogleTTSVoice voice : listVoices()) {
 290                 Locale locale = voice.getLocale();
 291                 Set<GoogleTTSVoice> localeVoices;
 292                 if (!voices.containsKey(locale)) {
 293                     localeVoices = new HashSet<>();
 294                     voices.put(locale, localeVoices);
 295                 } else {
 296                     localeVoices = voices.get(locale);
 297                 }
 298                 localeVoices.add(voice);
 299             }
 300         } else {
 301             logger.error("Google client is not initialized!");
 302         }
 303     }
 304
 305     @SuppressWarnings("null")
 306     private List<GoogleTTSVoice> listVoices() throws AuthenticationException, CommunicationException {
 307         HttpRequestBuilder builder = HttpRequestBuilder.getFrom(LIST_VOICES_URL)
 308                 .withHeader(HttpHeader.AUTHORIZATION.name(), getAuthorizationHeader());
 309
 310         try {
 311             ListVoicesResponse listVoicesResponse = gson.fromJson(builder.getContentAsString(),
 312                     ListVoicesResponse.class);
 313
 314             if (listVoicesResponse == null || listVoicesResponse.getVoices() == null) {
 315                 return List.of();
 316             }
 317
 318             List<GoogleTTSVoice> result = new ArrayList<>();
 319             for (Voice voice : listVoicesResponse.getVoices()) {
 320                 for (String languageCode : voice.getLanguageCodes()) {
 321                     result.add(new GoogleTTSVoice(Locale.forLanguageTag(languageCode), voice.getName(),
 322                             voice.getSsmlGender().name()));
 323                 }
 324             }
 325             return result;
 326         } catch (JsonSyntaxException e) {
 327             // do nothing
 328         } catch (IOException e) {
 329             throw new CommunicationException(String.format("An unexpected IOException occurred: %s", e.getMessage()));
 330         }
 331         return List.of();
 332     }
 333
 334     /**
 335      * Converts audio format to Google parameters.
 336      *
 337      * @param codec Requested codec
 338      * @return String array of Google audio format and the file extension to use.
 339      */
 340     private String[] getFormatForCodec(String codec) {
 341         switch (codec) {
 342             case AudioFormat.CODEC_MP3:
 343                 return new String[] { AudioEncoding.MP3.toString(), "mp3" };
 344             case AudioFormat.CODEC_PCM_SIGNED:
 345                 return new String[] { AudioEncoding.LINEAR16.toString(), "wav" };
 346             default:
 347                 throw new IllegalArgumentException("Audio format " + codec + " is not yet supported");
 348         }
 349     }
 350
 351     public byte[] synthesizeSpeech(String text, GoogleTTSVoice voice, String codec) {
 352         String[] format = getFormatForCodec(codec);
 353         String fileNameInCache = getUniqueFilenameForText(text, voice.getTechnicalName());
 354         File audioFileInCache = new File(cacheFolder, fileNameInCache + "." + format[1]);
 355         try {
 356             // check if in cache
 357             if (audioFileInCache.exists()) {
 358                 logger.debug("Audio file {} was found in cache.", audioFileInCache.getName());
 359                 return Files.readAllBytes(audioFileInCache.toPath());
 360             }
 361
 362             // if not in cache, get audio data and put to cache
 363             byte[] audio = synthesizeSpeechByGoogle(text, voice, format[0]);
 364             if (audio != null) {
 365                 saveAudioAndTextToFile(text, audioFileInCache, audio, voice.getTechnicalName());
 366             }
 367             return audio;
 368         } catch (AuthenticationException | CommunicationException e) {
 369             logger.warn("Error initializing Google Cloud TTS service: {}", e.getMessage());
 370             oAuthService = null;
 371             initialized = false;
 372             voices.clear();
 373         } catch (FileNotFoundException e) {
 374             logger.warn("Could not write file {} to cache: {}", audioFileInCache, e.getMessage());
 375         } catch (IOException e) {
 376             logger.debug("An unexpected IOException occurred: {}", e.getMessage());
 377         }
 378         return null;
 379     }
 380
 381     /**
 382      * Create cache entry.
 383      *
 384      * @param text Converted text.
 385      * @param cacheFile Cache entry file.
 386      * @param audio Byte array of the audio.
 387      * @param voiceName Used voice
 388      * @throws FileNotFoundException
 389      * @throws IOException in case of file handling exceptions
 390      */
 391     private void saveAudioAndTextToFile(String text, File cacheFile, byte[] audio, String voiceName)
 392             throws IOException, FileNotFoundException {
 393         logger.debug("Caching audio file {}", cacheFile.getName());
 394         try (FileOutputStream audioFileOutputStream = new FileOutputStream(cacheFile)) {
 395             audioFileOutputStream.write(audio);
 396         }
 397
 398         // write text to file for transparency too
 399         // this allows to know which contents is in which audio file
 400         String textFileName = removeExtension(cacheFile.getName()) + ".txt";
 401         logger.debug("Caching text file {}", textFileName);
 402         try (FileOutputStream textFileOutputStream = new FileOutputStream(new File(cacheFolder, textFileName))) {
 403             // @formatter:off
 404             StringBuilder sb = new StringBuilder("Config: ")
 405                     .append(config.toConfigString())
 406                     .append(",voice=")
 407                     .append(voiceName)
 408                     .append(System.lineSeparator())
 409                     .append("Text: ")
 410                     .append(text)
 411                     .append(System.lineSeparator());
 412             // @formatter:on
 413             textFileOutputStream.write(sb.toString().getBytes(StandardCharsets.UTF_8));
 414         }
 415     }
 416
 417     /**
 418      * Removes the extension of a file name.
 419      *
 420      * @param fileName the file name to remove the extension of
 421      * @return the filename without the extension
 422      */
 423     private String removeExtension(String fileName) {
 424         int extensionPos = fileName.lastIndexOf(EXTENSION_SEPARATOR);
 425         int lastSeparator = Math.max(fileName.lastIndexOf(UNIX_SEPARATOR), fileName.lastIndexOf(WINDOWS_SEPARATOR));
 426         return lastSeparator > extensionPos ? fileName : fileName.substring(0, extensionPos);
 427     }
 428
 429     /**
 430      * Call Google service to synthesize the required text
 431      *
 432      * @param text Text to synthesize
 433      * @param voice Voice parameter
 434      * @param audioFormat Audio encoding format
 435      * @return Audio input stream or {@code null} when encoding exceptions occur
 436      * @throws AuthenticationException
 437      * @throws CommunicationException
 438      */
 439     @SuppressWarnings("null")
 440     private byte[] synthesizeSpeechByGoogle(String text, GoogleTTSVoice voice, String audioFormat)
 441             throws AuthenticationException, CommunicationException {
 442         AudioConfig audioConfig = new AudioConfig(AudioEncoding.valueOf(audioFormat), config.pitch, config.speakingRate,
 443                 config.volumeGainDb);
 444         SynthesisInput synthesisInput = new SynthesisInput(text);
 445         VoiceSelectionParams voiceSelectionParams = new VoiceSelectionParams(voice.getLocale().getLanguage(),
 446                 voice.getLabel(), SsmlVoiceGender.valueOf(voice.getSsmlGender()));
 447
 448         SynthesizeSpeechRequest request = new SynthesizeSpeechRequest(audioConfig, synthesisInput,
 449                 voiceSelectionParams);
 450
 451         HttpRequestBuilder builder = HttpRequestBuilder.postTo(SYTNHESIZE_SPEECH_URL)
 452                 .withHeader(HttpHeader.AUTHORIZATION.name(), getAuthorizationHeader())
 453                 .withContent(gson.toJson(request), MimeTypes.Type.APPLICATION_JSON.name());
 454
 455         try {
 456             SynthesizeSpeechResponse synthesizeSpeechResponse = gson.fromJson(builder.getContentAsString(),
 457                     SynthesizeSpeechResponse.class);
 458
 459             if (synthesizeSpeechResponse == null) {
 460                 return null;
 461             }
 462
 463             byte[] encodedBytes = synthesizeSpeechResponse.getAudioContent().getBytes(StandardCharsets.UTF_8);
 464             return Base64.getDecoder().decode(encodedBytes);
 465         } catch (JsonSyntaxException e) {
 466             // do nothing
 467         } catch (IOException e) {
 468             throw new CommunicationException(String.format("An unexpected IOException occurred: %s", e.getMessage()));
 469         }
 470         return null;
 471     }
 472
 473     /**
 474      * Gets a unique filename for a give text, by creating a MD5 hash of it. It
 475      * will be preceded by the locale.
 476      * <p>
 477      * Sample: "en-US_00a2653ac5f77063bc4ea2fee87318d3"
 478      */
 479     private String getUniqueFilenameForText(String text, String voiceName) {
 480         try {
 481             MessageDigest md = MessageDigest.getInstance("MD5");
 482             byte[] bytesOfMessage = (config.toConfigString() + text).getBytes(StandardCharsets.UTF_8);
 483             String fileNameHash = String.format("%032x", new BigInteger(1, md.digest(bytesOfMessage)));
 484             return voiceName + "_" + fileNameHash;
 485         } catch (NoSuchAlgorithmException e) {
 486             // should not happen
 487             logger.error("Could not create MD5 hash for '{}'", text, e);
 488             return null;
 489         }
 490     }
 491
 492     boolean isInitialized() {
 493         return initialized;
 494     }
 495 }