git.basschouten.com Git - openhab-addons.git/blob

   1 /**
   2  * Copyright (c) 2010-2021 Contributors to the openHAB project
   3  *
   4  * See the NOTICE file(s) distributed with this work for additional
   5  * information.
   6  *
   7  * This program and the accompanying materials are made available under the
   8  * terms of the Eclipse Public License 2.0 which is available at
   9  * http://www.eclipse.org/legal/epl-2.0
  10  *
  11  * SPDX-License-Identifier: EPL-2.0
  12  */
  13 package org.openhab.voice.googletts.internal;
  14
  15 import java.io.File;
  16 import java.io.FileNotFoundException;
  17 import java.io.FileOutputStream;
  18 import java.io.IOException;
  19 import java.math.BigInteger;
  20 import java.nio.charset.StandardCharsets;
  21 import java.nio.file.Files;
  22 import java.security.MessageDigest;
  23 import java.security.NoSuchAlgorithmException;
  24 import java.util.ArrayList;
  25 import java.util.Arrays;
  26 import java.util.Base64;
  27 import java.util.Dictionary;
  28 import java.util.HashMap;
  29 import java.util.HashSet;
  30 import java.util.List;
  31 import java.util.Locale;
  32 import java.util.Map;
  33 import java.util.Set;
  34
  35 import org.eclipse.jdt.annotation.Nullable;
  36 import org.eclipse.jetty.http.HttpHeader;
  37 import org.eclipse.jetty.http.MimeTypes;
  38 import org.openhab.core.audio.AudioFormat;
  39 import org.openhab.core.auth.AuthenticationException;
  40 import org.openhab.core.auth.client.oauth2.AccessTokenResponse;
  41 import org.openhab.core.auth.client.oauth2.OAuthClientService;
  42 import org.openhab.core.auth.client.oauth2.OAuthException;
  43 import org.openhab.core.auth.client.oauth2.OAuthFactory;
  44 import org.openhab.core.auth.client.oauth2.OAuthResponseException;
  45 import org.openhab.core.i18n.CommunicationException;
  46 import org.openhab.core.io.net.http.HttpRequestBuilder;
  47 import org.openhab.voice.googletts.internal.dto.AudioConfig;
  48 import org.openhab.voice.googletts.internal.dto.AudioEncoding;
  49 import org.openhab.voice.googletts.internal.dto.ListVoicesResponse;
  50 import org.openhab.voice.googletts.internal.dto.SsmlVoiceGender;
  51 import org.openhab.voice.googletts.internal.dto.SynthesisInput;
  52 import org.openhab.voice.googletts.internal.dto.SynthesizeSpeechRequest;
  53 import org.openhab.voice.googletts.internal.dto.SynthesizeSpeechResponse;
  54 import org.openhab.voice.googletts.internal.dto.Voice;
  55 import org.openhab.voice.googletts.internal.dto.VoiceSelectionParams;
  56 import org.osgi.service.cm.Configuration;
  57 import org.osgi.service.cm.ConfigurationAdmin;
  58 import org.slf4j.Logger;
  59 import org.slf4j.LoggerFactory;
  60
  61 import com.google.gson.Gson;
  62 import com.google.gson.GsonBuilder;
  63 import com.google.gson.JsonSyntaxException;
  64
  65 /**
  66  * Google Cloud TTS API call implementation.
  67  *
  68  * @author Gabor Bicskei - Initial contribution and API
  69  */
  70 class GoogleCloudAPI {
  71
  72     private static final char EXTENSION_SEPARATOR = '.';
  73     private static final char UNIX_SEPARATOR = '/';
  74     private static final char WINDOWS_SEPARATOR = '\\';
  75
  76     private static final String BEARER = "Bearer ";
  77
  78     private static final String GCP_AUTH_URI = "https://accounts.google.com/o/oauth2/auth";
  79     private static final String GCP_TOKEN_URI = "https://accounts.google.com/o/oauth2/token";
  80     private static final String GCP_REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob";
  81     /**
  82      * Google Cloud Platform authorization scope
  83      */
  84     private static final String GCP_SCOPE = "https://www.googleapis.com/auth/cloud-platform";
  85
  86     /**
  87      * URL used for retrieving the list of available voices
  88      */
  89     private static final String LIST_VOICES_URL = "https://texttospeech.googleapis.com/v1/voices";
  90
  91     /**
  92      * URL used for synthesizing text to speech
  93      */
  94     private static final String SYTNHESIZE_SPEECH_URL = "https://texttospeech.googleapis.com/v1/text:synthesize";
  95
  96     /**
  97      * Logger
  98      */
  99     private final Logger logger = LoggerFactory.getLogger(GoogleCloudAPI.class);
 100
 101     /**
 102      * Supported voices and locales
 103      */
 104     private final Map<Locale, Set<GoogleTTSVoice>> voices = new HashMap<>();
 105
 106     /**
 107      * Cache folder
 108      */
 109     private File cacheFolder;
 110
 111     /**
 112      * Configuration
 113      */
 114     private @Nullable GoogleTTSConfig config;
 115
 116     /**
 117      * Status flag
 118      */
 119     private boolean initialized;
 120
 121     private final Gson gson = new GsonBuilder().create();
 122     private final ConfigurationAdmin configAdmin;
 123     private final OAuthFactory oAuthFactory;
 124
 125     private @Nullable OAuthClientService oAuthService;
 126
 127     /**
 128      * Constructor.
 129      *
 130      * @param cacheFolder Service cache folder
 131      */
 132     GoogleCloudAPI(ConfigurationAdmin configAdmin, OAuthFactory oAuthFactory, File cacheFolder) {
 133         this.configAdmin = configAdmin;
 134         this.oAuthFactory = oAuthFactory;
 135         this.cacheFolder = cacheFolder;
 136     }
 137
 138     /**
 139      * Configuration update.
 140      *
 141      * @param config New configuration.
 142      */
 143     void setConfig(GoogleTTSConfig config) {
 144         this.config = config;
 145
 146         String clientId = config.clientId;
 147         String clientSecret = config.clientSecret;
 148         if (clientId != null && !clientId.isEmpty() && clientSecret != null && !clientSecret.isEmpty()) {
 149             try {
 150                 final OAuthClientService oAuthService = oAuthFactory.createOAuthClientService(
 151                         GoogleTTSService.SERVICE_PID, GCP_TOKEN_URI, GCP_AUTH_URI, clientId, clientSecret, GCP_SCOPE,
 152                         false);
 153                 this.oAuthService = oAuthService;
 154                 getAccessToken();
 155                 initialized = true;
 156                 initVoices();
 157             } catch (AuthenticationException | CommunicationException e) {
 158                 logger.warn("Error initializing Google Cloud TTS service: {}", e.getMessage());
 159                 oAuthService = null;
 160                 initialized = false;
 161                 voices.clear();
 162             }
 163         } else {
 164             oAuthService = null;
 165             initialized = false;
 166             voices.clear();
 167         }
 168
 169         // maintain cache
 170         if (config.purgeCache) {
 171             File[] files = cacheFolder.listFiles();
 172             if (files != null && files.length > 0) {
 173                 Arrays.stream(files).forEach(File::delete);
 174             }
 175             logger.debug("Cache purged.");
 176         }
 177     }
 178
 179     /**
 180      * Fetches the OAuth2 tokens from Google Cloud Platform if the auth-code is set in the configuration. If successful
 181      * the auth-code will be removed from the configuration.
 182      *
 183      * @throws AuthenticationException
 184      * @throws CommunicationException
 185      */
 186     @SuppressWarnings("null")
 187     private void getAccessToken() throws AuthenticationException, CommunicationException {
 188         String authcode = config.authcode;
 189         if (authcode != null && !authcode.isEmpty()) {
 190             logger.debug("Trying to get access and refresh tokens.");
 191             try {
 192                 oAuthService.getAccessTokenResponseByAuthorizationCode(authcode, GCP_REDIRECT_URI);
 193             } catch (OAuthException | OAuthResponseException e) {
 194                 logger.debug("Error fetching access token: {}", e.getMessage(), e);
 195                 throw new AuthenticationException(
 196                         "Error fetching access token. Invalid authcode? Please generate a new one.");
 197             } catch (IOException e) {
 198                 throw new CommunicationException(
 199                         String.format("An unexpected IOException occurred: %s", e.getMessage()));
 200             }
 201
 202             config.authcode = null;
 203
 204             try {
 205                 Configuration serviceConfig = configAdmin.getConfiguration(GoogleTTSService.SERVICE_PID);
 206                 Dictionary<String, Object> configProperties = serviceConfig.getProperties();
 207                 if (configProperties != null) {
 208                     configProperties.put(GoogleTTSService.PARAM_AUTHCODE, "");
 209                     serviceConfig.update(configProperties);
 210                 }
 211             } catch (IOException e) {
 212                 // should not happen
 213                 logger.warn(
 214                         "Failed to update configuration for Google Cloud TTS service. Please clear the 'authcode' configuration parameter manualy.");
 215             }
 216         }
 217     }
 218
 219     @SuppressWarnings("null")
 220     private String getAuthorizationHeader() throws AuthenticationException, CommunicationException {
 221         final AccessTokenResponse accessTokenResponse;
 222         try {
 223             accessTokenResponse = oAuthService.getAccessTokenResponse();
 224         } catch (OAuthException | OAuthResponseException e) {
 225             logger.debug("Error fetching access token: {}", e.getMessage(), e);
 226             throw new AuthenticationException(
 227                     "Error fetching access token. Invalid authcode? Please generate a new one.");
 228         } catch (IOException e) {
 229             throw new CommunicationException(String.format("An unexpected IOException occurred: %s", e.getMessage()));
 230         }
 231         if (accessTokenResponse == null || accessTokenResponse.getAccessToken() == null
 232                 || accessTokenResponse.getAccessToken().isEmpty()) {
 233             throw new AuthenticationException("No access token. Is this thing authorized?");
 234         }
 235         return BEARER + accessTokenResponse.getAccessToken();
 236     }
 237
 238     /**
 239      * Loads supported audio formats
 240      *
 241      * @return Set of audio formats
 242      */
 243     Set<String> getSupportedAudioFormats() {
 244         Set<String> formats = new HashSet<>();
 245         for (AudioEncoding audioEncoding : AudioEncoding.values()) {
 246             if (audioEncoding != AudioEncoding.AUDIO_ENCODING_UNSPECIFIED) {
 247                 formats.add(audioEncoding.toString());
 248             }
 249         }
 250         return formats;
 251     }
 252
 253     /**
 254      * Supported locales.
 255      *
 256      * @return Set of locales
 257      */
 258     Set<Locale> getSupportedLocales() {
 259         return voices.keySet();
 260     }
 261
 262     /**
 263      * Supported voices for locale.
 264      *
 265      * @param locale Locale
 266      * @return Set of voices
 267      */
 268     Set<GoogleTTSVoice> getVoicesForLocale(Locale locale) {
 269         Set<GoogleTTSVoice> localeVoices = voices.get(locale);
 270         return localeVoices != null ? localeVoices : Set.of();
 271     }
 272
 273     /**
 274      * Google API call to load locales and voices.
 275      *
 276      * @throws AuthenticationException
 277      * @throws CommunicationException
 278      */
 279     private void initVoices() throws AuthenticationException, CommunicationException {
 280         if (oAuthService != null) {
 281             voices.clear();
 282             for (GoogleTTSVoice voice : listVoices()) {
 283                 Locale locale = voice.getLocale();
 284                 Set<GoogleTTSVoice> localeVoices;
 285                 if (!voices.containsKey(locale)) {
 286                     localeVoices = new HashSet<>();
 287                     voices.put(locale, localeVoices);
 288                 } else {
 289                     localeVoices = voices.get(locale);
 290                 }
 291                 localeVoices.add(voice);
 292             }
 293         } else {
 294             logger.error("Google client is not initialized!");
 295         }
 296     }
 297
 298     @SuppressWarnings("null")
 299     private List<GoogleTTSVoice> listVoices() throws AuthenticationException, CommunicationException {
 300         HttpRequestBuilder builder = HttpRequestBuilder.getFrom(LIST_VOICES_URL)
 301                 .withHeader(HttpHeader.AUTHORIZATION.name(), getAuthorizationHeader());
 302
 303         try {
 304             ListVoicesResponse listVoicesResponse = gson.fromJson(builder.getContentAsString(),
 305                     ListVoicesResponse.class);
 306
 307             if (listVoicesResponse == null || listVoicesResponse.getVoices() == null) {
 308                 return List.of();
 309             }
 310
 311             List<GoogleTTSVoice> result = new ArrayList<>();
 312             for (Voice voice : listVoicesResponse.getVoices()) {
 313                 for (String languageCode : voice.getLanguageCodes()) {
 314                     result.add(new GoogleTTSVoice(Locale.forLanguageTag(languageCode), voice.getName(),
 315                             voice.getSsmlGender().name()));
 316                 }
 317             }
 318             return result;
 319         } catch (JsonSyntaxException e) {
 320             // do nothing
 321         } catch (IOException e) {
 322             throw new CommunicationException(String.format("An unexpected IOException occurred: %s", e.getMessage()));
 323         }
 324         return List.of();
 325     }
 326
 327     /**
 328      * Converts audio format to Google parameters.
 329      *
 330      * @param codec Requested codec
 331      * @return String array of Google audio format and the file extension to use.
 332      */
 333     private String[] getFormatForCodec(String codec) {
 334         switch (codec) {
 335             case AudioFormat.CODEC_MP3:
 336                 return new String[] { AudioEncoding.MP3.toString(), "mp3" };
 337             case AudioFormat.CODEC_PCM_SIGNED:
 338                 return new String[] { AudioEncoding.LINEAR16.toString(), "wav" };
 339             default:
 340                 throw new IllegalArgumentException("Audio format " + codec + " is not yet supported");
 341         }
 342     }
 343
 344     public byte[] synthesizeSpeech(String text, GoogleTTSVoice voice, String codec) {
 345         String[] format = getFormatForCodec(codec);
 346         String fileNameInCache = getUniqueFilenameForText(text, voice.getTechnicalName());
 347         File audioFileInCache = new File(cacheFolder, fileNameInCache + "." + format[1]);
 348         try {
 349             // check if in cache
 350             if (audioFileInCache.exists()) {
 351                 logger.debug("Audio file {} was found in cache.", audioFileInCache.getName());
 352                 return Files.readAllBytes(audioFileInCache.toPath());
 353             }
 354
 355             // if not in cache, get audio data and put to cache
 356             byte[] audio = synthesizeSpeechByGoogle(text, voice, format[0]);
 357             if (audio != null) {
 358                 saveAudioAndTextToFile(text, audioFileInCache, audio, voice.getTechnicalName());
 359             }
 360             return audio;
 361         } catch (AuthenticationException | CommunicationException e) {
 362             logger.warn("Error initializing Google Cloud TTS service: {}", e.getMessage());
 363             oAuthService = null;
 364             initialized = false;
 365             voices.clear();
 366         } catch (FileNotFoundException e) {
 367             logger.warn("Could not write file {} to cache: {}", audioFileInCache, e.getMessage());
 368         } catch (IOException e) {
 369             logger.debug("An unexpected IOException occurred: {}", e.getMessage());
 370         }
 371         return null;
 372     }
 373
 374     /**
 375      * Create cache entry.
 376      *
 377      * @param text Converted text.
 378      * @param cacheFile Cache entry file.
 379      * @param audio Byte array of the audio.
 380      * @param voiceName Used voice
 381      * @throws FileNotFoundException
 382      * @throws IOException in case of file handling exceptions
 383      */
 384     private void saveAudioAndTextToFile(String text, File cacheFile, byte[] audio, String voiceName)
 385             throws IOException, FileNotFoundException {
 386         logger.debug("Caching audio file {}", cacheFile.getName());
 387         try (FileOutputStream audioFileOutputStream = new FileOutputStream(cacheFile)) {
 388             audioFileOutputStream.write(audio);
 389         }
 390
 391         // write text to file for transparency too
 392         // this allows to know which contents is in which audio file
 393         String textFileName = removeExtension(cacheFile.getName()) + ".txt";
 394         logger.debug("Caching text file {}", textFileName);
 395         try (FileOutputStream textFileOutputStream = new FileOutputStream(new File(cacheFolder, textFileName))) {
 396             // @formatter:off
 397             StringBuilder sb = new StringBuilder("Config: ")
 398                     .append(config.toConfigString())
 399                     .append(",voice=")
 400                     .append(voiceName)
 401                     .append(System.lineSeparator())
 402                     .append("Text: ")
 403                     .append(text)
 404                     .append(System.lineSeparator());
 405             // @formatter:on
 406             textFileOutputStream.write(sb.toString().getBytes(StandardCharsets.UTF_8));
 407         }
 408     }
 409
 410     /**
 411      * Removes the extension of a file name.
 412      *
 413      * @param fileName the file name to remove the extension of
 414      * @return the filename without the extension
 415      */
 416     private String removeExtension(String fileName) {
 417         int extensionPos = fileName.lastIndexOf(EXTENSION_SEPARATOR);
 418         int lastSeparator = Math.max(fileName.lastIndexOf(UNIX_SEPARATOR), fileName.lastIndexOf(WINDOWS_SEPARATOR));
 419         return lastSeparator > extensionPos ? fileName : fileName.substring(0, extensionPos);
 420     }
 421
 422     /**
 423      * Call Google service to synthesize the required text
 424      *
 425      * @param text Text to synthesize
 426      * @param voice Voice parameter
 427      * @param audioFormat Audio encoding format
 428      * @return Audio input stream or {@code null} when encoding exceptions occur
 429      * @throws AuthenticationException
 430      * @throws CommunicationException
 431      */
 432     @SuppressWarnings("null")
 433     private byte[] synthesizeSpeechByGoogle(String text, GoogleTTSVoice voice, String audioFormat)
 434             throws AuthenticationException, CommunicationException {
 435         AudioConfig audioConfig = new AudioConfig(AudioEncoding.valueOf(audioFormat), config.pitch, config.speakingRate,
 436                 config.volumeGainDb);
 437         SynthesisInput synthesisInput = new SynthesisInput(text);
 438         VoiceSelectionParams voiceSelectionParams = new VoiceSelectionParams(voice.getLocale().getLanguage(),
 439                 voice.getLabel(), SsmlVoiceGender.valueOf(voice.getSsmlGender()));
 440
 441         SynthesizeSpeechRequest request = new SynthesizeSpeechRequest(audioConfig, synthesisInput,
 442                 voiceSelectionParams);
 443
 444         HttpRequestBuilder builder = HttpRequestBuilder.postTo(SYTNHESIZE_SPEECH_URL)
 445                 .withHeader(HttpHeader.AUTHORIZATION.name(), getAuthorizationHeader())
 446                 .withContent(gson.toJson(request), MimeTypes.Type.APPLICATION_JSON.name());
 447
 448         try {
 449             SynthesizeSpeechResponse synthesizeSpeechResponse = gson.fromJson(builder.getContentAsString(),
 450                     SynthesizeSpeechResponse.class);
 451
 452             if (synthesizeSpeechResponse == null) {
 453                 return null;
 454             }
 455
 456             byte[] encodedBytes = synthesizeSpeechResponse.getAudioContent().getBytes(StandardCharsets.UTF_8);
 457             return Base64.getDecoder().decode(encodedBytes);
 458         } catch (JsonSyntaxException e) {
 459             // do nothing
 460         } catch (IOException e) {
 461             throw new CommunicationException(String.format("An unexpected IOException occurred: %s", e.getMessage()));
 462         }
 463         return null;
 464     }
 465
 466     /**
 467      * Gets a unique filename for a give text, by creating a MD5 hash of it. It
 468      * will be preceded by the locale.
 469      * <p>
 470      * Sample: "en-US_00a2653ac5f77063bc4ea2fee87318d3"
 471      */
 472     private String getUniqueFilenameForText(String text, String voiceName) {
 473         try {
 474             MessageDigest md = MessageDigest.getInstance("MD5");
 475             byte[] bytesOfMessage = (config.toConfigString() + text).getBytes(StandardCharsets.UTF_8);
 476             String fileNameHash = String.format("%032x", new BigInteger(1, md.digest(bytesOfMessage)));
 477             return voiceName + "_" + fileNameHash;
 478         } catch (NoSuchAlgorithmException e) {
 479             // should not happen
 480             logger.error("Could not create MD5 hash for '{}'", text, e);
 481             return null;
 482         }
 483     }
 484
 485     boolean isInitialized() {
 486         return initialized;
 487     }
 488 }