git.basschouten.com Git - openhab-addons.git/blob

   1 /**
   2  * Copyright (c) 2010-2022 Contributors to the openHAB project
   3  *
   4  * See the NOTICE file(s) distributed with this work for additional
   5  * information.
   6  *
   7  * This program and the accompanying materials are made available under the
   8  * terms of the Eclipse Public License 2.0 which is available at
   9  * http://www.eclipse.org/legal/epl-2.0
  10  *
  11  * SPDX-License-Identifier: EPL-2.0
  12  */
  13 package org.openhab.voice.porcupineks.internal;
  14
  15 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_CATEGORY;
  16 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_ID;
  17 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_NAME;
  18 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_PID;
  19
  20 import java.io.BufferedInputStream;
  21 import java.io.BufferedOutputStream;
  22 import java.io.File;
  23 import java.io.FileOutputStream;
  24 import java.io.IOException;
  25 import java.io.InputStream;
  26 import java.io.OutputStream;
  27 import java.net.URL;
  28 import java.nio.ByteBuffer;
  29 import java.nio.ByteOrder;
  30 import java.nio.file.Files;
  31 import java.nio.file.Path;
  32 import java.util.Locale;
  33 import java.util.Map;
  34 import java.util.Set;
  35 import java.util.concurrent.Future;
  36 import java.util.concurrent.ScheduledExecutorService;
  37 import java.util.logging.Level;
  38
  39 import org.eclipse.jdt.annotation.NonNullByDefault;
  40 import org.eclipse.jdt.annotation.Nullable;
  41 import org.openhab.core.OpenHAB;
  42 import org.openhab.core.audio.AudioFormat;
  43 import org.openhab.core.audio.AudioStream;
  44 import org.openhab.core.common.ThreadPoolManager;
  45 import org.openhab.core.config.core.ConfigurableService;
  46 import org.openhab.core.config.core.Configuration;
  47 import org.openhab.core.voice.KSErrorEvent;
  48 import org.openhab.core.voice.KSException;
  49 import org.openhab.core.voice.KSListener;
  50 import org.openhab.core.voice.KSService;
  51 import org.openhab.core.voice.KSServiceHandle;
  52 import org.openhab.core.voice.KSpottedEvent;
  53 import org.osgi.framework.BundleContext;
  54 import org.osgi.framework.Constants;
  55 import org.osgi.service.component.ComponentContext;
  56 import org.osgi.service.component.annotations.Activate;
  57 import org.osgi.service.component.annotations.Component;
  58 import org.slf4j.Logger;
  59 import org.slf4j.LoggerFactory;
  60
  61 import ai.picovoice.porcupine.Porcupine;
  62 import ai.picovoice.porcupine.PorcupineException;
  63
  64 /**
  65  * The {@link PorcupineKSService} is a keyword spotting implementation based on porcupine.
  66  *
  67  * @author Miguel Álvarez - Initial contribution
  68  */
  69 @NonNullByDefault
  70 @Component(configurationPid = SERVICE_PID, property = Constants.SERVICE_PID + "=" + SERVICE_PID)
  71 @ConfigurableService(category = SERVICE_CATEGORY, label = SERVICE_NAME, description_uri = SERVICE_CATEGORY + ":"
  72         + SERVICE_ID)
  73 public class PorcupineKSService implements KSService {
  74     private static final String PORCUPINE_FOLDER = Path.of(OpenHAB.getUserDataFolder(), "porcupine").toString();
  75     private static final String EXTRACTION_FOLDER = Path.of(OpenHAB.getUserDataFolder(), "porcupine", "extracted")
  76             .toString();
  77     private final Logger logger = LoggerFactory.getLogger(PorcupineKSService.class);
  78     private final ScheduledExecutorService executor = ThreadPoolManager.getScheduledPool("OH-voice-porcupineks");
  79     private PorcupineKSConfiguration config = new PorcupineKSConfiguration();
  80     private boolean loop = false;
  81     private @Nullable BundleContext bundleContext;
  82
  83     static {
  84         Logger logger = LoggerFactory.getLogger(PorcupineKSService.class);
  85         File directory = new File(PORCUPINE_FOLDER);
  86         if (!directory.exists()) {
  87             if (directory.mkdir()) {
  88                 logger.info("porcupine dir created {}", PORCUPINE_FOLDER);
  89             }
  90         }
  91         File childDirectory = new File(EXTRACTION_FOLDER);
  92         if (!childDirectory.exists()) {
  93             if (childDirectory.mkdir()) {
  94                 logger.info("porcupine extraction file dir created {}", EXTRACTION_FOLDER);
  95             }
  96         }
  97     }
  98
  99     @Activate
 100     protected void activate(ComponentContext componentContext, Map<String, Object> config) {
 101         this.config = new Configuration(config).as(PorcupineKSConfiguration.class);
 102         this.bundleContext = componentContext.getBundleContext();
 103         if (this.config.apiKey.isBlank()) {
 104             logger.warn("Missing pico voice api key to use Porcupine Keyword Spotter");
 105         }
 106     }
 107
 108     private String prepareLib(BundleContext bundleContext, String path) throws IOException {
 109         if (!path.contains("porcupine" + File.separator)) {
 110             // this should never happen
 111             throw new IOException("Path is not pointing to porcupine bundle files " + path);
 112         }
 113         // get a path relative to the porcupine bundle folder
 114         String relativePath;
 115         if (path.startsWith("porcupine" + File.separator)) {
 116             relativePath = path;
 117         } else {
 118             relativePath = path.substring(path.lastIndexOf(File.separator + "porcupine" + File.separator) + 1);
 119         }
 120         File localFile = new File(EXTRACTION_FOLDER,
 121                 relativePath.substring(relativePath.lastIndexOf(File.separator) + 1));
 122         if (!localFile.exists()) {
 123             if (File.separator.equals("\\")) {
 124                 // bundle requires unix path separator
 125                 logger.debug("use unix path separator");
 126                 relativePath = relativePath.replace("\\", "/");
 127             }
 128             URL porcupineResource = bundleContext.getBundle().getEntry(relativePath);
 129             logger.debug("extracting binary {} from bundle to extraction folder", relativePath);
 130             if (porcupineResource == null) {
 131                 throw new IOException("Missing bundle file: " + relativePath);
 132             }
 133             extractFromBundle(porcupineResource, localFile);
 134         } else {
 135             logger.debug("binary {} already extracted", relativePath);
 136         }
 137         return localFile.toString();
 138     }
 139
 140     private void extractFromBundle(URL resourceUrl, File targetFile) throws IOException {
 141         InputStream in = new BufferedInputStream(resourceUrl.openStream());
 142         OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile));
 143         byte[] buffer = new byte[1024];
 144         int lengthRead;
 145         while ((lengthRead = in.read(buffer)) > 0) {
 146             out.write(buffer, 0, lengthRead);
 147             out.flush();
 148         }
 149         in.close();
 150         out.close();
 151     }
 152
 153     @Override
 154     public String getId() {
 155         return SERVICE_ID;
 156     }
 157
 158     @Override
 159     public String getLabel(@Nullable Locale locale) {
 160         return SERVICE_NAME;
 161     }
 162
 163     @Override
 164     public Set<Locale> getSupportedLocales() {
 165         return Set.of(Locale.ENGLISH, new Locale("es"), Locale.FRENCH, Locale.GERMAN);
 166     }
 167
 168     @Override
 169     public Set<AudioFormat> getSupportedFormats() {
 170         return Set
 171                 .of(new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, 16, null, 16000L));
 172     }
 173
 174     @Override
 175     public KSServiceHandle spot(KSListener ksListener, AudioStream audioStream, Locale locale, String keyword)
 176             throws KSException {
 177         Porcupine porcupine;
 178         if (config.apiKey.isBlank()) {
 179             throw new KSException("Missing pico voice api key");
 180         }
 181         BundleContext bundleContext = this.bundleContext;
 182         if (bundleContext == null) {
 183             throw new KSException("Missing bundle context");
 184         }
 185         try {
 186             porcupine = initPorcupine(bundleContext, locale, keyword);
 187         } catch (PorcupineException | IOException e) {
 188             throw new KSException(e);
 189         }
 190         Future<?> scheduledTask = executor.submit(() -> processInBackground(porcupine, ksListener, audioStream));
 191         return new KSServiceHandle() {
 192             @Override
 193             public void abort() {
 194                 logger.debug("stopping service");
 195                 loop = false;
 196                 try {
 197                     Thread.sleep(100);
 198                 } catch (InterruptedException e) {
 199                 }
 200                 scheduledTask.cancel(true);
 201             }
 202         };
 203     }
 204
 205     private Porcupine initPorcupine(BundleContext bundleContext, Locale locale, String keyword)
 206             throws IOException, PorcupineException {
 207         // Suppress library logs
 208         java.util.logging.Logger globalJavaLogger = java.util.logging.Logger
 209                 .getLogger(java.util.logging.Logger.GLOBAL_LOGGER_NAME);
 210         Level currentGlobalLogLevel = globalJavaLogger.getLevel();
 211         globalJavaLogger.setLevel(java.util.logging.Level.OFF);
 212         String bundleLibraryPath = Porcupine.LIBRARY_PATH;
 213         if (bundleLibraryPath == null) {
 214             throw new PorcupineException("Unsupported environment, ensure Porcupine is supported by your system");
 215         }
 216         String libraryPath = prepareLib(bundleContext, bundleLibraryPath);
 217         String alternativeModelPath = getAlternativeModelPath(bundleContext, locale);
 218         String modelPath = alternativeModelPath != null ? alternativeModelPath
 219                 : prepareLib(bundleContext, Porcupine.MODEL_PATH);
 220         String keywordPath = getKeywordResourcePath(bundleContext, keyword, alternativeModelPath == null);
 221         logger.debug("Porcupine library path: {}", libraryPath);
 222         logger.debug("Porcupine model path: {}", modelPath);
 223         logger.debug("Porcupine keyword path: {}", keywordPath);
 224         logger.debug("Porcupine sensitivity: {}", config.sensitivity);
 225         try {
 226             return new Porcupine(config.apiKey, libraryPath, modelPath, new String[] { keywordPath },
 227                     new float[] { config.sensitivity });
 228         } finally {
 229             // restore log level
 230             globalJavaLogger.setLevel(currentGlobalLogLevel);
 231         }
 232     }
 233
 234     private String getPorcupineEnv() {
 235         // get porcupine env from resolved library path
 236         String searchTerm = "lib" + File.separator + "java" + File.separator;
 237         String env = Porcupine.LIBRARY_PATH.substring(Porcupine.LIBRARY_PATH.indexOf(searchTerm) + searchTerm.length());
 238         env = env.substring(0, env.indexOf(File.separator));
 239         return env;
 240     }
 241
 242     private @Nullable String getAlternativeModelPath(BundleContext bundleContext, Locale locale) throws IOException {
 243         String modelPath = null;
 244         if (locale.getLanguage().equals(Locale.GERMAN.getLanguage())) {
 245             Path dePath = Path.of(PORCUPINE_FOLDER, "porcupine_params_de.pv");
 246             if (Files.exists(dePath)) {
 247                 modelPath = dePath.toString();
 248             } else {
 249                 logger.warn(
 250                         "You can provide a specific model for de language in {}, english language model will be used",
 251                         PORCUPINE_FOLDER);
 252             }
 253         } else if (locale.getLanguage().equals(Locale.FRENCH.getLanguage())) {
 254             Path frPath = Path.of(PORCUPINE_FOLDER, "porcupine_params_fr.pv");
 255             if (Files.exists(frPath)) {
 256                 modelPath = frPath.toString();
 257             } else {
 258                 logger.warn(
 259                         "You can provide a specific model for fr language in {}, english language model will be used",
 260                         PORCUPINE_FOLDER);
 261             }
 262         } else if (locale.getLanguage().equals("es")) {
 263             Path esPath = Path.of(PORCUPINE_FOLDER, "porcupine_params_es.pv");
 264             if (Files.exists(esPath)) {
 265                 modelPath = esPath.toString();
 266             } else {
 267                 logger.warn(
 268                         "You can provide a specific model for es language in {}, english language model will be used",
 269                         PORCUPINE_FOLDER);
 270             }
 271         }
 272         return modelPath;
 273     }
 274
 275     private String getKeywordResourcePath(BundleContext bundleContext, String keyWord, boolean allowBuildIn)
 276             throws IOException {
 277         String localKeywordFile = keyWord.toLowerCase().replace(" ", "_") + ".ppn";
 278         Path localKeywordPath = Path.of(PORCUPINE_FOLDER, localKeywordFile);
 279         if (Files.exists(localKeywordPath)) {
 280             return localKeywordPath.toString();
 281         }
 282         if (allowBuildIn) {
 283             try {
 284                 Porcupine.BuiltInKeyword.valueOf(keyWord.toUpperCase().replace(" ", "_"));
 285             } catch (IllegalArgumentException e) {
 286                 throw new IllegalArgumentException(
 287                         "Unable to find model file for configured wake word neither is build-in. Should be at "
 288                                 + localKeywordPath);
 289             }
 290             String env = getPorcupineEnv();
 291             String keywordPath = "porcupine/resources/keyword_files/" + env + "/" + keyWord.replace(" ", "_") + "_"
 292                     + env + ".ppn";
 293             return prepareLib(bundleContext, keywordPath);
 294         } else {
 295             throw new IllegalArgumentException(
 296                     "Unable to find model file for configured wake word; there are no build-in wake words for your language. Should be at "
 297                             + localKeywordPath);
 298         }
 299     }
 300
 301     private void processInBackground(Porcupine porcupine, KSListener ksListener, AudioStream audioStream) {
 302         int numBytesRead;
 303         // buffers for processing audio
 304         int frameLength = porcupine.getFrameLength();
 305         ByteBuffer captureBuffer = ByteBuffer.allocate(frameLength * 2);
 306         captureBuffer.order(ByteOrder.LITTLE_ENDIAN);
 307         short[] porcupineBuffer = new short[frameLength];
 308         this.loop = true;
 309         while (loop) {
 310             try {
 311                 // read a buffer of audio
 312                 numBytesRead = audioStream.read(captureBuffer.array(), 0, captureBuffer.capacity());
 313                 if (!loop) {
 314                     break;
 315                 }
 316                 // don't pass to porcupine if we don't have a full buffer
 317                 if (numBytesRead != frameLength * 2) {
 318                     Thread.sleep(100);
 319                     continue;
 320                 }
 321                 // copy into 16-bit buffer
 322                 captureBuffer.asShortBuffer().get(porcupineBuffer);
 323                 // process with porcupine
 324                 int result = porcupine.process(porcupineBuffer);
 325                 if (result >= 0) {
 326                     logger.debug("keyword detected!");
 327                     ksListener.ksEventReceived(new KSpottedEvent());
 328                 }
 329             } catch (IOException | PorcupineException | InterruptedException e) {
 330                 String errorMessage = e.getMessage();
 331                 ksListener.ksEventReceived(new KSErrorEvent(errorMessage != null ? errorMessage : "Unexpected error"));
 332             }
 333         }
 334         porcupine.delete();
 335         logger.debug("Porcupine stopped");
 336     }
 337 }