git.basschouten.com Git - openhab-addons.git/blob

   1 /**
   2  * Copyright (c) 2010-2022 Contributors to the openHAB project
   3  *
   4  * See the NOTICE file(s) distributed with this work for additional
   5  * information.
   6  *
   7  * This program and the accompanying materials are made available under the
   8  * terms of the Eclipse Public License 2.0 which is available at
   9  * http://www.eclipse.org/legal/epl-2.0
  10  *
  11  * SPDX-License-Identifier: EPL-2.0
  12  */
  13 package org.openhab.voice.porcupineks.internal;
  14
  15 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_CATEGORY;
  16 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_ID;
  17 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_NAME;
  18 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_PID;
  19
  20 import java.io.BufferedInputStream;
  21 import java.io.BufferedOutputStream;
  22 import java.io.File;
  23 import java.io.FileOutputStream;
  24 import java.io.IOException;
  25 import java.io.InputStream;
  26 import java.io.OutputStream;
  27 import java.net.URL;
  28 import java.nio.ByteBuffer;
  29 import java.nio.ByteOrder;
  30 import java.nio.file.Files;
  31 import java.nio.file.Path;
  32 import java.util.Locale;
  33 import java.util.Map;
  34 import java.util.Set;
  35 import java.util.concurrent.Future;
  36 import java.util.concurrent.ScheduledExecutorService;
  37 import java.util.logging.Level;
  38
  39 import org.eclipse.jdt.annotation.NonNullByDefault;
  40 import org.eclipse.jdt.annotation.Nullable;
  41 import org.openhab.core.OpenHAB;
  42 import org.openhab.core.audio.AudioFormat;
  43 import org.openhab.core.audio.AudioStream;
  44 import org.openhab.core.common.ThreadPoolManager;
  45 import org.openhab.core.config.core.ConfigurableService;
  46 import org.openhab.core.config.core.Configuration;
  47 import org.openhab.core.voice.KSErrorEvent;
  48 import org.openhab.core.voice.KSException;
  49 import org.openhab.core.voice.KSListener;
  50 import org.openhab.core.voice.KSService;
  51 import org.openhab.core.voice.KSServiceHandle;
  52 import org.openhab.core.voice.KSpottedEvent;
  53 import org.osgi.framework.BundleContext;
  54 import org.osgi.framework.Constants;
  55 import org.osgi.service.component.ComponentContext;
  56 import org.osgi.service.component.annotations.Activate;
  57 import org.osgi.service.component.annotations.Component;
  58 import org.slf4j.Logger;
  59 import org.slf4j.LoggerFactory;
  60
  61 import ai.picovoice.porcupine.Porcupine;
  62 import ai.picovoice.porcupine.PorcupineException;
  63
  64 /**
  65  * The {@link PorcupineKSService} is a keyword spotting implementation based on porcupine.
  66  *
  67  * @author Miguel Álvarez - Initial contribution
  68  */
  69 @NonNullByDefault
  70 @Component(configurationPid = SERVICE_PID, property = Constants.SERVICE_PID + "=" + SERVICE_PID)
  71 @ConfigurableService(category = SERVICE_CATEGORY, label = SERVICE_NAME, description_uri = SERVICE_CATEGORY + ":"
  72         + SERVICE_ID)
  73 public class PorcupineKSService implements KSService {
  74     private static final String PORCUPINE_FOLDER = Path.of(OpenHAB.getUserDataFolder(), "porcupine").toString();
  75     private static final String EXTRACTION_FOLDER = Path.of(OpenHAB.getUserDataFolder(), "porcupine", "extracted")
  76             .toString();
  77     private final Logger logger = LoggerFactory.getLogger(PorcupineKSService.class);
  78     private final ScheduledExecutorService executor = ThreadPoolManager.getScheduledPool("OH-voice-porcupineks");
  79     private PorcupineKSConfiguration config = new PorcupineKSConfiguration();
  80     private boolean loop = false;
  81     private @Nullable BundleContext bundleContext;
  82
  83     static {
  84         Logger logger = LoggerFactory.getLogger(PorcupineKSService.class);
  85         File directory = new File(PORCUPINE_FOLDER);
  86         if (!directory.exists()) {
  87             if (directory.mkdir()) {
  88                 logger.info("porcupine dir created {}", PORCUPINE_FOLDER);
  89             }
  90         }
  91         File childDirectory = new File(EXTRACTION_FOLDER);
  92         if (!childDirectory.exists()) {
  93             if (childDirectory.mkdir()) {
  94                 logger.info("porcupine extraction file dir created {}", EXTRACTION_FOLDER);
  95             }
  96         }
  97     }
  98
  99     @Activate
 100     protected void activate(ComponentContext componentContext, Map<String, Object> config) {
 101         this.config = new Configuration(config).as(PorcupineKSConfiguration.class);
 102         this.bundleContext = componentContext.getBundleContext();
 103         if (this.config.apiKey.isBlank()) {
 104             logger.warn("Missing pico voice api key to use Porcupine Keyword Spotter");
 105         }
 106     }
 107
 108     private String prepareLib(BundleContext bundleContext, String path) throws IOException {
 109         if (!path.contains("porcupine" + File.separator)) {
 110             // this should never happen
 111             throw new IOException("Path is not pointing to porcupine bundle files " + path);
 112         }
 113         // get a path relative to the porcupine bundle folder
 114         String relativePath;
 115         if (path.startsWith("porcupine" + File.separator)) {
 116             relativePath = path;
 117         } else {
 118             relativePath = path.substring(path.lastIndexOf(File.separator + "porcupine" + File.separator) + 1);
 119         }
 120         File localFile = new File(EXTRACTION_FOLDER,
 121                 relativePath.substring(relativePath.lastIndexOf(File.separator) + 1));
 122         if (!localFile.exists()) {
 123             URL porcupineResource = bundleContext.getBundle().getEntry(relativePath);
 124             logger.debug("extracting binary {} from bundle to extraction folder", relativePath);
 125             extractFromBundle(porcupineResource, localFile);
 126         } else {
 127             logger.debug("binary {} already extracted", relativePath);
 128         }
 129         return localFile.toString();
 130     }
 131
 132     private void extractFromBundle(URL resourceUrl, File targetFile) throws IOException {
 133         InputStream in = new BufferedInputStream(resourceUrl.openStream());
 134         OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile));
 135         byte[] buffer = new byte[1024];
 136         int lengthRead;
 137         while ((lengthRead = in.read(buffer)) > 0) {
 138             out.write(buffer, 0, lengthRead);
 139             out.flush();
 140         }
 141         in.close();
 142         out.close();
 143     }
 144
 145     @Override
 146     public String getId() {
 147         return SERVICE_ID;
 148     }
 149
 150     @Override
 151     public String getLabel(@Nullable Locale locale) {
 152         return SERVICE_NAME;
 153     }
 154
 155     @Override
 156     public Set<Locale> getSupportedLocales() {
 157         return Set.of(Locale.ENGLISH, new Locale("es"), Locale.FRENCH, Locale.GERMAN);
 158     }
 159
 160     @Override
 161     public Set<AudioFormat> getSupportedFormats() {
 162         return Set
 163                 .of(new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, 16, null, 16000L));
 164     }
 165
 166     @Override
 167     public KSServiceHandle spot(KSListener ksListener, AudioStream audioStream, Locale locale, String keyword)
 168             throws KSException {
 169         Porcupine porcupine;
 170         if (config.apiKey.isBlank()) {
 171             throw new KSException("Missing pico voice api key");
 172         }
 173         BundleContext bundleContext = this.bundleContext;
 174         if (bundleContext == null) {
 175             throw new KSException("Missing bundle context");
 176         }
 177         try {
 178             porcupine = initPorcupine(bundleContext, locale, keyword);
 179         } catch (PorcupineException | IOException e) {
 180             throw new KSException(e);
 181         }
 182         Future<?> scheduledTask = executor.submit(() -> processInBackground(porcupine, ksListener, audioStream));
 183         return new KSServiceHandle() {
 184             @Override
 185             public void abort() {
 186                 logger.debug("stopping service");
 187                 loop = false;
 188                 try {
 189                     Thread.sleep(100);
 190                 } catch (InterruptedException e) {
 191                 }
 192                 scheduledTask.cancel(true);
 193             }
 194         };
 195     }
 196
 197     private Porcupine initPorcupine(BundleContext bundleContext, Locale locale, String keyword)
 198             throws IOException, PorcupineException {
 199         // Suppress library logs
 200         java.util.logging.Logger globalJavaLogger = java.util.logging.Logger
 201                 .getLogger(java.util.logging.Logger.GLOBAL_LOGGER_NAME);
 202         Level currentGlobalLogLevel = globalJavaLogger.getLevel();
 203         globalJavaLogger.setLevel(java.util.logging.Level.OFF);
 204         String bundleLibraryPath = Porcupine.LIBRARY_PATH;
 205         if (bundleLibraryPath == null) {
 206             throw new PorcupineException("Unsupported environment, ensure Porcupine is supported by your system");
 207         }
 208         String libraryPath = prepareLib(bundleContext, bundleLibraryPath);
 209         String alternativeModelPath = getAlternativeModelPath(bundleContext, locale);
 210         String modelPath = alternativeModelPath != null ? alternativeModelPath
 211                 : prepareLib(bundleContext, Porcupine.MODEL_PATH);
 212         String keywordPath = getKeywordResourcePath(bundleContext, keyword, alternativeModelPath == null);
 213         logger.debug("Porcupine library path: {}", libraryPath);
 214         logger.debug("Porcupine model path: {}", modelPath);
 215         logger.debug("Porcupine keyword path: {}", keywordPath);
 216         logger.debug("Porcupine sensitivity: {}", config.sensitivity);
 217         try {
 218             return new Porcupine(config.apiKey, libraryPath, modelPath, new String[] { keywordPath },
 219                     new float[] { config.sensitivity });
 220         } finally {
 221             // restore log level
 222             globalJavaLogger.setLevel(currentGlobalLogLevel);
 223         }
 224     }
 225
 226     private String getPorcupineEnv() {
 227         // get porcupine env from resolved library path
 228         String searchTerm = "lib" + File.separator + "java" + File.separator;
 229         String env = Porcupine.LIBRARY_PATH.substring(Porcupine.LIBRARY_PATH.indexOf(searchTerm) + searchTerm.length());
 230         env = env.substring(0, env.indexOf(File.separator));
 231         return env;
 232     }
 233
 234     private @Nullable String getAlternativeModelPath(BundleContext bundleContext, Locale locale) throws IOException {
 235         String modelPath = null;
 236         if (locale.getLanguage().equals(Locale.GERMAN.getLanguage())) {
 237             Path dePath = Path.of(PORCUPINE_FOLDER, "porcupine_params_de.pv");
 238             if (Files.exists(dePath)) {
 239                 modelPath = dePath.toString();
 240             } else {
 241                 logger.warn(
 242                         "You can provide a specific model for de language in {}, english language model will be used",
 243                         PORCUPINE_FOLDER);
 244             }
 245         } else if (locale.getLanguage().equals(Locale.FRENCH.getLanguage())) {
 246             Path frPath = Path.of(PORCUPINE_FOLDER, "porcupine_params_fr.pv");
 247             if (Files.exists(frPath)) {
 248                 modelPath = frPath.toString();
 249             } else {
 250                 logger.warn(
 251                         "You can provide a specific model for fr language in {}, english language model will be used",
 252                         PORCUPINE_FOLDER);
 253             }
 254         } else if (locale.getLanguage().equals("es")) {
 255             Path esPath = Path.of(PORCUPINE_FOLDER, "porcupine_params_es.pv");
 256             if (Files.exists(esPath)) {
 257                 modelPath = esPath.toString();
 258             } else {
 259                 logger.warn(
 260                         "You can provide a specific model for es language in {}, english language model will be used",
 261                         PORCUPINE_FOLDER);
 262             }
 263         }
 264         return modelPath;
 265     }
 266
 267     private String getKeywordResourcePath(BundleContext bundleContext, String keyWord, boolean allowBuildIn)
 268             throws IOException {
 269         String localKeywordFile = keyWord.toLowerCase().replace(" ", "_") + ".ppn";
 270         Path localKeywordPath = Path.of(PORCUPINE_FOLDER, localKeywordFile);
 271         if (Files.exists(localKeywordPath)) {
 272             return localKeywordPath.toString();
 273         }
 274         if (allowBuildIn) {
 275             try {
 276                 Porcupine.BuiltInKeyword.valueOf(keyWord.toUpperCase().replace(" ", "_"));
 277             } catch (IllegalArgumentException e) {
 278                 throw new IllegalArgumentException(
 279                         "Unable to find model file for configured wake word neither is build-in. Should be at "
 280                                 + localKeywordPath);
 281             }
 282             String env = getPorcupineEnv();
 283             String keywordPath = "porcupine/resources/keyword_files/" + env + "/" + keyWord.replace(" ", "_") + "_"
 284                     + env + ".ppn";
 285             return prepareLib(bundleContext, keywordPath);
 286         } else {
 287             throw new IllegalArgumentException(
 288                     "Unable to find model file for configured wake word; there are no build-in wake words for your language. Should be at "
 289                             + localKeywordPath);
 290         }
 291     }
 292
 293     private void processInBackground(Porcupine porcupine, KSListener ksListener, AudioStream audioStream) {
 294         int numBytesRead;
 295         // buffers for processing audio
 296         int frameLength = porcupine.getFrameLength();
 297         ByteBuffer captureBuffer = ByteBuffer.allocate(frameLength * 2);
 298         captureBuffer.order(ByteOrder.LITTLE_ENDIAN);
 299         short[] porcupineBuffer = new short[frameLength];
 300         this.loop = true;
 301         while (loop) {
 302             try {
 303                 // read a buffer of audio
 304                 numBytesRead = audioStream.read(captureBuffer.array(), 0, captureBuffer.capacity());
 305                 if (!loop) {
 306                     break;
 307                 }
 308                 // don't pass to porcupine if we don't have a full buffer
 309                 if (numBytesRead != frameLength * 2) {
 310                     Thread.sleep(100);
 311                     continue;
 312                 }
 313                 // copy into 16-bit buffer
 314                 captureBuffer.asShortBuffer().get(porcupineBuffer);
 315                 // process with porcupine
 316                 int result = porcupine.process(porcupineBuffer);
 317                 if (result >= 0) {
 318                     logger.debug("keyword detected!");
 319                     ksListener.ksEventReceived(new KSpottedEvent());
 320                 }
 321             } catch (IOException | PorcupineException | InterruptedException e) {
 322                 String errorMessage = e.getMessage();
 323                 ksListener.ksEventReceived(new KSErrorEvent(errorMessage != null ? errorMessage : "Unexpected error"));
 324             }
 325         }
 326         porcupine.delete();
 327         logger.debug("Porcupine stopped");
 328     }
 329 }