git.basschouten.com Git - openhab-addons.git/blob

   1 /**
   2  * Copyright (c) 2010-2023 Contributors to the openHAB project
   3  *
   4  * See the NOTICE file(s) distributed with this work for additional
   5  * information.
   6  *
   7  * This program and the accompanying materials are made available under the
   8  * terms of the Eclipse Public License 2.0 which is available at
   9  * http://www.eclipse.org/legal/epl-2.0
  10  *
  11  * SPDX-License-Identifier: EPL-2.0
  12  */
  13 package org.openhab.voice.porcupineks.internal;
  14
  15 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_CATEGORY;
  16 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_ID;
  17 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_NAME;
  18 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_PID;
  19
  20 import java.io.BufferedInputStream;
  21 import java.io.BufferedOutputStream;
  22 import java.io.File;
  23 import java.io.FileOutputStream;
  24 import java.io.IOException;
  25 import java.io.InputStream;
  26 import java.io.OutputStream;
  27 import java.net.URL;
  28 import java.nio.ByteBuffer;
  29 import java.nio.ByteOrder;
  30 import java.nio.file.Files;
  31 import java.nio.file.Path;
  32 import java.util.Locale;
  33 import java.util.Map;
  34 import java.util.Set;
  35 import java.util.concurrent.Future;
  36 import java.util.concurrent.ScheduledExecutorService;
  37 import java.util.concurrent.atomic.AtomicBoolean;
  38 import java.util.logging.Level;
  39
  40 import org.eclipse.jdt.annotation.NonNullByDefault;
  41 import org.eclipse.jdt.annotation.Nullable;
  42 import org.openhab.core.OpenHAB;
  43 import org.openhab.core.audio.AudioFormat;
  44 import org.openhab.core.audio.AudioStream;
  45 import org.openhab.core.common.ThreadPoolManager;
  46 import org.openhab.core.config.core.ConfigurableService;
  47 import org.openhab.core.config.core.Configuration;
  48 import org.openhab.core.voice.KSErrorEvent;
  49 import org.openhab.core.voice.KSException;
  50 import org.openhab.core.voice.KSListener;
  51 import org.openhab.core.voice.KSService;
  52 import org.openhab.core.voice.KSServiceHandle;
  53 import org.openhab.core.voice.KSpottedEvent;
  54 import org.osgi.framework.BundleContext;
  55 import org.osgi.framework.Constants;
  56 import org.osgi.service.component.ComponentContext;
  57 import org.osgi.service.component.annotations.Activate;
  58 import org.osgi.service.component.annotations.Component;
  59 import org.osgi.service.component.annotations.Modified;
  60 import org.slf4j.Logger;
  61 import org.slf4j.LoggerFactory;
  62
  63 import ai.picovoice.porcupine.Porcupine;
  64 import ai.picovoice.porcupine.PorcupineException;
  65
  66 /**
  67  * The {@link PorcupineKSService} is a keyword spotting implementation based on porcupine.
  68  *
  69  * @author Miguel Álvarez - Initial contribution
  70  */
  71 @NonNullByDefault
  72 @Component(configurationPid = SERVICE_PID, property = Constants.SERVICE_PID + "=" + SERVICE_PID)
  73 @ConfigurableService(category = SERVICE_CATEGORY, label = SERVICE_NAME
  74         + " Keyword Spotter", description_uri = SERVICE_CATEGORY + ":" + SERVICE_ID)
  75 public class PorcupineKSService implements KSService {
  76     private static final String PORCUPINE_FOLDER = Path.of(OpenHAB.getUserDataFolder(), "porcupine").toString();
  77     private static final String EXTRACTION_FOLDER = Path.of(OpenHAB.getUserDataFolder(), "porcupine", "extracted")
  78             .toString();
  79     private final Logger logger = LoggerFactory.getLogger(PorcupineKSService.class);
  80     private final ScheduledExecutorService executor = ThreadPoolManager.getScheduledPool("OH-voice-porcupineks");
  81     private PorcupineKSConfiguration config = new PorcupineKSConfiguration();
  82     private @Nullable BundleContext bundleContext;
  83
  84     static {
  85         Logger logger = LoggerFactory.getLogger(PorcupineKSService.class);
  86         File directory = new File(PORCUPINE_FOLDER);
  87         if (!directory.exists()) {
  88             if (directory.mkdir()) {
  89                 logger.info("porcupine dir created {}", PORCUPINE_FOLDER);
  90             }
  91         }
  92         File childDirectory = new File(EXTRACTION_FOLDER);
  93         if (!childDirectory.exists()) {
  94             if (childDirectory.mkdir()) {
  95                 logger.info("porcupine extraction file dir created {}", EXTRACTION_FOLDER);
  96             }
  97         }
  98     }
  99
 100     @Activate
 101     protected void activate(ComponentContext componentContext, Map<String, Object> config) {
 102         this.bundleContext = componentContext.getBundleContext();
 103         modified(config);
 104     }
 105
 106     @Modified
 107     protected void modified(Map<String, Object> config) {
 108         this.config = new Configuration(config).as(PorcupineKSConfiguration.class);
 109         if (this.config.apiKey.isBlank()) {
 110             logger.warn("Missing pico voice api key to use Porcupine Keyword Spotter");
 111         }
 112     }
 113
 114     private String prepareLib(BundleContext bundleContext, String path) throws IOException {
 115         if (!path.contains("porcupine" + File.separator)) {
 116             // this should never happen
 117             throw new IOException("Path is not pointing to porcupine bundle files " + path);
 118         }
 119         // get a path relative to the porcupine bundle folder
 120         String relativePath;
 121         if (path.startsWith("porcupine" + File.separator)) {
 122             relativePath = path;
 123         } else {
 124             relativePath = path.substring(path.lastIndexOf(File.separator + "porcupine" + File.separator) + 1);
 125         }
 126         File localFile = new File(EXTRACTION_FOLDER,
 127                 relativePath.substring(relativePath.lastIndexOf(File.separator) + 1));
 128         if (!localFile.exists()) {
 129             if ("\\".equals(File.separator)) {
 130                 // bundle requires unix path separator
 131                 logger.debug("use unix path separator");
 132                 relativePath = relativePath.replace("\\", "/");
 133             }
 134             URL porcupineResource = bundleContext.getBundle().getEntry(relativePath);
 135             logger.debug("extracting binary {} from bundle to extraction folder", relativePath);
 136             if (porcupineResource == null) {
 137                 throw new IOException("Missing bundle file: " + relativePath);
 138             }
 139             extractFromBundle(porcupineResource, localFile);
 140         } else {
 141             logger.debug("binary {} already extracted", relativePath);
 142         }
 143         return localFile.toString();
 144     }
 145
 146     private void extractFromBundle(URL resourceUrl, File targetFile) throws IOException {
 147         InputStream in = new BufferedInputStream(resourceUrl.openStream());
 148         OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile));
 149         byte[] buffer = new byte[1024];
 150         int lengthRead;
 151         while ((lengthRead = in.read(buffer)) > 0) {
 152             out.write(buffer, 0, lengthRead);
 153             out.flush();
 154         }
 155         in.close();
 156         out.close();
 157     }
 158
 159     @Override
 160     public String getId() {
 161         return SERVICE_ID;
 162     }
 163
 164     @Override
 165     public String getLabel(@Nullable Locale locale) {
 166         return SERVICE_NAME;
 167     }
 168
 169     @Override
 170     public Set<Locale> getSupportedLocales() {
 171         return Set.of(Locale.ENGLISH, new Locale("es"), Locale.FRENCH, Locale.GERMAN);
 172     }
 173
 174     @Override
 175     public Set<AudioFormat> getSupportedFormats() {
 176         return Set
 177                 .of(new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, 16, null, 16000L));
 178     }
 179
 180     @Override
 181     public KSServiceHandle spot(KSListener ksListener, AudioStream audioStream, Locale locale, String keyword)
 182             throws KSException {
 183         Porcupine porcupine;
 184         if (config.apiKey.isBlank()) {
 185             throw new KSException("Missing pico voice api key");
 186         }
 187         BundleContext bundleContext = this.bundleContext;
 188         if (bundleContext == null) {
 189             throw new KSException("Missing bundle context");
 190         }
 191         try {
 192             porcupine = initPorcupine(bundleContext, locale, keyword);
 193         } catch (PorcupineException | IOException e) {
 194             throw new KSException(e);
 195         }
 196         final AtomicBoolean aborted = new AtomicBoolean(false);
 197         Future<?> scheduledTask = executor
 198                 .submit(() -> processInBackground(porcupine, ksListener, audioStream, aborted));
 199         return new KSServiceHandle() {
 200             @Override
 201             public void abort() {
 202                 logger.debug("stopping service");
 203                 aborted.set(true);
 204                 try {
 205                     Thread.sleep(100);
 206                 } catch (InterruptedException e) {
 207                 }
 208                 scheduledTask.cancel(true);
 209             }
 210         };
 211     }
 212
 213     private Porcupine initPorcupine(BundleContext bundleContext, Locale locale, String keyword)
 214             throws IOException, PorcupineException {
 215         // Suppress library logs
 216         java.util.logging.Logger globalJavaLogger = java.util.logging.Logger
 217                 .getLogger(java.util.logging.Logger.GLOBAL_LOGGER_NAME);
 218         Level currentGlobalLogLevel = globalJavaLogger.getLevel();
 219         globalJavaLogger.setLevel(java.util.logging.Level.OFF);
 220         String bundleLibraryPath = Porcupine.LIBRARY_PATH;
 221         if (bundleLibraryPath == null) {
 222             throw new PorcupineException("Unsupported environment, ensure Porcupine is supported by your system");
 223         }
 224         String libraryPath = prepareLib(bundleContext, bundleLibraryPath);
 225         String alternativeModelPath = getAlternativeModelPath(bundleContext, locale);
 226         String modelPath = alternativeModelPath != null ? alternativeModelPath
 227                 : prepareLib(bundleContext, Porcupine.MODEL_PATH);
 228         String keywordPath = getKeywordResourcePath(bundleContext, keyword, alternativeModelPath == null);
 229         logger.debug("Porcupine library path: {}", libraryPath);
 230         logger.debug("Porcupine model path: {}", modelPath);
 231         logger.debug("Porcupine keyword path: {}", keywordPath);
 232         logger.debug("Porcupine sensitivity: {}", config.sensitivity);
 233         try {
 234             return new Porcupine(config.apiKey, libraryPath, modelPath, new String[] { keywordPath },
 235                     new float[] { config.sensitivity });
 236         } finally {
 237             // restore log level
 238             globalJavaLogger.setLevel(currentGlobalLogLevel);
 239         }
 240     }
 241
 242     private String getPorcupineEnv() {
 243         // get porcupine env from resolved library path
 244         String searchTerm = "lib" + File.separator + "java" + File.separator;
 245         String env = Porcupine.LIBRARY_PATH.substring(Porcupine.LIBRARY_PATH.indexOf(searchTerm) + searchTerm.length());
 246         env = env.substring(0, env.indexOf(File.separator));
 247         return env;
 248     }
 249
 250     private @Nullable String getAlternativeModelPath(BundleContext bundleContext, Locale locale) throws IOException {
 251         String modelPath = null;
 252         if (locale.getLanguage().equals(Locale.GERMAN.getLanguage())) {
 253             Path dePath = Path.of(PORCUPINE_FOLDER, "porcupine_params_de.pv");
 254             if (Files.exists(dePath)) {
 255                 modelPath = dePath.toString();
 256             } else {
 257                 logger.warn(
 258                         "You can provide a specific model for de language in {}, english language model will be used",
 259                         PORCUPINE_FOLDER);
 260             }
 261         } else if (locale.getLanguage().equals(Locale.FRENCH.getLanguage())) {
 262             Path frPath = Path.of(PORCUPINE_FOLDER, "porcupine_params_fr.pv");
 263             if (Files.exists(frPath)) {
 264                 modelPath = frPath.toString();
 265             } else {
 266                 logger.warn(
 267                         "You can provide a specific model for fr language in {}, english language model will be used",
 268                         PORCUPINE_FOLDER);
 269             }
 270         } else if ("es".equals(locale.getLanguage())) {
 271             Path esPath = Path.of(PORCUPINE_FOLDER, "porcupine_params_es.pv");
 272             if (Files.exists(esPath)) {
 273                 modelPath = esPath.toString();
 274             } else {
 275                 logger.warn(
 276                         "You can provide a specific model for es language in {}, english language model will be used",
 277                         PORCUPINE_FOLDER);
 278             }
 279         }
 280         return modelPath;
 281     }
 282
 283     private String getKeywordResourcePath(BundleContext bundleContext, String keyWord, boolean allowBuildIn)
 284             throws IOException {
 285         String localKeywordFile = keyWord.toLowerCase().replace(" ", "_") + ".ppn";
 286         Path localKeywordPath = Path.of(PORCUPINE_FOLDER, localKeywordFile);
 287         if (Files.exists(localKeywordPath)) {
 288             return localKeywordPath.toString();
 289         }
 290         if (allowBuildIn) {
 291             try {
 292                 Porcupine.BuiltInKeyword.valueOf(keyWord.toUpperCase().replace(" ", "_"));
 293             } catch (IllegalArgumentException e) {
 294                 throw new IllegalArgumentException(
 295                         "Unable to find model file for configured wake word neither is build-in. Should be at "
 296                                 + localKeywordPath);
 297             }
 298             String env = getPorcupineEnv();
 299             String keywordPath = Path
 300                     .of("porcupine", "resources", "keyword_files", env, keyWord.replace(" ", "_") + "_" + env + ".ppn")
 301                     .toString();
 302             return prepareLib(bundleContext, keywordPath);
 303         } else {
 304             throw new IllegalArgumentException(
 305                     "Unable to find model file for configured wake word; there are no build-in wake words for your language. Should be at "
 306                             + localKeywordPath);
 307         }
 308     }
 309
 310     private void processInBackground(Porcupine porcupine, KSListener ksListener, AudioStream audioStream,
 311             AtomicBoolean aborted) {
 312         int numBytesRead;
 313         // buffers for processing audio
 314         int frameLength = porcupine.getFrameLength();
 315         ByteBuffer captureBuffer = ByteBuffer.allocate(frameLength * 2);
 316         captureBuffer.order(ByteOrder.LITTLE_ENDIAN);
 317         short[] porcupineBuffer = new short[frameLength];
 318         while (!aborted.get()) {
 319             try {
 320                 // read a buffer of audio
 321                 numBytesRead = audioStream.read(captureBuffer.array(), 0, captureBuffer.capacity());
 322                 if (aborted.get()) {
 323                     break;
 324                 }
 325                 // don't pass to porcupine if we don't have a full buffer
 326                 if (numBytesRead != frameLength * 2) {
 327                     Thread.sleep(100);
 328                     continue;
 329                 }
 330                 // copy into 16-bit buffer
 331                 captureBuffer.asShortBuffer().get(porcupineBuffer);
 332                 // process with porcupine
 333                 int result = porcupine.process(porcupineBuffer);
 334                 if (result >= 0) {
 335                     logger.debug("keyword detected!");
 336                     ksListener.ksEventReceived(new KSpottedEvent());
 337                 }
 338             } catch (IOException | PorcupineException | InterruptedException e) {
 339                 String errorMessage = e.getMessage();
 340                 ksListener.ksEventReceived(new KSErrorEvent(errorMessage != null ? errorMessage : "Unexpected error"));
 341             }
 342         }
 343         porcupine.delete();
 344         logger.debug("Porcupine stopped");
 345     }
 346 }