git.basschouten.com Git - openhab-addons.git/blob

   1 /**
   2  * Copyright (c) 2010-2022 Contributors to the openHAB project
   3  *
   4  * See the NOTICE file(s) distributed with this work for additional
   5  * information.
   6  *
   7  * This program and the accompanying materials are made available under the
   8  * terms of the Eclipse Public License 2.0 which is available at
   9  * http://www.eclipse.org/legal/epl-2.0
  10  *
  11  * SPDX-License-Identifier: EPL-2.0
  12  */
  13 package org.openhab.voice.porcupineks.internal;
  14
  15 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_CATEGORY;
  16 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_ID;
  17 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_NAME;
  18 import static org.openhab.voice.porcupineks.internal.PorcupineKSConstants.SERVICE_PID;
  19
  20 import java.io.BufferedInputStream;
  21 import java.io.BufferedOutputStream;
  22 import java.io.File;
  23 import java.io.FileOutputStream;
  24 import java.io.IOException;
  25 import java.io.InputStream;
  26 import java.io.OutputStream;
  27 import java.net.URL;
  28 import java.nio.ByteBuffer;
  29 import java.nio.ByteOrder;
  30 import java.nio.file.Files;
  31 import java.nio.file.Path;
  32 import java.util.Locale;
  33 import java.util.Map;
  34 import java.util.Set;
  35 import java.util.concurrent.Future;
  36 import java.util.concurrent.ScheduledExecutorService;
  37 import java.util.logging.Level;
  38
  39 import org.eclipse.jdt.annotation.NonNullByDefault;
  40 import org.eclipse.jdt.annotation.Nullable;
  41 import org.openhab.core.OpenHAB;
  42 import org.openhab.core.audio.AudioFormat;
  43 import org.openhab.core.audio.AudioStream;
  44 import org.openhab.core.common.ThreadPoolManager;
  45 import org.openhab.core.config.core.ConfigurableService;
  46 import org.openhab.core.config.core.Configuration;
  47 import org.openhab.core.voice.KSErrorEvent;
  48 import org.openhab.core.voice.KSException;
  49 import org.openhab.core.voice.KSListener;
  50 import org.openhab.core.voice.KSService;
  51 import org.openhab.core.voice.KSServiceHandle;
  52 import org.openhab.core.voice.KSpottedEvent;
  53 import org.osgi.framework.BundleContext;
  54 import org.osgi.framework.Constants;
  55 import org.osgi.service.component.ComponentContext;
  56 import org.osgi.service.component.annotations.Activate;
  57 import org.osgi.service.component.annotations.Component;
  58 import org.osgi.service.component.annotations.Modified;
  59 import org.slf4j.Logger;
  60 import org.slf4j.LoggerFactory;
  61
  62 import ai.picovoice.porcupine.Porcupine;
  63 import ai.picovoice.porcupine.PorcupineException;
  64
  65 /**
  66  * The {@link PorcupineKSService} is a keyword spotting implementation based on porcupine.
  67  *
  68  * @author Miguel Álvarez - Initial contribution
  69  */
  70 @NonNullByDefault
  71 @Component(configurationPid = SERVICE_PID, property = Constants.SERVICE_PID + "=" + SERVICE_PID)
  72 @ConfigurableService(category = SERVICE_CATEGORY, label = SERVICE_NAME
  73         + " Keyword Spotter", description_uri = SERVICE_CATEGORY + ":" + SERVICE_ID)
  74 public class PorcupineKSService implements KSService {
  75     private static final String PORCUPINE_FOLDER = Path.of(OpenHAB.getUserDataFolder(), "porcupine").toString();
  76     private static final String EXTRACTION_FOLDER = Path.of(OpenHAB.getUserDataFolder(), "porcupine", "extracted")
  77             .toString();
  78     private final Logger logger = LoggerFactory.getLogger(PorcupineKSService.class);
  79     private final ScheduledExecutorService executor = ThreadPoolManager.getScheduledPool("OH-voice-porcupineks");
  80     private PorcupineKSConfiguration config = new PorcupineKSConfiguration();
  81     private boolean loop = false;
  82     private @Nullable BundleContext bundleContext;
  83
  84     static {
  85         Logger logger = LoggerFactory.getLogger(PorcupineKSService.class);
  86         File directory = new File(PORCUPINE_FOLDER);
  87         if (!directory.exists()) {
  88             if (directory.mkdir()) {
  89                 logger.info("porcupine dir created {}", PORCUPINE_FOLDER);
  90             }
  91         }
  92         File childDirectory = new File(EXTRACTION_FOLDER);
  93         if (!childDirectory.exists()) {
  94             if (childDirectory.mkdir()) {
  95                 logger.info("porcupine extraction file dir created {}", EXTRACTION_FOLDER);
  96             }
  97         }
  98     }
  99
 100     @Activate
 101     protected void activate(ComponentContext componentContext, Map<String, Object> config) {
 102         this.bundleContext = componentContext.getBundleContext();
 103         modified(config);
 104     }
 105
 106     @Modified
 107     protected void modified(Map<String, Object> config) {
 108         this.config = new Configuration(config).as(PorcupineKSConfiguration.class);
 109         if (this.config.apiKey.isBlank()) {
 110             logger.warn("Missing pico voice api key to use Porcupine Keyword Spotter");
 111         }
 112     }
 113
 114     private String prepareLib(BundleContext bundleContext, String path) throws IOException {
 115         if (!path.contains("porcupine" + File.separator)) {
 116             // this should never happen
 117             throw new IOException("Path is not pointing to porcupine bundle files " + path);
 118         }
 119         // get a path relative to the porcupine bundle folder
 120         String relativePath;
 121         if (path.startsWith("porcupine" + File.separator)) {
 122             relativePath = path;
 123         } else {
 124             relativePath = path.substring(path.lastIndexOf(File.separator + "porcupine" + File.separator) + 1);
 125         }
 126         File localFile = new File(EXTRACTION_FOLDER,
 127                 relativePath.substring(relativePath.lastIndexOf(File.separator) + 1));
 128         if (!localFile.exists()) {
 129             if (File.separator.equals("\\")) {
 130                 // bundle requires unix path separator
 131                 logger.debug("use unix path separator");
 132                 relativePath = relativePath.replace("\\", "/");
 133             }
 134             URL porcupineResource = bundleContext.getBundle().getEntry(relativePath);
 135             logger.debug("extracting binary {} from bundle to extraction folder", relativePath);
 136             if (porcupineResource == null) {
 137                 throw new IOException("Missing bundle file: " + relativePath);
 138             }
 139             extractFromBundle(porcupineResource, localFile);
 140         } else {
 141             logger.debug("binary {} already extracted", relativePath);
 142         }
 143         return localFile.toString();
 144     }
 145
 146     private void extractFromBundle(URL resourceUrl, File targetFile) throws IOException {
 147         InputStream in = new BufferedInputStream(resourceUrl.openStream());
 148         OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile));
 149         byte[] buffer = new byte[1024];
 150         int lengthRead;
 151         while ((lengthRead = in.read(buffer)) > 0) {
 152             out.write(buffer, 0, lengthRead);
 153             out.flush();
 154         }
 155         in.close();
 156         out.close();
 157     }
 158
 159     @Override
 160     public String getId() {
 161         return SERVICE_ID;
 162     }
 163
 164     @Override
 165     public String getLabel(@Nullable Locale locale) {
 166         return SERVICE_NAME;
 167     }
 168
 169     @Override
 170     public Set<Locale> getSupportedLocales() {
 171         return Set.of(Locale.ENGLISH, new Locale("es"), Locale.FRENCH, Locale.GERMAN);
 172     }
 173
 174     @Override
 175     public Set<AudioFormat> getSupportedFormats() {
 176         return Set
 177                 .of(new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, 16, null, 16000L));
 178     }
 179
 180     @Override
 181     public KSServiceHandle spot(KSListener ksListener, AudioStream audioStream, Locale locale, String keyword)
 182             throws KSException {
 183         Porcupine porcupine;
 184         if (config.apiKey.isBlank()) {
 185             throw new KSException("Missing pico voice api key");
 186         }
 187         BundleContext bundleContext = this.bundleContext;
 188         if (bundleContext == null) {
 189             throw new KSException("Missing bundle context");
 190         }
 191         try {
 192             porcupine = initPorcupine(bundleContext, locale, keyword);
 193         } catch (PorcupineException | IOException e) {
 194             throw new KSException(e);
 195         }
 196         Future<?> scheduledTask = executor.submit(() -> processInBackground(porcupine, ksListener, audioStream));
 197         return new KSServiceHandle() {
 198             @Override
 199             public void abort() {
 200                 logger.debug("stopping service");
 201                 loop = false;
 202                 try {
 203                     Thread.sleep(100);
 204                 } catch (InterruptedException e) {
 205                 }
 206                 scheduledTask.cancel(true);
 207             }
 208         };
 209     }
 210
 211     private Porcupine initPorcupine(BundleContext bundleContext, Locale locale, String keyword)
 212             throws IOException, PorcupineException {
 213         // Suppress library logs
 214         java.util.logging.Logger globalJavaLogger = java.util.logging.Logger
 215                 .getLogger(java.util.logging.Logger.GLOBAL_LOGGER_NAME);
 216         Level currentGlobalLogLevel = globalJavaLogger.getLevel();
 217         globalJavaLogger.setLevel(java.util.logging.Level.OFF);
 218         String bundleLibraryPath = Porcupine.LIBRARY_PATH;
 219         if (bundleLibraryPath == null) {
 220             throw new PorcupineException("Unsupported environment, ensure Porcupine is supported by your system");
 221         }
 222         String libraryPath = prepareLib(bundleContext, bundleLibraryPath);
 223         String alternativeModelPath = getAlternativeModelPath(bundleContext, locale);
 224         String modelPath = alternativeModelPath != null ? alternativeModelPath
 225                 : prepareLib(bundleContext, Porcupine.MODEL_PATH);
 226         String keywordPath = getKeywordResourcePath(bundleContext, keyword, alternativeModelPath == null);
 227         logger.debug("Porcupine library path: {}", libraryPath);
 228         logger.debug("Porcupine model path: {}", modelPath);
 229         logger.debug("Porcupine keyword path: {}", keywordPath);
 230         logger.debug("Porcupine sensitivity: {}", config.sensitivity);
 231         try {
 232             return new Porcupine(config.apiKey, libraryPath, modelPath, new String[] { keywordPath },
 233                     new float[] { config.sensitivity });
 234         } finally {
 235             // restore log level
 236             globalJavaLogger.setLevel(currentGlobalLogLevel);
 237         }
 238     }
 239
 240     private String getPorcupineEnv() {
 241         // get porcupine env from resolved library path
 242         String searchTerm = "lib" + File.separator + "java" + File.separator;
 243         String env = Porcupine.LIBRARY_PATH.substring(Porcupine.LIBRARY_PATH.indexOf(searchTerm) + searchTerm.length());
 244         env = env.substring(0, env.indexOf(File.separator));
 245         return env;
 246     }
 247
 248     private @Nullable String getAlternativeModelPath(BundleContext bundleContext, Locale locale) throws IOException {
 249         String modelPath = null;
 250         if (locale.getLanguage().equals(Locale.GERMAN.getLanguage())) {
 251             Path dePath = Path.of(PORCUPINE_FOLDER, "porcupine_params_de.pv");
 252             if (Files.exists(dePath)) {
 253                 modelPath = dePath.toString();
 254             } else {
 255                 logger.warn(
 256                         "You can provide a specific model for de language in {}, english language model will be used",
 257                         PORCUPINE_FOLDER);
 258             }
 259         } else if (locale.getLanguage().equals(Locale.FRENCH.getLanguage())) {
 260             Path frPath = Path.of(PORCUPINE_FOLDER, "porcupine_params_fr.pv");
 261             if (Files.exists(frPath)) {
 262                 modelPath = frPath.toString();
 263             } else {
 264                 logger.warn(
 265                         "You can provide a specific model for fr language in {}, english language model will be used",
 266                         PORCUPINE_FOLDER);
 267             }
 268         } else if (locale.getLanguage().equals("es")) {
 269             Path esPath = Path.of(PORCUPINE_FOLDER, "porcupine_params_es.pv");
 270             if (Files.exists(esPath)) {
 271                 modelPath = esPath.toString();
 272             } else {
 273                 logger.warn(
 274                         "You can provide a specific model for es language in {}, english language model will be used",
 275                         PORCUPINE_FOLDER);
 276             }
 277         }
 278         return modelPath;
 279     }
 280
 281     private String getKeywordResourcePath(BundleContext bundleContext, String keyWord, boolean allowBuildIn)
 282             throws IOException {
 283         String localKeywordFile = keyWord.toLowerCase().replace(" ", "_") + ".ppn";
 284         Path localKeywordPath = Path.of(PORCUPINE_FOLDER, localKeywordFile);
 285         if (Files.exists(localKeywordPath)) {
 286             return localKeywordPath.toString();
 287         }
 288         if (allowBuildIn) {
 289             try {
 290                 Porcupine.BuiltInKeyword.valueOf(keyWord.toUpperCase().replace(" ", "_"));
 291             } catch (IllegalArgumentException e) {
 292                 throw new IllegalArgumentException(
 293                         "Unable to find model file for configured wake word neither is build-in. Should be at "
 294                                 + localKeywordPath);
 295             }
 296             String env = getPorcupineEnv();
 297             String keywordPath = Path
 298                     .of("porcupine", "resources", "keyword_files", env, keyWord.replace(" ", "_") + "_" + env + ".ppn")
 299                     .toString();
 300             return prepareLib(bundleContext, keywordPath);
 301         } else {
 302             throw new IllegalArgumentException(
 303                     "Unable to find model file for configured wake word; there are no build-in wake words for your language. Should be at "
 304                             + localKeywordPath);
 305         }
 306     }
 307
 308     private void processInBackground(Porcupine porcupine, KSListener ksListener, AudioStream audioStream) {
 309         int numBytesRead;
 310         // buffers for processing audio
 311         int frameLength = porcupine.getFrameLength();
 312         ByteBuffer captureBuffer = ByteBuffer.allocate(frameLength * 2);
 313         captureBuffer.order(ByteOrder.LITTLE_ENDIAN);
 314         short[] porcupineBuffer = new short[frameLength];
 315         this.loop = true;
 316         while (loop) {
 317             try {
 318                 // read a buffer of audio
 319                 numBytesRead = audioStream.read(captureBuffer.array(), 0, captureBuffer.capacity());
 320                 if (!loop) {
 321                     break;
 322                 }
 323                 // don't pass to porcupine if we don't have a full buffer
 324                 if (numBytesRead != frameLength * 2) {
 325                     Thread.sleep(100);
 326                     continue;
 327                 }
 328                 // copy into 16-bit buffer
 329                 captureBuffer.asShortBuffer().get(porcupineBuffer);
 330                 // process with porcupine
 331                 int result = porcupine.process(porcupineBuffer);
 332                 if (result >= 0) {
 333                     logger.debug("keyword detected!");
 334                     ksListener.ksEventReceived(new KSpottedEvent());
 335                 }
 336             } catch (IOException | PorcupineException | InterruptedException e) {
 337                 String errorMessage = e.getMessage();
 338                 ksListener.ksEventReceived(new KSErrorEvent(errorMessage != null ? errorMessage : "Unexpected error"));
 339             }
 340         }
 341         porcupine.delete();
 342         logger.debug("Porcupine stopped");
 343     }
 344 }