recog~.c.bk

/*
[recog~] is a speech recognizer external for Pure Data written with the pocketsphinx API:
http://cmusphinx.sourceforge.net/
here is the copyright info for this excellent software:

 * ====================================================================
 * Copyright (c) 1999-2010 Carnegie Mellon University.  All rights
 * reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer. 
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * This work was supported in part by funding from the Defense Advanced 
 * Research Projects Agency and the National Science Foundation of the 
 * United States of America, and the CMU Sphinx Speech Consortium.
 *
 * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND 
 * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
 * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * ====================================================================

 my own copyright:

This code is entirely free to use for any reasonable purpose.
Copyright(c) David Medine February 3, 2011
http://imusic1.ucsd.edu/~dmedine
dmedine@ucsd.edu
****/

#include "m_pd.h"
#include <stdio.h>
#include <ctype.h>
#include <stdlib.h>
#include <math.h>
#include <float.h>
#include <pocketsphinx.h>
#include <string.h>

FILE *fp_ps;
static t_class *recog_tilde_class;

typedef struct histogram{
  t_int isGood;
}t_histogram;

typedef struct _recog_tilde{
  t_object x_obj;
  t_float f;

  t_outlet *list;

  t_float *inBuff, *dsBuff;
  int16 *decodeBuff, *decodeAutoBuff;
  t_int ioWritePoint, dsWritePoint, decodeWritePoint, decodeReadPoint;
  t_float ioReadPoint, dsReadPoint, thresh;
  t_int dsBuffSize, ioBuffSize;
  t_int decodeBuffSize, autoBuffSize;

  t_float rmndr;

  t_int decodeStart, decodeStop, automode;
  t_int decodeGo;

  t_int autoCounter, autoBlock;

  const char *hyp;

  t_int sampleRate;

  t_histogram *x_histo;
  //a histogram is associated with the charBuff 
  //used to compare strings and (hopefully) report correct
  //hyps

  t_int *wordHisto, wordCount;
  t_int *prevWordHisto, prevWordCount;
  char *charBuff;
  t_int charBuffSize;
  int32 charBuffrp;
  int32 prevLen; 
 
  //debugging:
  int writeout, writeread;
}t_recog_tilde;

  //structs from sphinx don't put these in the object struct??
ps_decoder_t *ps; //decoder 
cmd_ln_t *config; //configuration

t_symbol *hmm;
t_symbol *lm;
t_symbol *dict;
  


//----------------------------------
//----------------------------------
static void recog_tilde_decode(t_recog_tilde *x);
static void recog_tilde_reinit(t_recog_tilde *x);
static void output(t_recog_tilde *x);


//----------------------------------
//----------------------------------
static float spline_interpolate(t_float *buffer, t_int bufferLength, t_float findex)
//this routine is from musicDSP.org
//I literally copied someone else's code (works great)
{


  long index = findex;
  t_sample fr = findex - index;

  float p0=buffer[(index-2) % bufferLength];
  float p1=buffer[(index-1) % bufferLength];
  float p2=buffer[(index) % bufferLength];
  float p3=buffer[(index+1) % bufferLength];
  float p4=buffer[(index+2) % bufferLength];
  float p5=buffer[(index+3) % bufferLength];

  return p2 + 0.04166666666*fr*((p3-p1)*16.0+(p0-p4)*2.0
				+ fr *((p3+p1)*16.0-p0-p2*30.0- p4
				       + fr *(p3*66.0-p2*70.0-p4*33.0+p1*39.0+ p5*7.0- p0*9.0
					      + fr *( p2*126.0-p3*124.0+p4*61.0-p1*64.0- p5*12.0+p0*13.0
						      + fr *((p3-p2)*50.0+(p1-p4)*25.0+(p5-p0)*5.0)))));
}


static void block(t_recog_tilde *x, int n)
{
  int i, j;
  t_float f;

  if(x->automode == 0)//in this case just fill until it's full
    {
      //copy in converted samples
      for(i=0; i<x->dsBuffSize; i++)
	{
	  x->decodeBuff[x->decodeWritePoint++]=x->dsBuff[i];
	  if(x->decodeWritePoint >= x->decodeBuffSize)
	    {
	      x->decodeWritePoint -= x->decodeBuffSize;
	      if(x->automode == 0)
		post("you ran over the buffer\ntry not talking for so long next time");
	    }
	}
    
    }

  //if we are in automode, be more careful
  if(x->automode == 1)
    {
      for(i=0; i<x->dsBuffSize; i++)
	{
	  //copy in the new samples and wrap
	  x->decodeBuff[x->decodeWritePoint++] = x->dsBuff[i];
	  while(x->decodeWritePoint >= x->decodeBuffSize) x->decodeWritePoint -= x->decodeBuffSize;
	  
	  //keep track of how many samples have come in so far
	  x->autoCounter++;
	  
	  //if we have gotten to a block size
	  //shift and copy the appropriate decodeBuff material
	  //to autoBuff
	  if(x->autoCounter >= x->autoBlock)
	    {

	      //first do a safety flush. . .
	      x->decodeAutoBuff = (int16 *)t_getbytes(0);
	      x->decodeAutoBuff = (int16 *)t_resizebytes(x->decodeAutoBuff, 0, sizeof(int16) * x->autoBuffSize);

	      //determine the starting point. . .
	      x->decodeReadPoint = (x->decodeWritePoint - x->autoBuffSize);
	      //make sure to wrap:
	      while(x->decodeReadPoint<0) x->decodeReadPoint+=x->decodeBuffSize;
	      
	      //then copy in the appropriate samples. . .
	      for(j=0; j<x->autoBuffSize; j++) 
		{
		  x->decodeAutoBuff[j] = x->decodeBuff[x->decodeReadPoint++];
		  //and wrap
		  while(x->decodeReadPoint<=x->decodeBuffSize)x->decodeReadPoint-=x->decodeBuffSize;
		}

	      //finally, do the decoding and reset the counter
	      recog_tilde_decode(x);
	      x->autoCounter = 0;
	    }
 
	}
    }
}

//down sampling function
static void down_sample(t_recog_tilde *x, t_int n)
//args for this function are
//1. pointer to dataspace
//2. number of samples passed in

{
  int i;
  t_float conv, oneOverConv, f;
  t_int newsampnum;

  conv = (t_float)x->sampleRate/16000.0;
  f = (n/conv);


  newsampnum = (t_int)f;

  //routine to keep hold of the right number of samples
  f = f-newsampnum;
  x->rmndr+=f;
  if(x->rmndr>=1)
    {
      newsampnum++;
      x->rmndr = 0.0;
    }

  for(i=0; i<newsampnum; i++)
    {
      //find the interpolated value
      x->dsBuff[x->dsWritePoint]= spline_interpolate(x->inBuff, x->ioBuffSize, x->ioReadPoint);
      
      //convert it to signed pcm
      x->dsBuff[x->dsWritePoint++] *= 32768.0;

      x->ioReadPoint += conv;
      
      if(x->ioReadPoint >= x->ioBuffSize) x->ioReadPoint -= x->ioBuffSize;
      
      if(x->dsWritePoint >= x->dsBuffSize)//every 64 samples put it through the blocking routine
	{
	  x->dsWritePoint = 0;
	  block(x, x->dsBuffSize);
       	}
      
    }
 
  
}


//----------------------------------
//---------------------------------- 


//----------------------------------
//----------------------------------
//functions for parsing output text
static void string_sort(t_recog_tilde *x, char const *hyp, t_int len)
{
  int i;
  int j;

  j = 0;

  strncpy(x->charBuff+x->charBuffrp, hyp, len);
  //  post("charbuff: \n%s\n hyplen: \n%d", x->charBuff+x->charBuffrp, len);


  //keep track of words in the current (and previous) hypotheses
  for(i = 0; i<len; i++)
    {
      if(isspace(*(x->charBuff+x->charBuffrp+i)))
	{
	  *(x->wordHisto+j) = i;
	  j++;
	}
    }
  x->wordCount = (j+2);

  //compare the current hyp with the last




  //do this last
  x->charBuffrp += len;
  x->prevLen = len;
  x->prevWordHisto = (t_int *)t_resizebytes(x->prevWordHisto, 0, sizeof(t_int) * 50);
  for(i=0; i<j; i++)
    x->prevWordHisto[i] = x->wordHisto[i];
  x->prevWordCount = x->wordCount;

}


//decoder functions

//init the config:
static void recog_tilde_init(t_recog_tilde *x)
{
 if(lm!=0)
  config = cmd_ln_init(NULL, ps_args(), TRUE,
  			    "-hmm", hmm->s_name,
  			    "-lm", lm->s_name,
  			    "-dict", dict->s_name,
  			    NULL);
  
  else if(hmm && dict != 0)
    config = cmd_ln_init(NULL, ps_args(), TRUE,
			    "-hmm", hmm->s_name,
			    "-dict", dict->s_name,
			    NULL);
  else
    post("no hidden markov model or dictionary loaded\n...please load hmm directory and .dic file");


 ps = ps_init(config);
 cmd_ln_retain(ps);
 if (ps == NULL)
    pd_error(x, "configuration error!");
  else post("%d", sizeof(ps));

 fp_ps = fopen("goforward.raw", "w");

}

//do the decoding
static void recog_tilde_decode(t_recog_tilde *x)
{

  int rv;
  int i;
  char const *hyp, *uttid;
  int32 score;
  char buf[256];
  int hyplen;

  if (fp_ps == NULL) {
    perror("Failed to open goforward.raw");
    return 1;
  }

  fseek(fp_ps, 0, SEEK_SET);
  rv = ps_start_utt(ps, "goforward");
  if (rv < 0)
    return 1;
  while (!feof(fp_ps)) {
    size_t nsamp;
    nsamp = fread(buf, 2, 512, fp_ps);
    rv = ps_process_raw(ps, buf, nsamp, FALSE, FALSE);
  }
  /* rv = ps_end_utt(ps); */
  /* if (rv < 0) */
  /*   return 1; */
  /* hyp = ps_get_hyp(ps, &score, &uttid); */
  /* if (hyp == NULL) */
  /*   return 1; */
  /* printf("Recognized: %s\n", hyp); */

  /* rv =  ps_start_utt(ps, NULL); */
  /* //post("this should be 0: %d", rv); */

  /* if(x->automode == 1) */
  /*   rv = ps_process_raw(ps, x->decodeAutoBuff, x->autoBuffSize, FALSE, FALSE); */
  /* else */
  /*   rv = ps_process_raw(ps, x->decodeBuff, x->decodeWritePoint, FALSE, FALSE); */
  /* post("this should be the number of frames: %d", rv); */
  
  /* rv = ps_end_utt(ps); */
  /* //post("this should also be 0: %d", rv); */
  /* hyp = ps_get_hyp(ps, &score, &uttid); */
  /* if(hyp != NULL) */
  /*   { */
  /*     hyplen = strlen(hyp); */
  /*     if(x->automode == 1) */
  /* 	{ */
  /* 	  post("hyp :\n%s", hyp); */
  /* 	  string_sort(x, hyp, (t_int) strlen(hyp)); */

  /* 	} */
  /*     else */
  /* 	post("%s", hyp); */
      

  /*   } */

  /* /\*   output(x); *\/ */
    

  x->dsWritePoint = x->decodeWritePoint = 0;
  
}

//debugging function:
static void output(t_recog_tilde *x)
{
  t_atom *outv;
  int32 outc, i;
  t_float f, pcmtofconv;

  pcmtofconv = 1/32768.0;
  outc = x->decodeWritePoint;

  outv = (t_atom *)getbytes(0);
  outv = (t_atom *)resizebytes(outv, 0, outc * sizeof(t_atom));

  
  for(i = 0; i<x->decodeWritePoint; i++)
    {
      f = pcmtofconv*(t_float)x->decodeBuff[i];
      SETFLOAT(outv+i, f);
    }
  

  outlet_list(x->list, 0, outc, outv);

  freebytes(outv, outc * sizeof(t_atom));
}


//begin the decoding cycle--this methodology
//will change once the current system is functioning
//perhaps there will be an 'auto' mode...
void recog_tilde_output(t_recog_tilde *x)
{
  x->writeout = 1;
}

void recog_tilde_auto(t_recog_tilde *x)
{
  x->decodeGo = 1;
  x->automode = 1;
  x->decodeReadPoint = 0;
  recog_tilde_reinit(x);
}

void recog_tilde_start(t_recog_tilde *x)
{
  x->decodeGo = 1;
  x->automode = 0;
  recog_tilde_reinit(x);
}

void recog_tilde_stop(t_recog_tilde *x)
{



  if(x->decodeGo == 1)
    {
      x->decodeGo = 0;
      x->automode = 0;
      recog_tilde_decode(x);


    }


}

void recog_tilde_reinit(t_recog_tilde *x)
{
 
  /* if(x->decodeBuff)//this is a good test to see if we need to do this */
  /*   { */

  /*     if(x->inBuff)freebytes(x->inBuff, sizeof(t_float) * x->ioBuffSize); */
  /*     // */
      
  /*     if(x->dsBuff)freebytes(x->dsBuff, sizeof(t_float) * x->dsBuffSize); */
  /*     // */
      
  /*     if(x->decodeBuff)freebytes(x->decodeBuff, sizeof(int16) * x->decodeBuffSize); */
  /*     // */
      
  /*     if(x->decodeAutoBuff)freebytes(x->decodeAutoBuff, sizeof(int16) * x->autoBuffSize); */
  /*     // */
      
  /*     x->inBuff = (t_float *)t_getbytes(0); */
  /*     x->inBuff = (t_float *)t_resizebytes(x->inBuff, 0, sizeof(t_float) * x->ioBuffSize); */
      
  /*     // */
  /*     x->dsBuff = (t_float *)t_getbytes(0); */
  /*     x->dsBuff = (t_float *)t_resizebytes(x->dsBuff, 0, sizeof(t_float) * x->dsBuffSize); */
      
  /*     // */
  /*     x->decodeBuff = (int16 *)t_getbytes(0); */
  /*     x->decodeBuff = (int16 *)t_resizebytes(x->decodeBuff, 0, sizeof(int16) * x->decodeBuffSize); */
      
  /*     // */
  /*     x->decodeAutoBuff = (int16 *)t_getbytes(0); */
  /*     x->decodeAutoBuff = (int16 *)t_resizebytes(x->decodeAutoBuff, 0, sizeof(int16) * x->autoBuffSize); */

  /*     // */
  /*     x->hyp = (const char *)t_getbytes(0); */
  /*     x->hyp = (const char *)t_resizebytes(x->hyp, 0, sizeof(char) * x->charBuffSize); */
      
  /*     // */
  /*     x->charBuff = (char *)t_getbytes(0); */
  /*     x->charBuff = (char *)t_resizebytes(x->charBuff, 0, sizeof(char) * x->charBuffSize); */
      
  /*     // */
  /*     x->wordHisto = (t_int *)t_getbytes(0); */
  /*     x->wordHisto = (t_int *)t_resizebytes(x->wordHisto, 0, sizeof(t_int) * 50); */
      
  /*     // */
  /*     x->prevWordHisto = (t_int *)t_getbytes(0); */
  /*     x->prevWordHisto = (t_int *)t_resizebytes(x->prevWordHisto, 0, sizeof(t_int) * 50); */
      
      
  /*     // */
  /*     x->x_histo = (t_histogram *)t_getbytes(0); */
  /*     x->x_histo = (t_histogram *)t_resizebytes(x->x_histo, 0, sizeof(t_int) * 50); */
           
  /*     //reset the read/write points */
  /*     x->rmndr = x->dsReadPoint = x->ioReadPoint = 0.0; */
  /*     x->dsWritePoint = x->ioWritePoint = x->decodeWritePoint = x->decodeReadPoint = x->decodeWritePoint = 0; */
  /*     x->wordCount = x->prevWordCount = 0; */
  /*   } */



}

//----------------------------------
//----------------------------------

//pd glue

//perform routine
t_int *recog_tilde_perform(t_int *w)
{
  t_recog_tilde *x = (t_recog_tilde *)(w[1]);
  t_sample *in = (t_sample *)(w[2]);
  t_sample *out = (t_sample *)(w[3]);
  t_int n = (t_int)(w[4]);

  int i;


  for(i=0; i<n; i++)
    {  
      x->inBuff[x->ioWritePoint++] = in[i];
      if(x->ioWritePoint>=x->ioBuffSize)
	x->ioWritePoint -= x->ioBuffSize;

      if(x->writeout)
	{
	  out[i] = x->decodeAutoBuff[x->writeread++];
	  if(x->writeread>=x->autoBuffSize)
	    {
	      x->writeout = 0;
	      x->writeread = 0;
	    }
	}
      else out[i] = 0.0;
    }

 

  if(x->decodeGo == 0)
    return (w+5);
  else
    
    {
      down_sample(x, n);
   
    }
  return (w+5);
  
}

static void recog_tilde_dsp(t_recog_tilde *x, t_signal **sp)
{
  dsp_add(recog_tilde_perform, 4, x, sp[0]->s_vec, sp[1]->s_vec, sp[0]->s_n);
}

static void *recog_tilde_new(int argc, t_atom *argv)
{
  t_recog_tilde *x = (t_recog_tilde *)pd_new(recog_tilde_class);

  outlet_new(&x->x_obj, &s_signal);

  x->list = outlet_new(&x->x_obj, &s_list); 
  x->sampleRate = sys_getsr();
  x->dsBuffSize = 64;
  x->ioBuffSize = 64;
  x->decodeBuffSize = 262144;//about 16.4 seconds worth @ 16k
  x->autoBuffSize = 65536;//4.1 secs @ 16k
  x->autoBlock = 16384;
  x->charBuffSize = 999999;

  x->rmndr = x->dsReadPoint = x->ioReadPoint = 0.0;
  x->dsWritePoint = x->ioWritePoint = x->decodeWritePoint = x->decodeReadPoint = 0;

  x->decodeStart = x->decodeStop = x->decodeGo = x->prevLen = x->charBuffrp = 0;

  x->automode = x->autoCounter = 0;

  x->wordCount = x->prevWordCount = x->writeout = x->writeread = 0;
  
  
  //sphinx setup stuff:
  //recodsider this whole thing:
  hmm=gensym("./model/hmm/en_US/hub4wsj_sc_8k");
  dict=gensym("./model/lm/en/turtle.dic");
  lm=gensym("./model/lm/en/turtle.DMP");

  //
  x->inBuff = (t_float *)t_getbytes(0);
  x->inBuff = (t_float *)t_resizebytes(x->inBuff, 0, sizeof(t_float) * x->ioBuffSize);
  
  //
  x->dsBuff = (t_float *)t_getbytes(0);
  x->dsBuff = (t_float *)t_resizebytes(x->dsBuff, 0, sizeof(t_float) * x->dsBuffSize);

  //
  x->decodeBuff = (int16 *)t_getbytes(0);
  x->decodeBuff = (int16 *)t_resizebytes(x->decodeBuff, 0, sizeof(int16) * x->decodeBuffSize);

  //
  x->decodeAutoBuff = (int16 *)t_getbytes(0);
  x->decodeAutoBuff = (int16 *)t_resizebytes(x->decodeAutoBuff, 0, sizeof(int16) * x->autoBuffSize);

  //
  x->hyp = (const char *)t_getbytes(0);
  x->hyp = (const char *)t_resizebytes(x->hyp, 0, sizeof(char) * x->charBuffSize);
  
  //
  x->charBuff = (char *)t_getbytes(0);
  x->charBuff = (char *)t_resizebytes(x->charBuff, 0, sizeof(char) * x->charBuffSize);
  
  //
  x->wordHisto = (t_int *)t_getbytes(0);
  x->wordHisto = (t_int *)t_resizebytes(x->wordHisto, 0, sizeof(t_int) * 50);
  
  //
  x->prevWordHisto = (t_int *)t_getbytes(0);
  x->prevWordHisto = (t_int *)t_resizebytes(x->prevWordHisto, 0, sizeof(t_int) * 50);

 
  //
  x->x_histo = (t_histogram *)t_getbytes(0);
  x->x_histo = (t_histogram *)t_resizebytes(x->x_histo, 0, sizeof(t_int) * 50);

  post("size of char ptr : %d", sizeof(char *));
  recog_tilde_init(x); 

  return (void*)x;
}

static void recog_tilde_free(t_recog_tilde *x)
{

  fclose(fp_ps);

  free(x->inBuff);
  //

  free(x->dsBuff);
  //

  free(x->decodeBuff);
  //

  free(x->decodeAutoBuff);
  //

  free(x->hyp);
  //

  free(x->charBuff);
  //

  free(x->charBuff);
  //

  ps_free(ps);
  //

  free(x->wordHisto);
  //

  free(x->prevWordHisto);
  //

  free(x->x_histo);
  //
}

void recog_tilde_setup(void)
{
  recog_tilde_class = class_new(gensym("recog~"),
				(t_newmethod)recog_tilde_new,
				(t_method)recog_tilde_free,
				sizeof(t_recog_tilde),
				0,
				A_GIMME,
				0);

  class_addmethod(recog_tilde_class,
		  (t_method)recog_tilde_dsp,
		  gensym("dsp"),
		  0);

  CLASS_MAINSIGNALIN(recog_tilde_class, t_recog_tilde, f);

 class_addmethod(recog_tilde_class,
		  (t_method)recog_tilde_output,
		  gensym("output"),
		  0); 

class_addmethod(recog_tilde_class,
		  (t_method)recog_tilde_start,
		  gensym("start"),
		  0);

 class_addmethod(recog_tilde_class,
		  (t_method)recog_tilde_stop,
		  gensym("stop"),
		  0);

 class_addmethod(recog_tilde_class,
		  (t_method)recog_tilde_auto,
		  gensym("auto"),
		  0);


/*  class_addbang(recog_tilde_class, */
/* 	       write_frame_to_sf); */

}