The Interactive Mirror

emiliopaolini · Jun 30, 2014

Developed by: Emilio Paolini, Francesco Buso Contini, Matteo Di Fede.
We have realized a mirror that it was able to answer to many questions, representing the information on the screen when necessary.
Ex: What time is it? What's ther weather like?
Project's Description: to realize the project we used a single-board UDOO, a webcam, a screen and a micro SD running Android 4.3. First we have written the code for Arduino. The source-code was able to receive a signal from sensor and send it to Android. After we have mounted the mic and the Autofocus Camera 5.0. We have realized two applications: the firts app was a simple output on the screen of the camera. The second app was an app with black screen and a central button. Once pressed the button, app starts to listen and with Text To Speech, it was able to answer. Finally we link the two apps and we have the final app. After the button was changed with an image button representing the mic.
Waiting mic:
Listening mic:

gfasino · Jul 8, 2014

Can you post the source?

peter247 · Jul 8, 2014

Just look for raspberry pi and the magic mirror !!!!!

emiliopaolini · Jul 15, 2014

This is the Main Class

package org.udoo.udoodroidcondemo;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.Locale;

import me.palazzetti.adktoolkit.AdkManager;

import org.udoo.udoodroidcondemo.sounds.Effect;

import android.annotation.SuppressLint;
import android.app.Activity;
import android.content.Context;
import android.content.Intent;
import android.content.SharedPreferences;
import android.graphics.SurfaceTexture;
import android.hardware.Camera;
import android.hardware.usb.UsbManager;
import android.os.AsyncTask;
import android.os.Bundle;
import android.os.CountDownTimer;
import android.speech.RecognitionListener;
import android.speech.RecognizerIntent;
import android.speech.SpeechRecognizer;
import android.speech.tts.TextToSpeech;
import android.util.Log;
import android.view.TextureView;
import android.view.View;
import android.view.animation.Animation;
import android.view.animation.AnimationUtils;
import android.widget.Button;
import android.widget.ImageButton;
import android.widget.TextView;
import android.widget.Toast;

@SuppressLint("NewApi") public class MainActivity extends Activity implements TextureView.SurfaceTextureListener {

private final String TAG = "DroidCon Demo";
private final String PREFS_NAME = "MarioPrefs";
private Camera mCamera;
private Camera.CameraInfo info;
private TextureView mTextureView;

// ADK
private AdkManager mAdkManager;

private final String HELLO_SENDSTRING = "7";

// speech variables
SpeechRecognizer mSpeechRecognizer;

// speech key
private ArrayList<String> ora = new ArrayList<String>(Arrays.asList("ora",
"che ore sono", "dimmi l'ora", "orario", "ora attuale", "dimmi che ore sono"));
private ArrayList<String> meteo= new ArrayList<String>(Arrays.asList("meteo",
"che tempo c'è", "previsioni meteo","tempo","previsioni per oggi"));
private ArrayList<String> grazie = new ArrayList<String>(Arrays.asList("tutto bene grazie", "grazie",
"grazie a te"));
private ArrayList<String> sopra = new ArrayList<String>(Arrays.asList("sopra la panca",
"sotto la panca", "la capra campa", "la capra crepa"));
private ArrayList<String> Siena = new ArrayList<String>(Arrays.asList("conosci siena",
"siena", "amo siena", "qual'è la tua città"));
private ArrayList<String> intelligente = new ArrayList<String>(Arrays.asList("stupido",
"non capisci niente", "non sei intelligente","sei intelligente"));
private ArrayList<String> pizza_strings = new ArrayList<String>(Arrays.asList("pizza", "ti piace la pizza",
"vuoi pizza"));
private ArrayList<String> nome= new ArrayList<String>(Arrays.asList("nome", "il tuo nome",
"chi sei ", "come ti chiami", "che cosa sei", "cosa sei"));
private ArrayList<String> ciao = new ArrayList<String>(Arrays.asList("ciao","salve","hey" ));
private ArrayList<String> da_dove= new ArrayList<String>(Arrays.asList("da dove", " da dove vieni"));
private ArrayList<String> arrivederci= new ArrayList<String>(Arrays.asList("arrivederci","addio","a dopo","ci vediamo"));
private ArrayList<String> allwords = new ArrayList<String>();
private ArrayList<String> storie = new ArrayList<String>(Arrays.asList("storia",
"racconta", "raccontami una storia", "narrami una storia", "parlami di cose belle"));
private ArrayList<String> perle = new ArrayList<String>(Arrays.asList("perla",
"perle", "dammi una perla", "perla di saggezza", "dimmi una perla di saggezza", "dammi una perla di saggezza"));
private ArrayList<String> tortura = new ArrayList<String>(Arrays.asList("torturami",
"tortura", "fammi soffrire"));

//array risposte

String meteo_ris[] = {"io non verrò mai detto","accendi il computer e guarda su internet, io ora non ho voglia","bho, dimmelo tu",
"a vita è troppo corta per chiedermi le previsioni meteo", "guarda fuori dalla finestra",
"le previsioni per oggi prevedono, , no, sono uno specchio non sono giuliacci"};

String da_dove_ris[] = {"vengo da siena, in italia","vengo dalla fabbrica iudù",
"provengo dalle geniali menti di 3 studenti"};

String ciao_ris[] = {"Ciao!","bèlla!","ciao! mi ricordi mia nonna!","Ciao! sei molto procione",
"E T ha una voce più bella"};

String storie_ris[] = {"g g","c'era una volta un bambino troppo curioso. Fu mangiato da un lupo. Quindi smetti di chiedermi queste cose e vai a studiare",
"Gianfrancioschio era una persona talmente poco importante , che la sua storia ancora prima di cominciare. Fine. "};

String perle_ris[]= {"Quando la vita ti dà dei limoni , lanciali a quelli che possengono auto costose","La storia è scritta da chi ha penne funzionanti","a pagamento non è tradimento"};

String tortura_ris[]={"Al mercatino di Lino Maiale, tu trovi un affare su ogni scaffale. Frutta fresca e matura verdura di stagione, sono davvero una grande occasione. Il suo negozio è il più ordinato di tutto quanto il vicinato. Lui lava e spazza tre volte al giorno, pulisce e spolvera tutto intorno. Nel suo mercatino trovi ogni cosa, dalle carote allo sciampo rosa. Pale magliette cuffie e berretti, torte di panna dolci confetti. "
+ "Lino incarta la tua spesa, tu paghi e ti dà il resto. Poi aprendoti la porta, lui ti dice: Torna presto! E adesso coloriamo le mele di rosso e la lattuga di verde."};

// gui
TextView debug_tv;
ImageButton voiceButton;
Animation animationFadeIn;
Animation animationFadeOut;
ImageButton twitterButton;

private String mLastFetchedId = "1";

TextToSpeech tts;
Effect cuteEffect;
Effect goodEffect;
Effect badEffect;
Effect moonwalk;

// Oggetti di prova
Button bottoneNuovo = null;

@Override
protected void onCreate(Bundle savedInstanceState) {

super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);

mAdkManager = new AdkManager((UsbManager) getSystemService(Context.USB_SERVICE));
registerReceiver(mAdkManager.getUsbReceiver(), mAdkManager.getDetachedFilter());

mTextureView = (TextureView) findViewById(R.id.textView);
mTextureView.setSurfaceTextureListener(this);

debug_tv = (TextView) findViewById(R.id.textView);
debug_tv.setVisibility(View.GONE);

bottoneNuovo = (Button) findViewById(R.id.button);

// voiceButton.setOnClickListener(new OnClickListener() {
//
// @Override
// public void onClick(View v) {
// startVoiceRecognitionActivity();
// }
// });

allwords.addAll(ciao);
allwords.addAll(nome);
allwords.addAll(meteo);
allwords.addAll(ora);
allwords.addAll(grazie);
allwords.addAll(sopra);
allwords.addAll(Siena);
allwords.addAll(intelligente);
allwords.addAll(storie);
allwords.addAll(perle);
allwords.addAll(tortura);
allwords.addAll(pizza_strings);
allwords.addAll(da_dove);
allwords.addAll(arrivederci);

animationFadeIn = AnimationUtils.loadAnimation(this, R.anim.fadein);
animationFadeOut = AnimationUtils.loadAnimation(this, R.anim.fadeout);

cuteEffect = new Effect(this, R.raw.thanku);
goodEffect = new Effect(this, R.raw.whohoo);
badEffect = new Effect(this, R.raw.no);
moonwalk = new Effect(this, R.raw.moonwalk);

mSpeechRecognizer = SpeechRecognizer.createSpeechRecognizer(getApplicationContext());
MyRecognitionListener listener = new MyRecognitionListener();
mSpeechRecognizer.setRecognitionListener(listener);

// Get last stored values
SharedPreferences settings = getSharedPreferences(PREFS_NAME, 0);
mLastFetchedId = settings.getString("lastId", "1");

tts = new TextToSpeech(getApplicationContext(),
new TextToSpeech.OnInitListener() {
@Override
public void onInit(int status) {
if (status == TextToSpeech.SUCCESS) {

int result = tts.setLanguage(Locale.ITALY);
tts.setPitch(1.0F);

if (result == TextToSpeech.LANG_MISSING_DATA) {
Log.e("TTS", "Lang missing data");
// missing data, install it
Intent installIntent = new Intent();
installIntent.setAction(TextToSpeech.Engine.ACTION_INSTALL_TTS_DATA);
startActivity(installIntent);
} else {
tts.speak("Ti ascolto, dimmi tutto!", TextToSpeech.QUEUE_FLUSH, null);
}

} else {
Log.e("TTS", "Initilization Failed!");
}
}
});
}
public void onSurfaceTextureAvailable1(SurfaceTexture surfaceTexture, int width, int height) {

info = new Camera.CameraInfo();
int i=0;
mCamera = Camera.open(i);

}
@Override
protected void onPause() {
super.onPause();
if(mSpeechRecognizer!=null){
mSpeechRecognizer.stopListening();
mSpeechRecognizer.cancel();
mSpeechRecognizer.destroy();

}
if(tts !=null){
tts.stop();
tts.shutdown();
}
mSpeechRecognizer = null;
tts = null;
mAdkManager.close();
}

@Override
protected void onStop() {
super.onStop();
SharedPreferences settings = getSharedPreferences(PREFS_NAME, 0);
SharedPreferences.Editor editor = settings.edit();
editor.putString("lastId", mLastFetchedId);
editor.commit();
}

@Override
protected void onDestroy() {
super.onDestroy();
cuteEffect.release();
goodEffect.release();
badEffect.release();
unregisterReceiver(mAdkManager.getUsbReceiver());
}

@Override
protected void onResume() {
super.onResume();
if(mSpeechRecognizer == null){
mSpeechRecognizer = SpeechRecognizer.createSpeechRecognizer(getApplicationContext());
MyRecognitionListener listener = new MyRecognitionListener();
mSpeechRecognizer.setRecognitionListener(listener);
}
mAdkManager.open();

new AdkReadTask().execute();
}

public class AdkReadTask extends AsyncTask<Void, String, Void>
{
private boolean running = true;

public void pause(){
running = false;
}

public Void doInBackground(Void... params)
{
// Log.i("ADK demo bi", "start adkreadtask");
while(running)
{
publishProgress(mAdkManager.readSerial()) ;
}
return null;
}

public void onProgressUpdate(View v,String... progress) {
int distance=(int)progress[0].charAt(0);
if(distance==1)
{
startVoiceRecognitionActivity(v);
}
// Log.i(TAG, "received: " + (int)progress[0].charAt(0));
}
}

int a = (int)(Math.random()*10);

public void onButtonClicked( View v )
{
startVoiceRecognitionActivity(v );
}

private boolean searchCommands (String result) {
boolean found = false;
String stringFounded = "";
for (String string : allwords ) {
if (result.toLowerCase(Locale.ITALY).contains(string.toLowerCase(Locale.ITALY))) {
stringFounded = string;
found = true;

Calendar calendar = new GregorianCalendar();

String orario;
int ore = calendar.get(Calendar.HOUR);
int minuti = calendar.get(Calendar.MINUTE);
int secondi = calendar.get(Calendar.SECOND);

if(calendar.get(Calendar.AM_PM) == 0)
orario = "del mattino";
else
orario = "Pomeridiane";
System.out.println("Sono le : " + ore + ":"
+ minuti + ":" + secondi + " " + orario);
int a ;

//if (ciao.contains(stringFounded)) sendHello();
if (ciao.contains(stringFounded))tts.speak(ciao_ris[(int)(Math.random()*5)], TextToSpeech.QUEUE_FLUSH, null);
else if (nome.contains(stringFounded)) tts.speak("Salve , sono Lollo lo specchio interattivo , non ci crederai ma ho 3 genitori", TextToSpeech.QUEUE_FLUSH, null);
else if (da_dove.contains(stringFounded))tts.speak(da_dove_ris[(int)(Math.random()*3)], TextToSpeech.QUEUE_FLUSH, null);
else if (meteo.contains(stringFounded))tts.speak(meteo_ris[(int)(Math.random()*5)], TextToSpeech.QUEUE_FLUSH, null);
else if (ora.contains(stringFounded)) tts.speak("Sono le ore: " + ore +", "+ minuti + "minuti e, " + secondi + "secondi " + orario , TextToSpeech.QUEUE_FLUSH, null);
else if (grazie.contains(stringFounded)) tts.speak("prego, non c'è di che!", TextToSpeech.QUEUE_FLUSH, null);
else if (sopra.contains(stringFounded)) tts.speak("stai zitto! non sono il fottuto iphone di Emilio, e la prossima volta quella stronza di siri potrebbe andare a quel paese! ", TextToSpeech.QUEUE_FLUSH, null);
else if (Siena.contains(stringFounded)) tts.speak("conosco Siena come le mie tasche! ah! già, sono uno specchio, non ho le tasche", TextToSpeech.QUEUE_FLUSH, null);
else if (intelligente.contains(stringFounded)) tts.speak("no, non sono intelligente, ma sono capace di riflettere!", TextToSpeech.QUEUE_FLUSH, null);
else if (storie.contains(stringFounded)) tts.speak(storie_ris[(int)(Math.random()*3)], TextToSpeech.QUEUE_FLUSH, null);
else if (perle.contains(stringFounded))tts.speak(perle_ris[(int)(Math.random()*3)], TextToSpeech.QUEUE_FLUSH, null);
else if (tortura.contains(stringFounded)) tts.speak(tortura_ris[0], TextToSpeech.QUEUE_FLUSH, null);
else if (pizza_strings.contains(stringFounded)) tts.speak("Forse non ti sei accorto , ma sono uno specchio! non mangio la pizza! ", TextToSpeech.QUEUE_FLUSH, null);
else if (arrivederci.contains(stringFounded)) sendGoodby();

break;
}
}
return found;
}

private void sendGoodby() {
Log.i(TAG, "hello case");
tts.speak("Arrivederci", TextToSpeech.QUEUE_FLUSH, null);
mAdkManager.writeSerial(HELLO_SENDSTRING);
returnToNormalState(5000);
}

private void returnToNormalState(int millis) {
new CountDownTimer(millis, 1000) {

public void onTick(long millisUntilFinished) {
//do nothing, just let it tick
}

public void onFinish() {
Log.i(TAG, "returnToNormalState");
}
}.start();
}

@SuppressWarnings("unused")
private void showToastMessage(String message){
Toast.makeText(this, message, Toast.LENGTH_SHORT).show();
}

/**
* Fire an intent to start the voice recognition activity.
* @param v
*/
public void startVoiceRecognitionActivity(View v) {
Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
intent.putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, this.getPackageName());

mSpeechRecognizer.startListening(intent);

new CountDownTimer(5000, 1000) {

public void onTick(long millisUntilFinished) {
//do nothing, just let it tick
}

public void onFinish() {
Log.i(TAG, "stop countdown");
mSpeechRecognizer.stopListening();
//voiceButton.setBackgroundResource(R.drawable.nose_up);
}
}.start();
}

class MyRecognitionListener implements RecognitionListener {

@Override
public void onBeginningOfSpeech() {
Log.d("Speech", "onBeginningOfSpeech");
}

@Override
public void onBufferReceived(byte[] buffer) {
Log.d("Speech", "onBufferReceived");
}

@Override
public void onEndOfSpeech() {
Log.d("Speech", "onEndOfSpeech");
}

@Override
public void onError(int error) {
Log.d("Speech", "onError: " + error);
if (error == 7 || error == 5 || error == 6 ){
tts.speak("scusa, non ho capito", TextToSpeech.QUEUE_FLUSH, null);
}
}

@Override
public void onEvent(int eventType, Bundle params) {
Log.d("Speech", "onEvent");
}

@Override
public void onPartialResults(Bundle partialResults) {
Log.d("Speech", "onPartialResults");
}

@Override
public void onReadyForSpeech(Bundle params) {
Log.d("Speech", "onReadyForSpeech");
//voiceButton.setBackgroundResource(R.drawable.nose_down);
}

@Override
public void onResults(Bundle results) {
Log.d("Speech", "onResults");
ArrayList<String> resultsArray = results.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION);
// showToastMessage("messaggio: " + resultsArray.get(0));
debug_tv.setText("Received text: " + resultsArray.get(0));

if (!searchCommands(resultsArray.get(0))) {
tts.speak("scusa, non ho capito", TextToSpeech.QUEUE_FLUSH, null);
//showToastMessage("Sentence is not recognized");
}

for (int i = 0; i < resultsArray.size();i++ ) {
Log.d("Speech", "result=" + resultsArray.get(i));
}
}

@Override
public void onRmsChanged(float rmsdB) {
// Log.d("Speech", "onRmsChanged");
}

}

@Override
public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width,
int height) {
// TODO Auto-generated method stub

}

@Override
public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) {
// TODO Auto-generated method stub
return false;
}

@Override
public void onSurfaceTextureUpdated(SurfaceTexture surface) {
// TODO Auto-generated method stub

}

@Override
public void onSurfaceTextureAvailable(SurfaceTexture surface, int width,
int height) {
// TODO Auto-generated method stub

}

}
Click to expand...

Log in or Sign up

The Interactive Mirror

emiliopaolini New Member

gfasino New Member

peter247 Member

emiliopaolini New Member

Share This Page

Log in or Sign up

The Interactive Mirror

emiliopaolini New Member

gfasino New Member

peter247 Member

emiliopaolini New Member

Share This Page

Useful Searches