KT AI MAKERS KIT 사용설명서 (Node JS 편).indd

Transcription

1 KT AI MAKERS KIT

2

3 3

4 4

5 5

6 6

7 7

8 8

9

10

11

12 12 01

13 13 02

14 03

15 15 04

16 16 05

17 17 06

18 18 07

19 19 08

20 20

21 21

22 22

23 23

24 24

25 25

26 26

27 27

28 28

29 29

30 30

31 31

32 32

33 33

34 34

35 35

36 36

37 37

38 38

39 39

40 40

41 41

42 42

43 43

44 44

45 45

46 46

47 47

48 48

49 49

50 50

51

52 52

53 53

54 54

55 55

56 56

57 57

58 58

59 59

60 60

61 61

62 62

63 63

64 64 const record=require('node-record-lpcm16'); const kwstext=['','','','']; const Speaker=require('speaker'); const fs=require('fs'); //node version check const nodeversion=process.version.split('.')[0]; let ktkws=null; if(nodeversion==='v6') ktkws=require('./ktkws'); else if(nodeversion==='v8') ktkws=require('./ktkws_v8'); //for play sample sound const soundbuffer=fs.readfilesync('../data/sample_sound.wav'); const pcmplay=new Speaker({ channels:1, bitdepth:16, samplerate:16000 //for setting kws type const kwsflag=parseint(process.argv[2]); let res=ktkws.initialize('../data/kwsmodel.pack'); console.log('initialize KWS:'+res); res=ktkws.startkws(kwsflag); console.log('start KWS:'+res); //for getting microphone input function initmic(){ return record.start({ sampleratehertz: 16000, threshold: 0, verbose: false, recordprogram: 'arecord', }) }; let mic=initmic(); mic.on('data',(data)=>{ //push pcm data to ktkws library result=ktkws.pushbuffer(data); if(result===1) { console.log("kws Detected"); //play sample sound pcmplay.write(soundbuffer); } console.log('say :'+kwstext[kwsflag]);

65 65 const record=require('node-record-lpcm16'); const kwstext=['','','','']; const Speaker=require('speaker'); const fs=require('fs'); //node version check const nodeversion=process.version.split('.')[0]; let ktkws=null; if(nodeversion==='v6') ktkws=require('./ktkws'); else if(nodeversion==='v8') ktkws=require('./ktkws_v8'); //for play sample sound const soundbuffer=fs.readfilesync('../data/sample_sound.wav'); const pcmplay=new Speaker({ channels:1, bitdepth:16, samplerate:16000

66 66 //for setting kws type const kwsflag=parseint(process.argv[2]); let res=ktkws.initialize('../data/kwsmodel.pack'); console.log('initialize KWS:'+res); res=ktkws.startkws(kwsflag); console.log('start KWS:'+res); //for getting microphone input function initmic(){ return record.start({ sampleratehertz: 16000, threshold: 0, verbose: false, recordprogram: 'arecord', }) };

67 67 let mic=initmic(); mic.on('data',(data)=>{ //push pcm data to ktkws library result=ktkws.pushbuffer(data); if(result===1) { console.log("kws Detected"); //play sample sound pcmplay.write(soundbuffer); } console.log('say :'+kwstext[kwsflag]);

68 68

69 69

70 70

71 71

72 72

73 73 const record=require('node-record-lpcm16'); const aikit=require('./aimakerskitutil'); const client_id=''; const client_key=''; const client_secret=''; const json_path=''; const cert_path='../data/ca-bundle.pem'; const proto_path='../data/gigagenierpc.proto'; function initmic(){ return record.start({ sampleratehertz: 16000, threshold: 0, verbose: false, recordprogram: 'arecord', silence: '10.0', }) }; let writeflag=0; //aikit.initialize(client_id,client_key,client_secret,cert_path,proto_path); aikit.initializejson(json_path,cert_path,proto_path); const ktstt=aikit.getvoice2text(); ktstt.on('error',(error)=>{ console.log('error:'+error); ktstt.on('data',(data)=>{ console.log('data:'+json.stringify(data)); ktstt.on('end',()=>{ console.log('pcm end'); record.stop(); writeflag=0; ktstt.end(); ktstt.write({reqoptions:{mode:0,lang:0} writeflag=1; const mic=initmic(); mic.on('data',(data)=>{ if(writeflag===1) ktstt.write({audiocontent:data console.log('say something');

74 74 const record=require('node-record-lpcm16'); const aikit=require('./aimakerskitutil'); const client_id=''; const client_key=''; const client_secret=''; const json_path=''; const cert_path='../data/ca-bundle.pem'; const proto_path='../data/gigagenierpc.proto'; function initmic(){ return record.start({ sampleratehertz: 16000, threshold: 0, verbose: false, recordprogram: 'arecord', silence: '10.0', }; })

75 75 let writeflag=0; //aikit.initialize(client_id,client_key,client_secret,cert_path,proto_path); aikit.initializejson(json_path,cert_path,proto_path); const ktstt=aikit.getvoice2text(); ktstt.on('error',(error)=>{ console.log('error:'+error); ktstt.on('data',(data)=>{ console.log('data:'+json.stringify(data)); ktstt.on('end',()=>{ console.log('pcm end'); record.stop(); writeflag=0; ktstt.end();

76 76 ktstt.write({reqoptions:{mode:0,lang:0} writeflag=1; const mic=initmic(); mic.on('data',(data)=>{ if(writeflag===1) ktstt.write({audiocontent:data console.log('say something');

77 77

78 78

79 79

80 80 const aikit=require('./aimakerskitutil'); const client_id=''; const client_key=''; const client_secret=''; const json_path=''; const cert_path='../data/ca-bundle.pem'; const proto_path='../data/gigagenierpc.proto'; //aikit.initialize(client_id,client_key,client_secret,cert_path,proto_path); aikit.initializejson(json_path,cert_path,proto_path); aikit.gettext2voiceurl({lang:0,text:'..'},(err,msg)=>{ console.log('err:'+json.stringify(err)+' msg:'+json.stringify(msg)); })

81 81

82 82

83 83

84 84

85 85

86 86

87 87 const Speaker=require('speaker'); const pcmplay=new Speaker({ channels:1, bitdepth:16, samplerate:16000 const aikit=require('./aimakerskitutil'); const client_id=''; const client_key=''; const client_secret=''; const json_path=''; const cert_path='../data/ca-bundle.pem'; const proto_path='../data/gigagenierpc.proto'; //aikit.initialize(client_id,client_key,client_secret,cert_path,proto_path); aikit.initializejson(json_path,cert_path,proto_path); kttts=aikit.gettext2voicestream({text:'',lang:0,mode:0 kttts.on('error',(error)=>{ console.log('error:'+error); kttts.on('data',(data)=>{ if(data.streamingresponse==='resoptions' && data.resoptions. resultcd===200) console.log('stream send. format:'+data.resoptions.format); if(data.streamingresponse==='audiocontent') { pcmplay.write(data.audiocontent); } else console.log('msg received:'+json.stringify(data)); kttts.on('end',()=>{ console.log('pcm end'); function finish(){ console.log('tts played'); }; settimeout(finish,5000);

88 88 const Speaker=require('speaker'); const pcmplay=new Speaker({ channels:1, bitdepth:16, samplerate:16000 const aikit=require('./aimakerskitutil'); const client_id=''; const client_key=''; const client_secret=''; const json_path=''; const cert_path='../data/ca-bundle.pem'; const proto_path='../data/gigagenierpc.proto';

89 89 //aikit.initialize(client_id,client_key,client_secret,cert_path,proto_path); aikit.initializejson(json_path,cert_path,proto_path); kttts=aikit.gettext2voicestream({text:'..',lang:0,mode:0 kttts.on('error',(error)=>{ console.log('error:'+error); kttts.on('data',(data)=>{ if(data.streamingresponse==='resoptions' && data.resoptions.resultcd===200) console.log('stream send. format:'+data.resoptions.format); if(data.streamingresponse==='audiocontent') { pcmplay.write(data.audiocontent); } else console.log('msg received:'+json.stringify(data));

90 90 kttts.on('end',()=>{ console.log('pcm end'); function finish(){ console.log('tts played'); }; settimeout(finish,5000);

91 91

92 92

93 93

94 94

95 95

96 96 const aikit=require('./aimakerskitutil'); const client_id=''; const client_key=''; const client_secret=''; const json_path=''; const cert_path='../data/ca-bundle.pem'; const proto_path='../data/gigagenierpc.proto'; //aikit.initialize(client_id,client_key,client_secret,cert_path,proto_path); aikit.initializejson(json_path,cert_path,proto_path); aikit.querybytext({querytext:'',usersession:'12345',deviceid:'hellodevie'},(err,msg)=>{ if(err){ console.log(json.stringify(err)); } else { console.log('msg:'+json.stringify(msg)); } })

97 97

98 98

99 99

100 100

101 101

102 102

103 103 const record=require('node-record-lpcm16'); const aikit=require('./aimakerskitutil'); const client_id=''; const client_key=''; const client_secret=''; const json_path=''; const cert_path='../data/ca-bundle.pem'; const proto_path='../data/gigagenierpc.proto'; //aikit.initialize(client_id,client_key,client_secret,cert_path,proto_path); aikit.initializejson(json_path,cert_path,proto_path); function initmic(){ return record.start({ sampleratehertz: 16000, threshold: 0, verbose: false, recordprogram: 'arecord', silence: '10.0', }) }; const query=aikit.querybyvoice((err,msg)=>{ if(err){ console.log(json.stringify(err)); } else { console.log('msg:'+json.stringify(msg)); } record.stop(); }) query.write({reqoptions:{lang:0,usersession:'12345',deviceid:'d tp808iqktzq'} const mic=initmic(); mic.on('data',(data)=>{ query.write({audiocontent:data console.log('say something');

104 104 const query=aikit.querybyvoice((err,msg)=>{ if(err){ console.log(json.stringify(err)); } else { console.log('msg:'+json.stringify(msg)); } record.stop(); }) query.write({reqoptions:{lang:0,usersession:'12345',deviceid:'d tp808iqktzq'} const mic=initmic(); mic.on('data',(data)=>{ query.write({audiocontent:data console.log('say something');

105 105

106 106

107 107

108 108

109 109

110 110 const record=require('node-record-lpcm16'); const aikit=require('./aimakerskitutil'); //node version check const nodeversion=process.version.split('.')[0]; let ktkws=null; if(nodeversion==='v6') ktkws=require('./ktkws'); else if(nodeversion==='v8') ktkws=require('./ktkws_v8'); //for playing pcm sound const Speaker=require('speaker'); const fs=require('fs'); const soundbuffer=fs.readfilesync('../data/sample_sound.wav'); const pcmplay=new Speaker({ channels:1, bitdepth:16, samplerate:16000 const client_id=''; const client_key=''; const client_secret=''; const json_path=''; const cert_path='../data/ca-bundle.pem'; const proto_path='../data/gigagenierpc.proto'; const kwstext=['','','','']; const kwsflag=parseint(process.argv[2]); function initmic(){ return record.start({ sampleratehertz: 16000, threshold: 0, verbose: false, recordprogram: 'arecord', }) };

111 111 ktkws.initialize('../data/kwsmodel.pack'); ktkws.startkws(kwsflag); let mic=initmic(); //aikit.initialize(client_id,client_key,client_secret,cert_path,proto_path); aikit.initializejson(json_path,cert_path,proto_path); let mode=0;//0:kws, 1:stt let ktstt=null; mic.on('data',(data)=>{ if(mode===0){ result=ktkws.pushbuffer(data); if(result===1) { console.log("kws Detected"); pcmplay.write(soundbuffer); settimeout(startstt,1000); } } else { ktstt.write({audiocontent:data } console.log('say :'+kwstext[kwsflag]); function startstt(){ ktstt=aikit.getvoice2text(); ktstt.on('error',(error)=>{ console.log('error:'+error); ktstt.on('data',(data)=>{ console.log('stt result:'+json.stringify(data)); if(data.resultcd!==200) mode=0; ktstt.on('end',()=>{ console.log('stt text stream end'); mode=0; ktstt.write({reqoptions:{mode:0,lang:0} mode=1; };

112 112 //aikit.initialize(client_id,client_key,client_secret,cert_path,proto_path); aikit.initializejson(json_path,cert_path,proto_path); let mode=0;//0:kws, 1:stt let ktstt=null; mic.on('data',(data)=>{ if(mode===0){ result=ktkws.pushbuffer(data); if(result===1) { console.log("kws Detected"); pcmplay.write(soundbuffer); settimeout(startstt,1000); } } else { ktstt.write({audiocontent:data }

113 113 console.log('say :'+kwstext[kwsflag]); function startstt(){ ktstt=aikit.getvoice2text(); ktstt.on('error',(error)=>{ console.log('error:'+error); ktstt.on('data',(data)=>{ console.log('stt result:'+json.stringify(data)); if(data.resultcd!==200) mode=0; ktstt.on('end',()=>{ console.log('stt text stream end'); mode=0; ktstt.write({reqoptions:{mode:0,lang:0} mode=1; };

114 114

115 115

116 116

117 117 const record=require('node-record-lpcm16'); const aikit=require('./aimakerskitutil'); const Speaker=require('speaker'); const fs=require('fs'); //for playing pcm sound const soundbuffer=fs.readfilesync('../data/sample_sound.wav'); const pcmplay=new Speaker({ channels:1, bitdepth:16, samplerate:16000 //node version check const nodeversion=process.version.split('.')[0]; let ktkws=null; if(nodeversion==='v6') ktkws=require('./ktkws'); else if(nodeversion==='v8') ktkws=require('./ktkws_v8'); const client_id=''; const client_key=''; const client_secret=''; const json_path='./clientkey.json'; const cert_path='../data/ca-bundle.pem'; const proto_path='../data/gigagenierpc.proto'; const kwstext=['','','','']; const kwsflag=parseint(process.argv[2]); let pcm=null; function initmic(){ return record.start({ sampleratehertz: 16000, threshold: 0, verbose: false, recordprogram: 'arecord', }) };

118 118 ktkws.initialize('../data/kwsmodel.pack'); ktkws.startkws(kwsflag); let mic=initmic(); //aikit.initialize(client_id,client_key,client_secret,cert_path,proto_path); aikit.initializejson(json_path,cert_path,proto_path); let mode=0;//0:kws, 1:queryByVoice let ktstt=null; mic.on('data',(data)=>{ if(mode===0){ //1) result=ktkws.pushbuffer(data); if(result===1) { //2) console.log("kws Detected"); pcmplay.write(soundbuffer); settimeout(startqueryvoice,1000); // 3) } } else { ktstt.write({audiocontent:data //4) } console.log('say :'+kwstext[kwsflag]); function startqueryvoice(){ ktstt=aikit.querybyvoice((err,msg)=>{ if(err){ console.log(json.stringify(err)); mode=0; } else { console.log('msg:'+json.stringify(msg)); const action=msg.action[0]; if(action){ const acttype=action.acttype; let mesg=action.mesg; console.log('acttype:'+acttype+' mesg:'+mesg);

119 119 //5) if(acttype==='99' acttype==='601' acttype==='631' acttype==='607' acttype==='608' acttype==='606' acttype==='9999'){ if(acttype==='9999'){ let textprocess=action.serviceid.split('systemmsg') [1].split(',')[0].substring(3,); mesg=textprocess.substring(0,textprocess.length-1); } //6) let kttts=aikit.gettext2voicestream({text:mesg,lang:0,mode:0 kttts.on('error',(error)=>{ console.log('error:'+error); //7) kttts.on('data',(data)=>{ if(data.streamingresponse==='resoptions' && data.resoptions. resultcd===200) console.log('stream send. format:'+data.resoptions.format); if(data.streamingresponse==='audiocontent') { pcmplay.write(data.audiocontent); } else console.log('msg received:'+json.stringify(data)); kttts.on('end',()=>{ console.log('pcm end'); mode=0;//9) } else mode=0//9) } else mode=0;//9) } ktstt.write({reqoptions:{lang:0,usersession:'12345',deviceid:'d tp808iqktzq'} mode=1;//8) };

120 120 let mode=0;//0:kws, 1:queryByVoice let ktstt=null; mic.on('data',(data)=>{ if(mode===0){ //1) result=ktkws.pushbuffer(data); if(result===1) { //2) console.log("kws Detected"); pcmplay.write(soundbuffer); settimeout(startqueryvoice,1000); // 3) } } else { ktstt.write({audiocontent:data //4) } console.log('say :'+kwstext[kwsflag]);

121 121 function startqueryvoice(){ ktstt=aikit.querybyvoice((err,msg)=>{ if(err){ console.log(json.stringify(err)); mode=0; } else { console.log('msg:'+json.stringify(msg)); const action=msg.action[0]; if(action){ const acttype=action.acttype; let mesg=action.mesg; console.log('acttype:'+acttype+' mesg:'+mesg);

122 122 //5) if(acttype==='99' acttype==='601' acttype==='631' acttype==='607' acttype==='608' acttype==='606' acttype==='9999'){ if(acttype==='9999'){ let textprocess=action.serviceid.split('systemmsg') [1].split(',')[0].substring(3,); mesg=textprocess.substring(0,textprocess.length-1); } //6) let kttts=aikit.gettext2voicestream({text:mesg,lang:0,mode:0 kttts.on('error',(error)=>{ console.log('error:'+error);

123 123 }; //7) kttts.on('data',(data)=>{ if(data.streamingresponse==='resoptions' && data.resoptions. resultcd===200) console.log('stream send. format:'+data.resoptions.format); if(data.streamingresponse==='audiocontent') { pcmplay.write(data.audiocontent); } else console.log('msg received:'+json.stringify(data)); kttts.on('end',()=>{ console.log('pcm end'); mode=0;//9) } else mode=0//9) } else mode=0;//9) } ktstt.write({reqoptions:{lang:0,usersession:'12345',deviceid:'d tp808iqktzq'} mode=1;//8)

124 124

125 125

126 126

127 127 const gpio=require('rpi-gpio'); const record=require('node-record-lpcm16'); const aikit=require('./aimakerskitutil'); const Speaker=require('speaker'); const fs=require('fs'); const soundbuffer=fs.readfilesync('../data/sample_sound.wav'); const pcmplay=new Speaker({ channels:1, bitdepth:16, samplerate:16000 gpio.setup(29,gpio.dir_in,gpio.edge_both); gpio.setup(31,gpio.dir_low,write); function write(err){ if(err) console.log('write Error:'+err); }; const client_id=''; const client_key=''; const client_secret=''; const json_path=''; const cert_path='../data/ca-bundle.pem'; const proto_path='../data/gigagenierpc.proto'; let pcm=null; function initmic(){ return record.start({ sampleratehertz: 16000, threshold: 0, verbose: false, recordprogram: 'arecord', }) };

128 128 aikit.initializejson(json_path,cert_path,proto_path); let sysstat=0; gpio.on('change',(channel,value)=>{ if(channel===29){ console.log('channel:'+channel+' value is '+value); if(value===false){ if(sysstat===0) { sysstat=1; console.log('button Pressed. Start Service'); gpio.write(31,true); pcmplay.write(soundbuffer); settimeout(startqueryvoice,1000); } } } function startqueryvoice(){ let ktstt=aikit.querybyvoice((err,msg)=>{ if(err){ console.log(json.stringify(err)); record.stop(); } else { console.log('msg:'+json.stringify(msg)); const action=msg.action[0]; if(action){ const acttype=action.acttype; const mesg=action.mesg; console.log('acttype:'+acttype+' mesg:'+mesg); if(acttype==='99' acttype==='601' acttype==='631' acttype==='607' acttype==='608' acttype==='606'){ let kttts=aikit.gettext2voicestream({text:mesg,lang:0,mode:0 kttts.on('error',(error)=>{

129 129 console.log('error:'+error); kttts.on('data',(data)=>{ if(data.streamingresponse==='resoptions'&& data.resoptions.resultcd===200) console.log('stream send. format:'+data.resoptions.format); if(data.streamingresponse==='audiocontent') { pcmplay.write(data.audiocontent); } else console.log('msg received:'+json.stringify(data)); kttts.on('end',()=>{ console.log('pcm end'); record.stop(); } else record.stop(); } else record.stop(); }; gpio.write(31,false); sysstat=0; } ktstt.write({reqoptions:{lang:0,usersession:'12345',deviceid:'d tp808iqktzq'} let mic=initmic(); mic.on('data',(data)=>{ ktstt.write({audiocontent:data

130 130 const gpio=require('rpi-gpio'); const record=require('node-record-lpcm16'); const aikit=require('./aimakerskitutil'); const Speaker=require('speaker'); const fs=require('fs'); const soundbuffer=fs.readfilesync('../data/sample_sound.wav'); const pcmplay=new Speaker({ channels:1, bitdepth:16, samplerate:16000 gpio.setup(29,gpio.dir_in,gpio.edge_both); gpio.setup(31,gpio.dir_low,write);

131 131 let sysstat=0; gpio.on('change',(channel,value)=>{ if(channel===29){ console.log('channel:'+channel+' value is '+value); if(value===false){ if(sysstat===0) { sysstat=1; console.log('button Pressed. Start Service'); gpio.write(31,true); pcmplay.write(soundbuffer); settimeout(startqueryvoice,1000); } } }

132 132 }; gpio.write(31,false); sysstat=0; } ktstt.write({reqoptions:{lang:0,usersession:'12345',deviceid:'d tp808iqktzq'} let mic=initmic(); mic.on('data',(data)=>{ ktstt.write({audiocontent:data

133 $ git clone example.git

134 134

135 135 Alternate Function Alternate Function 12C1 SDA 12C1 SCL SPIO MOSI SPIO MISO SPIO SCLK SPI1 MISO 3.3V PWR GPIO 2 GPIO 3 GPIO 4 GND GPIO 17 GPIO 27 GPIO V PWR GPIO 10 GPIO 9 GPIO 11 GND Reserved GPIO 5 GPIO 6 GPIO 13 GPIO 19 GPIO 26 GND V PWR 5V PWR GND UARTO TX UARTO TX GPIO 18 GND GPIO 23 GPIO 24 GND GPIO 25 GPIO 8 GPIO 7 Reserved GND GPIO 12 GND GPIO 16 GPIO 20 GPIO 21 SPIO CS0 SPIO CS1 SPI1 CS0 SPI1 MOSI SOI1 SCLK

136 136

137 137 const gpio = require('rpi-gpio'); const record = require('node-record-lpcm16'); const aikit = require('./aimakerskitutil'); const Speaker = require('speaker'); const fs = require('fs'); const soundbuffer = fs.readfilesync('../data/sample_sound.wav'); const pcmplay = new Speaker({ channels: 1, bitdepth: 16, samplerate: function initmic() { return record.start({ sampleratehertz: 16000, threshold: 0, verbose: false, recordprogram: 'arecord', }) }; const nodeversion = process.version.split('.')[0]; let ktkws=null; if(nodeversion === 'v6') ktkws = require('./ktkws'); else if(nodeversion === 'v8') ktkws = require('./ktkws_v8'); gpio.setup(31, gpio.dir_low, write); function write(err) { if(err) console.log('write Error:'+err); };

138 138 const json_path = '/home/pi/downloads/clientkey.json'; const cert_path = '../data/ca-bundle.pem'; const proto_path = '../data/gigagenierpc.proto'; const kwstext = ['','','','']; const kwsflag = parseint(process.argv[2]); ktkws.initialize('../data/kwsmodel.pack'); ktkws.startkws(kwsflag); let mic = initmic(); aikit.initializejson(json_path, cert_path, proto_path); let mode = 0; let ktstt = null; mic.on('data', (data) => { if(mode === 0) { result = ktkws.pushbuffer(data); if(result === 1) { console.log("kws Detected"); pcmplay.write(soundbuffer); settimeout(startstt, 1000); } } else { ktstt.write({audiocontent: data } console.log('say :' + kwstext[kwsflag]); function startstt() { ktstt = aikit.getvoice2text(); ktstt.on('error',(error) => { console.log('error:'+error);

139 139 ktstt.on('data', (data) => { console.log('stt result:' + JSON.stringify(data)); if (data.resultcd === 201) { controlled(data, function(result) { let responsestring = ""; if (result === 1) { responsestring = '.'; } else if (result === 2) { responsestring = '.'; } else { responsestring = '.'; } } playtts(responsestring, () => { mode = 0; if(data.resultcd!== 200 && data.resultcd!== 201) mode = 0; ktstt.on('end',() => { console.log('stt text stream end');

140 140 }; ktstt.write({reqoptions: {mode: 0, lang: 0} mode=1; function controlled (data, callback){ resulttext = data.recognizedtext; if (resulttext.search("") >= 0){ gpio.write(31, true); callback(1); } else if(resulttext.search("") >= 0){ gpio.write(31, false); callback(2); } } else { callback(0); }

141 141 function playtts (returnvoice, callback) { } kttts = aikit.gettext2voicestream({text: returnvoice, lang: 0, mode: 0 kttts.on('error', (error) => { console.log('error:'+error); kttts.on('data', (data) => { if (data.streamingresponse === 'resoptions' && data.resoptions.resultcd === 200){ console.log('stream send. format:'+data.resoptions.format); } if (data.streamingresponse ==='audiocontent') { pcmplay.write(data.audiocontent); } else { console.log('msg received:'+json.stringify(data)); } kttts.on('end', () => { console.log('pcm end'); settimeout(() => { console.log('tts played'); callback(); }, 800);

142 142

143 143

144 144 $ wget $ tar zxvf bcm tar.gz $ cd bcm $./configure $ make $ sudo make check $ sudo make install $ npm i node-dht-sensor

145 145 const record = require('node-record-lpcm16'); const aikit = require('./aimakerskitutil'); const Speaker = require('speaker'); var sensor = require('node-dht-sensor'); const fs = require('fs'); const soundbuffer = fs.readfilesync('../data/sample_sound.wav'); const pcmplay = new Speaker({ channels: 1, bitdepth: 16, samplerate: function initmic() { return record.start({ sampleratehertz: 16000, threshold: 0, verbose: false, recordprogram: 'arecord', }) }; const nodeversion = process.version.split('.')[0]; let ktkws=null; if(nodeversion === 'v6') ktkws = require('./ktkws'); else if(nodeversion === 'v8') ktkws = require('./ktkws_v8'); const json_path = '/home/pi/downloads/clientkey.json';

146 146 const cert_path = '../data/ca-bundle.pem'; const proto_path = '../data/gigagenierpc.proto'; const kwstext = ['','','','']; const kwsflag = parseint(process.argv[2]); ktkws.initialize('../data/kwsmodel.pack'); ktkws.startkws(kwsflag); let mic = initmic(); aikit.initializejson(json_path, cert_path, proto_path); let mode = 0; let ktstt = null; mic.on('data', (data) => { if(mode === 0) { result = ktkws.pushbuffer(data); if(result === 1) { console.log("kws Detected"); pcmplay.write(soundbuffer); settimeout(startstt, 1000); } } else { ktstt.write({audiocontent: data } console.log('say :' + kwstext[kwsflag]); function startstt() { ktstt = aikit.getvoice2text();

147 147 ktstt.on('error',(error) => { console.log('error:'+error); ktstt.on('data', (data) => { console.log('stt result:' + JSON.stringify(data)); if (data.resultcd === 201) { checkcommand(data, function(result) { let responsestring = ""; readsensor(11, 3, (err, temperature, humidity) => { } if(!err){ if (result === 1) { responsestring = '' + temperature + '.'; } else if (result === 2) { responsestring = '' + humidity + '.'; } else { responsestring = '.'; } playtts(responsestring, () => { mode = 0; } else{ console.log("") } if(data.resultcd!== 200 && data.resultcd!== 201) mode = 0;

148 148 }; ktstt.on('end',() => { console.log('stt text stream end'); ktstt.write({reqoptions: {mode: 0, lang: 0} mode=1; function checkcommand (data, callback){ resulttext = data.recognizedtext; if (resulttext.search("") >= 0){ callback(1); } else if(resulttext.search("") >= 0){ callback(2); } else { callback(0); } } function playtts (returnvoice, callback) { kttts = aikit.gettext2voicestream({text: returnvoice, lang: 0, mode: 0 kttts.on('error', (error) => { console.log('error:'+error); kttts.on('data', (data) => { if (data.streamingresponse === 'resoptions' && data.resoptions.resultcd === 200){ console.log('stream send. format:'+data.resoptions.format); }

149 149 if (data.streamingresponse ==='audiocontent') { pcmplay.write(data.audiocontent); } else { console.log('msg received:'+json.stringify(data)); } kttts.on('end', () => { console.log('pcm end'); settimeout(() => { console.log('tts played'); callback(); }, 800); } function readsensor(dht_ver, DHT_GPIO, callback) { sensor.read(dht_ver, DHT_GPIO, function(err, temperature, humidity) { if (!err) { console.log('temp: ' + temperature.tofixed(1) + ' C, ' + 'humidity: ' + humidity.tofixed(1) + '%' ); callback(err, temperature, humidity); } }

150 150 $ npm i youtube-audio-stream $ npm i fluent-ffmpeg $ sudo apt-get install ffmpeg

151 151 const record = require('node-record-lpcm16'); const aikit = require('./aimakerskitutil'); const fs = require('fs'); const Speaker = require('speaker'); let ktkws = null; const nodeversion = process.version.split('.')[0]; if(nodeversion === 'v6') ktkws = require('./ktkws'); else if(nodeversion === 'v8') ktkws = require('./ktkws_v8'); const request = require('request'); const json_path = '/home/pi/downloads/clientkey.json'; const cert_path = '../data/ca-bundle.pem'; const proto_path = '../data/gigagenierpc.proto'; const kwstext = ['', '', '', '']; const kwsflag = parseint(process.argv[2]); const soundbuffer = fs.readfilesync('../data/sample_sound.wav'); const pcmplay = new Speaker({ channels: 1, bitdepth: 16, samplerate: function initmic() { return record.start({ sampleratehertz: 16000, threshold: 0, verbose: false, recordprogram: 'arecord' };

152 152 ktkws.initialize('../data/kwsmodel.pack'); ktkws.startkws(kwsflag); let mic = initmic(); aikit.initializejson(json_path, cert_path, proto_path); let mode = 0; // 0 : kws, 1 : stt let ktstt = null; let player = null; mic.on('data', (data) => { if(mode === 0) { result = ktkws.pushbuffer(data); if(result === 1) { console.log("kws Detected"); pcmplay.write(soundbuffer); settimeout(startstt, 1000); } } else { ktstt.write({audiocontent: data if(player!= null) { player.stop(); } } function starttts(text, speaker) { let kttts = aikit.gettext2voicestream({text: text, lang: 0, mode: 0 kttts.on('error', (error) => console.log('error:' + error)); kttts.on('data', (data) => { if (data.streamingresponse === 'resoptions' && data.resoptions.resultcd === 200) { console.log('stream send. format:' + data.resoptions.format); }

153 153 } if (data.streamingresponse === 'audiocontent') { speaker.write(data.audiocontent); } else { console.log('msg received:' + JSON.stringify(data)); } kttts.on('end', () => console.log('pcm end')); console.log('say :' + kwstext[kwsflag]); function startstt() { ktstt = aikit.getvoice2text(); ktstt.on('error', (error) => console.log('error:' + error)); ktstt.on('data', async (data) => { console.log('stt result:' + JSON.stringify(data)); if (data.resultcd === 201) { const recognizedtext = data.recognizedtext; if (recognizedtext.includes('') && (recognizedtext.includes('') recognizedtext.includes(''))) { let target = recognizedtext.split('')[0]; console.log('play Target:' + target); const ttscomment = ' ' + target + ' '; starttts(ttscomment, pcmplay); player = new YoutubePlayer(await getyoutubeurl(target), pcmplay); player.play();

154 154 } else if(recognizedtext.includes('')) { if (player!= null) { player.stop(); player = null; const ttscomment = '.'; starttts(ttscomment, pcmplay); } } else { if (player!= null) { player.play(); } } mode = 0; } ktstt.on('end', () => { console.log('stt text stream end'); mode = 0; ktstt.write({reqoptions: {mode: 0, lang: 0} mode = 1; };

155 155 async function getyoutubeurl(keyword) { const youtubeplaybaseurl = ' let urls = await getyoutubesearchlist(keyword); let targeturl = youtubeplaybaseurl + urls[0]; console.log('yt url:' + targeturl); } return targeturl; function getyoutubesearchlist(target) { const querytext = target; } return new Promise((resolve, reject) => { console.log('querytext: ' + querytext); const searchurl = ' const queryurl = encodeuri(searchurl + querytext); request(queryurl, (err, res, body) => { let splitbywatch = body.split('href=\"\/watch?'); let isfirst = true; let urllist = []; splitbywatch.foreach((splittext) => { if (isfirst === true) { isfirst = false; } else { let splitbyquot = splittext.split('"'); urllist.push(splitbyquot[0]); } resolve(urllist);

156 156 const getyoutubestream = require('youtube-audio-stream'); const ffmpeg = require('fluent-ffmpeg'); class YoutubePlayer { constructor(url, speaker) { this.url = url; this.status = 'stop'; this.stream = this._initstream(); this.speaker = speaker; } _initstream() { var ffstream = ffmpeg({source: getyoutubestream(this.url)}).inputformat('mp3').format('wav').outputoptions([ '-b:a 332k', '-ar 16000', '-ab 64', '-ac 1', ]).pipe(); } return ffstream;

157 157 } play () { if(this.status === 'stop') { this.stream.pipe(this.speaker); this.status = 'play'; } } stop () { if(this.status === 'play') { this.stream.unpipe(); this.status = 'stop'; } }

158 (sleep 10 && cd ~/ai-makers-kit/nodejs && node ex9_btnsttdss.js)& $ nano.profile