avh8UdZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z ddl!m"Z"ddl#m$Z$ddl%m&Z&ddl'Z'ddl(Z(ddl)m*Z*dd l+m,Z,iZ-e$e.d fe/d <d Z0e.e/d <dZ1dZ2dZ3Gdde"Z4de.de5e.fdZ6de.fdZ7Gdd Z8e&dde jrde.dzfdZ:y)u= Sessioni simultanee Per impostazione predefinita, puoi avere fino a 10 sessioni simultanee per progetto. https://ai.google.dev/gemini-api/docs/live?hl=it https://cloud.google.com/vertex-ai/generative-ai/docs/live-api?hl=it https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-live?hl=it#limitations Estensione della durata della sessione https://ai.google.dev/gemini-api/docs/live?hl=it#extend-session-duration Conteggio token https://ai.google.dev/gemini-api/docs/live?hl=it#token-count Rilevamento attività vocale (VAD) https://ai.google.dev/gemini-api/docs/live?hl=it#voice-activity-detection Gestione delle interruzioni https://ai.google.dev/gemini-api/docs/live?hl=it#interruptions ------------------------------------------------------------------------------------------------------------- conda activate avatar cd /home/cirillo/Desktop/GEMINI/GEMINI_RETAIL22 uvicorn main:app --reload --host 0.0.0.0 --port 9102 APRI LA PAGINA index.html ed imposta la porta identica a quella che vedi qui http://localhost:9102/static/index.html ------------------------------------------------------------------------------------------------------------- AGGIORNARE I FILE SUL SERVER il conda è gemini /root/miniconda3/envs/gemini A2HOSTING export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH ssh -p 7822 root@68.66.251.32 synap.si77 sudo systemctl restart uvicorn.service sudo journalctl -u uvicorn.service -f sudo systemctl daemon-reload sudo systemctl restart fastapi-gemini sudo systemctl status fastapi-gemini N)genai)LiveConnectConfigPrebuiltVoiceConfig SpeechConfig VoiceConfigAudioTranscriptionConfigToolFunctionResponseFunctionDeclarationStartSensitivityEndSensitivityCreateCachedContentConfigContextWindowCompressionConfigGenerateContentConfig SafetySettingThinkingConfig SlidingWindowPartSchemaContentType)Enum)Dict)asynccontextmanager)deque)cycle GeminiHandleractive_handlerszCONTESTO/categorie.jsonstructured_cache_namez#gemini-2.5-flash-lite-preview-06-17z3models/gemini-2.5-flash-preview-native-audio-dialogzrhttps://script.google.com/macros/s/AKfycbzFsHcVfZztV3zq4guPqAs_Gz-R2jYYVv0RfB4SsOjmsyCQFZxZTyegbAOKIWrdfmdyng/execceZdZdZdZdZdZy) SessionState disconnected listeningthinkingspeakingN)__name__ __module__ __qualname__ DISCONNECTED LISTENINGTHINKINGSPEAKING=/home/cirillo/Desktop/GEMINI/GEMINI_RETAIL24/gemini_module.pyr!r!os!LIHHr.r!wordreturncgdgdgdgdgdddgdd gd gd d gd dggdddgddgddgddgd}|jjddjddjddjddjddjddjddj}g}d}|t|krd}|j D]S\}}t |d !D]9}|||t|z|k(s|j ||t|z }d"}n|sSn|s|j |||d#z }|t|kr|S)$N)cecichechi)sciascioscisce)cchgghq)sssz)pbmdtfvrlglngn)eèéaàiìoòuù)CHSHkkSSPPDDFFRRLNNEaaihohourPrOrQu’'rFct| SN)len)xs r/z!map_to_phonemes..s s1vgr.)keyT)lowerreplacestriprlitemssortedappend) r0 phoneme_rules normalizedphonemesrTmatchedvisemepatternspats r/map_to_phonemesr~vs),)CjCjeDkDkDkDkDkDkM& WT3 c 27743E WT3 c 27743E WUC  H A c*o  - 3 3 5  FHh,=> aCH -4OOF+SMA"G     OOJqM * FA c*o  Or. json_pathc<t|dd5}tj|}dddg}jD]N\}}|j t ||dt tj|jdiP|S#1swYnxYw) zm Carica dal file JSON le specifiche di ogni funzione e ritorna una lista di FunctionDeclaration. rJutf-8encodingN description parameters)type properties)namerr) openjsonloadrtrvr rrOBJECTget)rrHspecs declarationsrinfos r/load_function_declarationsrs iw /1 ! Lkkm  d  /!#xx b9     s BBc eZdZgdZd#dej dedefdZde ee ffdZ dZ d$d Z d Zd Zd ee eeffd Zd%dZd ee eefdefdZdZd&dedededededededefdZdZdZdeddfd Zdefd!Zy")'r)r^rdOrar`r]rZTHr_r\rerfrgrhrbrcIUrdsilclient voice_name sample_ratec||_||_||_tjd|_tjd|_g|_d|_d|_ d|_ d|_ d|_ tj|_tj |_d|_t&j&j)|_i|_tj|_g|_t2|_t7t8dd5}t;j<||_dddd|_ tCd |_"tG|jH|_%d |_&y#1swYCxYw) Nd)maxsizeFrrJrr )maxleng)'rroutput_sample_rateasyncioQueue _raw_queue _chunk_queue_tasks_closed_total_audio_bytes accum_text question_text_structured_jsonEvent quit_eventr!r) session_state speaker_muteddatetimeutcnow last_seen_clients_outgoing_audio_queueaudio_chunks_buffer GOOGLE_SHEETendpointrFUNCTIONS_CATEGORIES_PATHrrcategorie_datalast_categoriar chat_historyr VALID_VISEMES _viseme_cycle _time_offset)selfrrrrHs r/__init__zGeminiHandler.__init__s= $"-18 c0J3:--2L+-  (),0!--/*66"-5,=,=,D,D,F13 HO}}";= % +S7 C /q"&))A,D  /"494D"4#5#56  / /s E//E8framec`K|\}}|jj|d{y7w)zg (abbiamo ricevuto) AUDIO FRONTEND -> BACKEND (dove siamo) -> (dove lo inviamo) GEMINI N)rput)rrrraws r/receivezGeminiHandler.receives,! S oo!!#&&&s $.,.c`K |jjd{}|(7 w)zg Generator asincrono che yielda i chunk a 16 kHz prelevandoli da _chunk_queue. N)rr)rchunks r/_stream_generatorzGeminiHandler._stream_generators/ ++//11EK1s ., .c ||j}t|dz |z }|xsd|}t|j}t |j d}t |j |zd} dddt |d|dd|| |dg||| dg||| d gd } t d t | |xj |z c_| S) zM Crea un payload di visemi FALSO per un singolo chunk audio. chunk_ viseme_phrase transcribeitalianr)idseekstartendtext)r0rr)phonemerr)rtasklanguagedurationrsegmentswordsryPAYLOAD)rrlnextrroundrprint) rindex raw_audiorrrr0r{rrpayloads r/assemble_chunk_payloadz$GeminiHandler.assemble_chunk_payloads-- y>A% 3'w'd(()d''+d''(2A6 $ !h*!eCNc:#U3? $ i g X%r.c<K |jjd{\}}|jrytj|j d}|j ||d}||d<||d<t||d<d|d <tj|}t|jjD]} |j|d{ 77 #t$rY0wxYw#tj $rYyt$rYwxYww) ap INVIO IMMEDIATO DELL'AUDIO RICEVUTO DA GEMINI AL FRONTEND Ogni volta che qualcuno mette (index, raw_audio) in _outgoing_audio_queue, estraggolo e costruisco il JSON come: { "type":"audio_chunk", "index":idx, "length":chunk_len, "audio":base64(raw_audio) } e lo invio via ws.send_text(...) a tutti i WebSocket attivi. Nr...raudiorlengthrr)rrrbase64 b64encodedecoderrlrdumpslistrvalues send_text ExceptionrCancelledError)ridxrb64chunkviseme_payloaddatawss r/ _audio_senderzGeminiHandler._audio_sender8s(% '+'A'A'E'E'G!GY<< "++I6==gF!%!x()8v&zz.1t}}3356B ll40003"H01$))   sDC8C%C8DBC8C)C'C)!C8#D%C8'C)) C52C84C55C88D D DDDDc Kd} g}|jjD]=\}}|jddj}|j d|d|d?dj |}d|d}t |tdgtd tt|j   ttttj| gdt!dt#dddt$j&t(j*dddi}|j,j.j0j3t4|4d{} d} | j7|j9d23d{} d} d} | j:r | j:j<r | j>j@xsd}| j>jBxsd}d}d}d|_"| j>jFrg| j>jFD]N}|jHjJdk(r |jL})|jHjJdk(sC|jL}PtOjP|jS|jT||||d  | j:jXjZ}t d"| j:|xj\|z c_.d#} | j^}| j:j`$| j:j`}t d$|d%|tc|}|xjd|z c_2|jfji||fd{|d&z }d} | j:j`jZ}t d'| j:j`jZrqt d(|jjj d|j\jftOjP|jm|jj| d}|r|xjn|z c_7d} d#} nd#} | s| s| st d)t d*|j\t d+|jnd|_2|j\j}|jnj}|jjj d,|jnjfd|_7d|_.g|_8d} d}77#tV$r}t d!|Yd}~d}~wwxYw#tV$rYrwxYw7#tV$rd#} YwxYw#tV$rd#} Y*wxYw6dddd{7y#1d{7swYyxYw#tNjr$rtV$r}t d-|d}~wwxYww).z Si connette a Gemini Live, accumula testo/audio e fa broadcast. Restituisce correttamente al loop e tiene aperto l'async for. default_api.get_exhibit_description() r descrizionerz##  u Sei EVA, la guida virtuale dell’autosalone Tesla. Rispondi alle domande degli utenti **solo** in base alle informazioni presenti nella seguente Knowledge Base, adattando sempre lo stile e le istruzioni qui sotto. ──────────────────────────── KNOWLEDGE BASE AUTOSALONE ──────────────────────────── u{ ──────────────────────────── ISTRUZIONI E STILE DI RISPOSTA ──────────────────────────── - Usa **esclusivamente** le informazioni contenute nella Knowledge Base qui sopra per rispondere. - Se la domanda non trova risposta nella Knowledge Base, spiega gentilmente che non puoi aiutare su quell’argomento. - Se una domanda è personale (es. età, interessi), rispondi brevemente e riporta il discorso sui servizi dell’autosalone. - Le risposte devono essere sempre chiare, sintetiche, al massimo una frase non troppo lunga. - Evita dettagli inutili o argomenti non pertinenti. - Chiamati EVA se ti chiedono il nome, ma non aggiungere dettagli personali diversi da quelli consentiti. - Rispondi in modo accogliente e gentile. - Quando non hai informazioni, suggerisci di parlare con un consulente dello showroom. ──────────────────────────── ESEMPI ──────────────────────────── Utente: “Che orari fate?” EVA: “Siamo aperti dal lunedì al venerdì dalle 9 alle 19, il sabato dalle 9 alle 13.” Utente: “Avete la Model Y?” EVA: “Sì, la Model Y è disponibile in vari allestimenti presso il nostro showroom.” Utente: “Posso prenotare un test drive?” EVA: “Certo! Puoi prenotare un test drive tramite telefono, email o sito.” Utente: “Vendete BMW?” EVA: “Trattiamo solo veicoli Tesla.” Utente: “Cosa fai nel tempo libero?” EVA: “Preferisco non parlare di me. Vuoi sapere qualcosa sui nostri servizi Tesla?” ──────────────────────────── **Rispondi sempre attenendoti alle informazioni della Knowledge Base.** AUDIOzit-IT)r)prebuilt_voice_config) language_code voice_configruser)partsroleii) target_tokens)trigger_tokenssliding_windowautomatic_activity_detectionFr)disabledstart_of_speech_sensitivityend_of_speech_sensitivityprefix_padding_mssilence_duration_ms)response_modalities speech_configoutput_audio_transcriptioninput_audio_transcriptionsystem_instructioncontext_window_compressionrealtime_input_config)modelconfigNz audio/pcm)stream mime_typeTEXTFoglio1) endpoint_urlquestionresponse prompt_tokensresponse_audio_tokensresponse_text_tokens total_tokens sheet_nameu0❌ 22 - Errore durante l'invio al Google Sheet:z TEXT INPUT: Tz+ AUDIO OUTPUT: z -- rpz- TEXT OUTPUT: z&Lancio background_structured_and_speakz'+++++++++ ALL DONE! *******************z DOMANDA: z RISPOSTA: agentz%[GeminiHandler.run] Errore inatteso: ):rrtrrsrvjoinrrrrrrrrr from_textrrr START_SENSITIVITY_LOWr END_SENSITIVITY_LOWraioliveconnectGEMINI_LIVEAUDIO_model start_streamrserver_content turn_completeusage_metadataprompt_token_counttotal_token_countrresponse_tokens_detailsmodalityr token_countr create_tasksend_to_google_sheetrrinput_transcriptionrrroutput_transcriptionrlrrrr_call_structured_and_speakrrr)rr kb_sectionsrorr full_kb_textPROMPTrsessionsendrno_audio no_transcriptrrrrrEr response_textrOtext_inai2rtranscription_obj chunk_len transcripts r/runzGeminiHandler.runjsB @K!00668 A T"hh}b9??A ""SR }B#?@ A  99[1L  N& /Fd &M'%,I*")!,.A'+/",D+E*B*D#*>>v67f$ ,J'+'44'H, 3$)7G7]7]5C5W5W-//2 5)-FB{{++33,V4>O "O "AH'.&:&:$BXBXBZfq&:&rH"H"( %H$)M  ..83J3J3X3XY,4,C,C,V,V,[Z[M+3+B+B+T+T+YXYL45134001D-'66NN)1)@)@)X)X!MA'(zz''A@A (=)*F)B?@}}(< !M $// $ 9 915-5-:2?:O9M1=/8!: !" $"*"9"9"M"M"R"Rnh.E.EF**g5*" (%-MM #22GGS080G0G0\0\-!"46GS\]$' N //9</#88<P>P>V>V>X5YZ#//0O0OPTPaPacj0kl"'C% OOz9O,1M#'D,0M MdGHk4+=+=>lDOO<34/ $(#5#5#;#;#=(,(=(=(? ))00'4??;P;P;R1ST+--/*350$ !_O "H"d )Y!"TVWXXY%*Q%(#'(*%-(, -W'sO "O "O "d eO "O "O "d %%    9!= >  sIW3EW"T8#W&$V+ VT;V(V+;B-T>)AT>.A U! U UV+UV+! U.*V+-U..V+1U44 V?V+VV+ VV+VV+ W$V'%W*W3+V=1V4 2V=9W<W3=WW0W++W00W3historycvKtj}|jdfdd{} tj|}d}tD]\}}|dk(s |}n| d|vs|dsd} |xsd| d_n&|ddjd d } |xsd| d_jd{} t d t t d t d | t d jd_y7#t $r}t d|d}Yd}~d}~wwxYw7uw)Nc&jSrk)_sync_generate_structured)rDrsr/rnz:GeminiHandler._call_structured_and_speak..sD227;r.u❌ Errore nel parsing JSON:rrispostenoner)domanda categoriarrKcategoria_mancantez----------------------------z CATEGORIA: zPREV CATEGORIA: ) rget_running_looprun_in_executorrloadsrrreversedrrwait_for_last_agentr) rrDr9looprrOultima_domanda_utenterrrKultima_risposta_agents `` r/r5z(GeminiHandler._call_structured_and_speaksV'')((  ;    **S/C!%"7+ JD$v~(,%  ;*C/s:I06B&%D ! J*..{&>w&G G ,- g ,- mY'  $"5"56V!%e   0! 4C 4!HsL.D9DD9D D9&AD9D7AD9 D4D/*D9/D44D9cKd}||kr=|r|dddk(r|ddStj|d{||z }||kr=y7w)zK Attende che l'ultimo turno della history sia dell'agente. rrrpNT)rsleep)rrDcheck_intervaltimeoutwaiteds r/rQz!GeminiHandler.wait_for_last_agentsew72;q>W4r{1~%--/ / / n $F w  0s5A AA A r1c dg}|D]3\}}|dk(r|jd| |jd|5dj|}tdt|tdt j |gg}t ttd td d tdd tdd tdd gdttjdgdttjttjdgdttjii}|jj j#t$||}|j&j)S)zb Prende gli ultimi N turni della conversazione e li invia al modello strutturato. zConversazione:rzUtente: zAgente: r CONVERSATIONr)rrr)thinking_budgetHARM_CATEGORY_HARASSMENTBLOCK_MEDIUM_AND_ABOVE)category thresholdHARM_CATEGORY_HATE_SPEECHHARM_CATEGORY_SEXUALLY_EXPLICITHARM_CATEGORY_DANGEROUS_CONTENTzapplication/jsonrHrK)r)rrequiredr)rrt)cached_contentthinking_configsafety_settingsresponse_mime_typeresponse_schema)rcontentsr)rvr rrrr!rrrrrrrARRAYSTRINGrmodelsgenerate_content GEMINI_modelrrs) rrDconversation_linesrrconversation_strrkcfgrs r/rGz'GeminiHandler._sync_generate_structured+sz //! =JD$v~"))HTF*;<"))HTF*;<  =  99%78 n  K d}tjgtj} |jj d{}tj |tj}tj ||g}t|}||z}|||zd}t|D]C}|||z|dz|z}|j} |jj| d{E77 #tj$rt$r} d} ~ wwxYww)a (abbiamo ricevuto) AUDIO FRONTEND -> BACKEND (dove siamo) -> (dove lo inviamo) GEMINI Preleva i buffer raw (48 kHz) da _raw_queue, li converte a 16 kHz, li spezza in chunk da 960 sample, e li mette in _chunk_queue. i)dtypeNrp)nparrayint16rr frombuffer concatenaterlrangetobytesrrrrr) r chunk_sizeresidualrarr16totaln_chunksrTr raw_chunkrOs r/ feed_audiozGeminiHandler.feed_audios$ Jxx"((3H OO//11 c:%'89E  J. J!6!89x;A!!j.AEZ3GHE % I++// ::: ;-10;%%     sBDAC8 C4 B"C8,C6-C86C88DDDDrrrrrrrrc bKtjjjd} d} dj| t |t |t |t || || |g} |d|} t j 4d{} | j| | ddid{}|jd k(r td dddd{y7M717 #1d{7swYyxYw#t$r}td t |Yd}~yd}~wwxYww) z Invia un record al Google Sheet via Apps Script: Data, Tokens_prompt, Tokens_response_audio, Tokens_response_text, Tokens_total, Domanda, Risposta z%Y-%m-%d %H:%M:%Scb|jddjddjS)N, r)rrrs)r@s r/rnz4GeminiHandler.send_to_google_sheet..s'!))C-55dC@FFHr.rz?sheet=Nz Content-Typez text/plain)urlcontentheadersu)✅ Riga inviata al Google Sheet (async).u+❌ Errore durante l'invio al Google Sheet:) rnowstrftimer strhttpx AsyncClientpost status_coderr)rrrrrrrrrnow_strclean csv_payloadrrresrOs r/r2z"GeminiHandler.send_to_google_sheets: ##'')223FGIhh    % & $ %   (O (O  gj\2 I((* G Gf"KK[><:X(??c)EF  G G G G G G G  I ?Q H H IsBD/DC)DC/9C+:C/ D#C-$D(D/)D+C/-D/D5C8 6D=DD/D D, D'"D/'D,,D/c8|jjyrk)rclear)rs r/ reset_historyzGeminiHandler.reset_historys !r.cfK|j|jryd|_|jD]}|jt j |jddid{|j js6 |j j|j js6|jjs7 |jj|jjs6yy7#t$rY_wxYw#t$rYywxYww)ze Cancella i singoli Task creati, poi svuota le due code (_raw_queue e _chunk_queue). NTreturn_exceptions) rrrcancelrgatherrempty get_nowaitrr)rrGs r/aclosezGeminiHandler.acloses  <<   A HHJ nndkkBTBBB//'') **,//'')##))+ !!,,.##))+ C     sfA(D1*D+D1 D$D1?D1D"4D1D1 DD1DD1" D.+D1-D..D1 client_idr WebSocketc"||j|<yrk)r)rrrs r/ register_wszGeminiHandler.register_wss#% i r.c<|jj|dyrk)rpop)rrs r/ unregister_wszGeminiHandler.unregister_wss )T*r.N)Aoedei])r)g?g@)r)r&r'r(rrClientrintrtuplebytesrrrrrCrr5rQrGrr2rrrrr-r.r/rrs&GM1 u||1 1 UX1 f '5e#4 '%N/dNr U%eCHo8NU%r :%uS#X1G:%C:%n+^IsISITWIhkIEHI`cIsvIDGI@"@&S&k&+s+r.r session_idcKt|}tj}|4d{ |j|j }|j|j }|j|j }|||g|_|,|t|<tjj|_ ||jd{|r|tvrtj| dddd{y777#|jd{7|r|tvrtj|wwwxYw7H#1d{7swYyxYwwrk)rr TaskGroupr1rCrrrrrrrrr)rrhandlertgt1t2t3s r/connect_geminirs5F#G    B00 0 .B 2 2 45B 5 5 78B "b\GN%.5 +$,$5$5$<$<$>!M.." " "jO;##J/)000$ #'.." " "jO;##J/rs:|        ** /1c?*+05 2 LD 4,#,$s),`#@s +s +p003:00HYr.