︎
Alessio Grancini

︎

︎

︎ 

︎ Work

︎ Press 
︎︎︎Friendship AI



Tags:
︎ VR Game

Status :
︎Fundraising

Role :
Designer, Developer  


︎ Collaboration with Kerenza Harris


︎ Kaleidoscope
︎ 2020 07  


︎ Project Breakdown 


. “Friendship AI” is a pitched project on the fundraising platform Kaleidoscope

. the project aims to merge Virtual Reality, AI ChatBot dialougues and multiplayer functionalities

. the player mission is defeating geometric entities that are taking over on the natural immersive environment of the game

. B6-S acts as NPC and AI-Companion, growing his experience and with so his help for leveling up through the experience

. B6-S experience and dialougues skills grows thanks to words detection functionalities

. the detection of words unlocks prebuilt database defined through the IBM Watson Platform



︎

. Concept Trailer : Friendship AI



︎ IBM Watson : integration and potential
Watson set up and word detection

this projects uses the following Watson services

. SST - Speech To Text for communicating to the AI assistant 

. TTS - Text to Speech for interpreting the answer from the chat bot and “saying it loud”

. the two functions are bridged by a streamer that interprets the written question from the SST service and send it to the AI assistant. Successively the TTS repeats

STT ︎︎︎  Watson ML ︎︎︎ TTS 

. as the AI detects word in the streaming function in game, is possible exporting strings on a simple excel sheet and starting to create a second database wich will be fed to the watson website once again

. the Watson AI portal works through “Entities” and “Intents” which are both a way to channel keywords and key-sentences for the AI to be trained

. this implementation is very basic, but gives the idea how an Excel file can be started to structure itself following the pattern used on the IBM portal








︎︎︎ What’s next  
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
using Excel;
using System.Data;
using System.IO;
using UnityEngine;

public class ExcelReader : MonoBehaviour
{
    private void Start()
    {
        DataRowCollection dataRow = ReadExcel(Application.streamingAssetsPath + "/WordDetection.xlsx");
        //loop starts from 1 - excludes the header
        for (int i = 1; i < dataRow.Count; i++)
        {
            Debug.Log("Word id" + dataRow[i][0] + "--" + "Word" + dataRow[i][1]);
        }
    }
    // this goes by name sheet index id 
    private static DataRowCollection ReadExcel(string path, int sheet = 0)
    {
        FileStream stream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read);
        IExcelDataReader reader = readerFactory.CreateOpenXmlReader(stream);
        DataSet result = reader.AsDataSet();
        return result.Tables[sheet].Rows;
    }
    // this goes by name id
    private static DataRowCollection ReadExcel(string path, string name)
    {
        FileStream stream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read);
        IExcelDataReader reader = readerFactory.CreateOpenXmlReader(stream); //Read 2007 and later versions
        DataSet result = reader.AsDataSet();
        return result.Tables[name].Rows;
    }
}


 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
using Excel;
using System.Data;
using System.IO;
using UnityEngine;
using UnityEngine.UI;


public class ExcelWriter : MonoBehaviour
{

    public InputField detectText; // input field is string 

    private void Start()
      {
          // *it might be regulated asynchronously through the session with the AI bot.
          string _filePath = Application.streamingAssetsPath + "/excelDetect.xlsx";
          string _sheetName = "detect";
         // protection - simplifcation - might be different when set up a larger workflow 
         FileInfo _excelName = new FileInfo(_filePath);
          if (_excelName.Exists)
          {
             _excelName.Delete();
             _excelName = new FileInfo(_filePath);
          }
         // open file
         using (ExcelPackage package = new ExcelPackage(_excelName))
         {
             //Add column names
             worksheet.Cells[1, 1].Value = "Word ID";
             worksheet.Cells[1, 2].Value = "Word Detected";
             //Add a row of data
             worksheet.Cells[2, 1].Value = 1;
             worksheet.Cells[2, 2].Value = detectText.text;
             //Save
              package.Save();
        }
     }
}

︎

. VR Multiplayer : the “issue” 

. many third parties services provide a great multiplayer service as Photon or the Unity Network itself

. in this scenario the player prefab handler needed some modification as natively [at this time] the non-VR template would not correctly perform using two VR headset in the same virtual space


︎ Shared Code Sample:
VR Multiplayer Set up/Fix

* a similar script is also applied to the controllers.

* the second scripts provides a different color to every player 

* this script looks at the work of Daniel Hampikian



 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class CameraOrigin : Photon.MonoBehaviour {

    public Camera cam;
    public GameObject ContL;
    public GameObject ContR;
    //
    public GameObject sostL;
    public GameObject sostR;
    //
    public GameObject VRoriginOther;
    //
    public GameObject player;
    public GameObject pointers;

    void Awake () {
        if (photonView.isMine) return;
        cam.enabled = false;
        ContL.SetActive (false);
        ContR.SetActive (false);
        pointers.SetActive(false);
        ContL.name = "OtherPlayerL";
        ContR.name = "OtherPlayerR";
        VRoriginOther.name = "OtherPlayerVRorigin";
    }

    void Update () {
        if (!photonView.isMine) {

            Transform OtherPlayerL2child;
            Transform OtherPlayerR2child;
           
            OtherPlayerL2child = player.transform.Find ("OtherPlayerVRorigin/OtherPlayerL");
            OtherPlayerR2child = player.transform.Find ("OtherPlayerVRorigin/OtherPlayerR");

            sostL = GameObject.FindGameObjectWithTag ("contTempL");
            sostR = GameObject.FindGameObjectWithTag ("contTempR");

            sostL.transform.position = OtherPlayerL2child.transform.position;
            sostR.transform.position = OtherPlayerR2child.transform.position;

            sostL.transform.rotation = OtherPlayerL2child.transform.rotation;
            sostR.transform.rotation = OtherPlayerR2child.transform.rotation;
        }

    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class RedefineColors : MonoBehaviour
{
   public GameObject[] AllObjs ;
   public Color [] AllCol;
   public void Update()
    {
            AllObjs= GameObject.FindGameObjectsWithTag("Color");
            for(int i = 0; i < AllObjs.Length; i++)
            {
              AllObjs[i].GetComponent<Renderer>().material.SetColor ("_BaseColor", AllCol[i]);
            }      
    }
}
︎

. Previous Project Exploration

00.00 interactive environments with VFX graph 

04.48 multiplayer test