make kradar json file(nuscenes ver.)

kradar 데이터에서 nuscenes data format 형식으로 json 파일 생성.

sample.json 마지막 줄의 comma를 지워줘야 함.

 

1. sample.json

#sample.json

import os

import json

from collections import OrderedDict

file_data = OrderedDict()

with open('sample.json', 'a+', encoding="utf-8") as make_file:

    make_file.write('[')

    make_file.write('\n')

for j in range(1, 59):

    seq = os.path.join('seq_' + str(j))

    path_dir = os.path.join('./'+str(j))

    file_list = os.listdir(path_dir)

    #create token list for prev, next

    tkn_list = ['']

    for i in file_list:

        tkn = os.path.join(seq + '_'+i[:-4])

        tkn_list.append(tkn)

    tkn_list.append('')

    for idx, i in enumerate(file_list):

       

        file_dir = os.path.join(path_dir, i)

        f = open(file_dir)

       

        #timestamp

        ts = f.readline().split(',')[1][11:-1]

        ts = int(ts.split('.')[0])

       

        file_data["sample_token"] = tkn_list[idx+1]

        file_data["timestamp"] = ts

        file_data["prev"] = tkn_list[idx]

        file_data["next"] = tkn_list[idx+2]

        file_data["scene_token"] = seq

        #print(json.dumps(file_data, ensure_ascii=False, indent="\t"))

        with open('sample.json', 'a+', encoding="utf-8") as make_file:

            json.dump(file_data, make_file, ensure_ascii=False, indent="\t")

            make_file.write(',')

            make_file.write('\n')

with open('sample.json', 'a+', encoding="utf-8") as make_file:

    make_file.write(']')

 

2. sample_data.json

#sample_data.json

import os

import json

from collections import OrderedDict

file_data = OrderedDict()

with open('sample_data.json', 'a+', encoding="utf-8") as make_file:

    make_file.write('[')

    make_file.write('\n')

for j in range(1, 59):

    seq = os.path.join('seq_' + str(j))

    path_dir = os.path.join('./'+str(j))

    file_list = os.listdir(path_dir)

    #create sample_token list

    #create ego_token list for prev, next

    ego_tkn_list = ['']

    tkn_list = ['']

    for i in file_list:

        tkn = os.path.join(seq + '_'+i[:-4])

        ego_tkn = os.path.join(tkn + '_ego')

        tkn_list.append(tkn)

        ego_tkn_list.append(ego_tkn)

    tkn_list.append('')

    ego_tkn_list.append('')

    #create calibrated_sensor_token

    calib_tkn = os.path.join(seq + '_radar_front')

    for idx, i in enumerate(file_list):

       

        file_dir = os.path.join(path_dir, i)

        f = open(file_dir)

       

        #timestamp

        ts = f.readline().split(',')[1][11:-1]

        ts = int(ts.split('.')[0])

       

        #data for json file

        file_data["token"] = ego_tkn_list[idx+1]

        file_data["sample_token"] = tkn_list[idx+1]

        file_data["ego_pose_token"] = ego_tkn_list[idx+1]

        file_data["calibrated_sensor_token"] = calib_tkn

        file_data["timestamp"] = ts

        file_data["fileformat"] = "pcd"

        file_data["is_key_frame"] = ""

        file_data["height"] = 0

        file_data["width"] = 0

        file_data["file_name"] = ""

        file_data["prev"] = ego_tkn_list[idx]

        file_data["next"] = ego_tkn_list[idx+2]

        #print(json.dumps(file_data, ensure_ascii=False, indent="\t"))

        with open('sample_data.json', 'a+', encoding="utf-8") as make_file:

            json.dump(file_data, make_file, ensure_ascii=False, indent="\t")

            make_file.write(',')

            make_file.write('\n')

with open('sample_data.json', 'a+', encoding="utf-8") as make_file:

    make_file.write(']')

   

 

3. sample_annotation.json

이 경우에는 위와 다르게 데이터 정렬이 frame 순이 아니라 객체 순으로 되어있다.

따라서 객체를 기준으로 json파일을 만들고 나중에 합치는 방향으로 작성하였다.

 

또한, kradar에서 orientation이 theta값으로 되어 있으므로 이를 아래와 같은 quaternion rotation정보로 변환시킬 필요가 있다. 

"rotation":                <float> [4] -- Bounding box orientation as quaternion: w, x, y, z.

from math import radians, cos, sin

from scipy.spatial.transform import Rotation

def theta2rotation(theta):

   

    rotation = Rotation.from_euler('z', theta, degrees=True)

    quaternion = rotation.as_quat() # x, y, z, w

    nuscenes_quaternion = [quaternion[3], quaternion[0], quaternion[1], quaternion[2]] # w, x, y, z

    return nuscenes_quaternion

 

객체 기준의 json파일 만들기

#sample_annotation.json

import os

import json

from collections import OrderedDict

from math import radians, cos, sin

from scipy.spatial.transform import Rotation

def theta2rotation(theta):

   

    rotation = Rotation.from_euler('z', theta, degrees=True)

    quaternion = rotation.as_quat() # x, y, z, w

    nuscenes_quaternion = [quaternion[3], quaternion[0], quaternion[1], quaternion[2]] # w, x, y, z

    return nuscenes_quaternion

#main

file_data = OrderedDict()

for j in range(1, 59):

    obj_dir = os.path.join('./obj/' + str(j))

    os.mkdir(obj_dir)

    seq = os.path.join('seq_' + str(j))

    path_dir = os.path.join('./new/'+str(j))

    file_list = os.listdir(path_dir)

    dic = {}

    #dictionary for prev, next (object 기준)

    for idx, i in enumerate(file_list):

       

        file_dir = os.path.join(path_dir, i)

        f = open(file_dir)

       

        #첫 줄에서 sample_token 읽어옴.

        temp = f.readline()

        while True:

            temp = f.readline()

            if temp == '':

                break

            obj_id = temp.split(',')[2][1:]

            if obj_id in dic:

                dic[obj_id].append(temp.split(',')[11][25:])

            else:

                dic[obj_id] = ['']

                dic[obj_id].append(temp.split(',')[11][25:])

    for a in dic.keys():

        dic[a].append('')

    for idx, i in enumerate(file_list):

       

        file_dir = os.path.join(path_dir, i)

        f = open(file_dir)

       

        #첫 줄에서 sample_token 읽어옴.

        temp = f.readline()

        sample_token = temp.split(',')[3][1:]

        #객체 정보

        while True:

            temp = f.readline()

            #다 읽으면 밖으로 나감

            if temp == '':

                break

            obj_id = temp.split(',')[2][1:]

            x, y, z = float(temp.split(',')[4][1:]), float(temp.split(',')[5][1:]), float(temp.split(',')[6][1:])

            theta = float(temp.split(',')[7][1:])

            theta = theta2rotation(theta)

            l, w, h = float(temp.split(',')[8][1:])*2, float(temp.split(',')[9][1:])*2, float(temp.split(',')[10][1:])*2

            # for prev, next

            dic_list = dic[obj_id]

            dic_idx = dic_list.index(temp.split(',')[11][25:])

            #json 내용

            file_data['token'] = temp.split(',')[11][25:]

            file_data['sample_token'] = sample_token[13:]

            file_data['instance_token'] = temp.split(',')[12][16:-1]

            #file_data['class'] = temp.split(',')[3][1:] #for category

            file_data['visibility_token'] = '0' #null값

            file_data['attribute_tokens'] = '' #null값

            file_data['translation'] = [x, y, z]

            file_data['size'] = [l, w, h]

            file_data['rotation'] = theta

            file_data['prev'] = dic[obj_id][dic_idx-1]

            file_data['next'] = dic[obj_id][dic_idx+1]

            file_data['num_lidar_pts'] = 0 #null값

            file_data['num_radar_pts'] = 0 #null값

            #각 객체별로 json파일 생성

            file_name = os.path.join('./obj/'+ str(j) + '/' + obj_id + '.json')

            #이미 있는경우

            if os.path.exists(file_name):

                with open(file_name, 'a+', encoding="utf-8") as make_file:

                    make_file.write(',')

                    make_file.write('\n')

                    json.dump(file_data, make_file, ensure_ascii=False, indent="\t")

            #처음

            else:

                with open(file_name, 'a+', encoding="utf-8") as make_file:

                    make_file.write('[')

                    make_file.write('\n')

                    json.dump(file_data, make_file, ensure_ascii=False, indent="\t")

for j in range(1, 59):

    obj_dir = os.path.join('./obj/' + str(j))

    file_list = os.listdir(obj_dir)

   

    for file_name in file_list:

        file_dir = os.path.join(obj_dir, file_name)

        with open(file_dir, 'a+') as file:

            file.write(']')

json 파일을 load하기 위해서는 json파일이 완성되어 있어야 하기 때문에 [{...},{...},...] 이런 형태를 만들어 줄 필요가 있다.

 

 

마지막으로 각 객체에 대한 json파일에서 데이터를 읽어와 객체를 기준으로 순차적인 sample_annotation.json을 생성.

#combine to sample_annotation.json

import json

combined_data = []

for j in range(1, 59):

    obj_dir = os.path.join('./obj/' + str(j))

    file_list = os.listdir(obj_dir)

   

    for file_name in file_list:

        file_dir = os.path.join(obj_dir, file_name)

        with open(file_dir, 'r') as file:

            data = json.load(file)

            for i in range(len(data)):

                combined_data.append(data[i])

   

with open('sample_annotation.json', 'w') as file:

    json.dump(combined_data, file, indent='\t')

 

4. scene.json

#scene.json

import os

import json

from collections import OrderedDict

file_data = OrderedDict()

with open('scene.json', 'a+', encoding="utf-8") as make_file:

    make_file.write('[')

    make_file.write('\n')

for j in range(1, 59):

    seq = os.path.join('seq_' + str(j))

    path_dir = os.path.join('./new/'+str(j))

    file_list = os.listdir(path_dir)

    file_data['token'] = seq

    file_data['log_token'] = ''

    file_data['nbr_samples'] = len(file_list)

    file_data['first_sample_token'] = file_list[0][:-4]

    file_data['last_sample_token'] = file_list[-1][:-4]

    file_data['name'] = seq

    file_data['description'] = ''

    with open('scene.json', 'a+', encoding="utf-8") as make_file:

        json.dump(file_data, make_file, ensure_ascii=False, indent="\t")

        make_file.write(',')

        make_file.write('\n')

with open('scene.json', 'a+', encoding="utf-8") as make_file:

    make_file.write(']')

 

5. instance.json

#instance.json

import os

import json

from collections import OrderedDict

file_data = OrderedDict()

with open('instance.json', 'a+', encoding="utf-8") as make_file:

    make_file.write('[')

    make_file.write('\n')

for j in range(1, 59):

    obj_dir = os.path.join('./obj/' + str(j))

    file_list = os.listdir(obj_dir)

    for i in file_list:

        file_dir = os.path.join(obj_dir, i)

        f = json.load(open(file_dir))

        file_data['token'] = f[0]['instance_token']

        file_data['category_token'] = f[0]['class']

        file_data['nbr_annotations'] = len(f)

        file_data['first_annotation_token'] = f[0]['token']

        file_data['last_annotation_token'] = f[-1]['token']

        with open('instance.json', 'a+', encoding="utf-8") as make_file:

            json.dump(file_data, make_file, ensure_ascii=False, indent="\t")

            make_file.write(',')

            make_file.write('\n')

with open('instance.json', 'a+', encoding="utf-8") as make_file:

    make_file.write(']')      

   

 

6. category.json

#category.json

import os

import json

from collections import OrderedDict

file_data = OrderedDict()

with open('category.json', 'a+', encoding="utf-8") as make_file:

    make_file.write('[')

    make_file.write('\n')

category = ['Sedan', 'Bus or Truck', 'Pedestrian', 'Bicycle', 'Motorcycle'] #kradar category format

nusc_category = ['car', 'bus', 'pedestrian', 'bicycle', 'motorcycle'] #category_to_tracking_name format

for i in range(len(category)):

    file_data['token'] = category[i]

    file_data['name'] = nusc_category[i]

    file_data['description'] = ''

    with open('category.json', 'a+', encoding="utf-8") as make_file:

            json.dump(file_data, make_file, ensure_ascii=False, indent="\t")

            make_file.write(',')

            make_file.write('\n')

with open('category.json', 'a+', encoding="utf-8") as make_file:

    make_file.write(']')