TiledViz
Loading...
Searching...
No Matches
BuildOldDatabase.py
1#!/bin/env python3
2
3import sys,os
4import argparse
5import re
6import json
7import datetime
8import traceback
9
10errors=sys.stderr
11
12from TVDb import tvdb
13
14POSTGRES_HOST="postgres"
15POSTGRES_PORT="6431"
16POSTGRES_USER="tiledviz"
17POSTGRES_DB="TiledViz"
18
19oldnodefile='TVDatabase/OLDcases/ANATOMIST/nodes.js'
20oldcasefile='TVDatabase/OLDcases/ANATOMIST/case_config_mandelbrot'
21projectNAME='testCASE'
22sessionNAME='testSESSION'
23datapath='/home/brain/BrainDatas3/archi-sulci'
24
25def parse_args(argv):
26 parser = argparse.ArgumentParser(
27 'Transform old nodes.js database to TiledViz database.')
28 parser.add_argument('-c', '--oldcasefile', default=oldcasefile,
29 help='Old case_config_machine path (default: ANATOMIST Mandelbrot).')
30 parser.add_argument('-f', '--oldnodefile', default=oldnodefile,
31 help='Old nodes.js path (default: ANATOMIST Mandelbrot).')
32 parser.add_argument('--host', default=POSTGRES_HOST,
33 help='Database host (default: '+POSTGRES_HOST+')')
34 parser.add_argument('--port', default=POSTGRES_PORT,
35 help='Port (default: '+POSTGRES_PORT+')')
36 parser.add_argument('-l', '--login', default=POSTGRES_USER,
37 help='Database login (default: '+POSTGRES_USER+')')
38 parser.add_argument('-n', '--databasename', default=POSTGRES_DB,
39 help='Database name (default: '+POSTGRES_DB+')')
40 parser.add_argument('-t', '--usertest', default='ddurandi',
41 help='User name for test (default: ddurandi)')
42 parser.add_argument('-p', '--projectNAME', default=projectNAME,
43 help='Project for test (default: '+projectNAME+')')
44 parser.add_argument('-s', '--sessionNAME', default=sessionNAME,
45 help='Session for test (default: '+sessionNAME+')')
46 parser.add_argument('-C', '--haveconnection', action='store_true',
47 help='Activate a connection object for the TileSet')
48 parser.add_argument('-H', '--connecthost', default='HPCmachine',
49 help='Create a connection object to this host')
50 parser.add_argument('-r', '--testsets', default='testTileSets',
51 help='Type for test (default: testTileSets)')
52 parser.add_argument('-m', '--typeoftiles', default='CONNECTION',
53 help='Type of tiles (default: CONNECTION, PICTURE, HTML)')
54 parser.add_argument('-d', '--datapath', default=datapath,
55 help='Path for data (default: '+datapath+')')
56 parser.add_argument('-o', '--passwordtest', default='5#f_@-m/1ArtOvx0',
57 help='Project for test (default: "5#f_@-m/1ArtOvx0")')
58
59 args = parser.parse_args(argv[1:])
60 return args
61
62def read_old_casefile(case_config_file,args):
63
64 try:
65 f=open(args.oldcasefile,'r')
66 except Exception:
67 print('unable to open old case file.')
68 errors.write(traceback.format_exc() + '\n')
69 errors.write("\n\n")
70 exit(1)
71
72 re_def=re.compile("^\\w*=[A-Za-z0-9/_-]*")
73 while 1:
74 line = f.readline()
75 if not line:
76 break
77 else:
78 matchdef=re_def.match(line)
79 #print("matchdef :",matchdef)
80 if (matchdef):
81 key=line.split('=')[0]
82 #print(line,matchdef.span()[1])
83 value=line[0:matchdef.span()[1]].split('=')[1]
84 #print ("line :",line," | key : ",key," | value : ",value)
85 if (key == "CASE_NAME" ):
86 args.projectNAME=value
87 elif (key == "DATA_PATH" ):
88 args.datapath=value
89 elif (re.match(r'.*(session|SESSION).*',key)):
90 args.sessionNAME=value
91 #else:
92 #print ("bad line :",line)
93 # #pour la source :
94 # # case=open("case_config_mandelbrot") #DATA_PATH=/home/brain/BrainDatas3/archi-sulci
95 # # anatomist_json=json.load("case_data_config.json") #=>boucle sur les subjects de data_list
96
97
98class OldJson:
99
100 def __init__(self,nodesfilename):
101 try:
102 f=open(nodesfilename,'r')
103 except Exception:
104 print('unable to open old node.js file.')
105 errors.write(traceback.format_exc() + '\n')
106 errors.write("\n\n")
107 exit(2)
108
109 #outnodejson="/tmp/tmpoutnode.json"
110 # try:
111 # g=open(outnodejson,'w')
112 # except Exception:
113 # print('unable to open '+outnodejson+' file for writing.')
114 # errors.write(traceback.format_exc() + '\n')
115 # errors.write("\n\n")
116 # exit(2)
117
118 line = f.readline()
119 line = f.readline()
120
121 #replace first line
122 if (re.match(r'.*\‍[',line)):
123 line__="[ "
124 else:
125 line__="" #re.sub("\s*var\s*text_\s*=\s*{","{",line).replace('nodes:','\"nodes\":')
126 myjson=""
127
128 lineno=1
129 #print(str(lineno)+" : "+line_)
130
131 line = f.readline()
132 line_= line
133
134 simplecote_def=re.compile("^\\s*\"\\w*\"\\s*:\\s*'.*'")
135 simple_def=re.compile("\'")
136 double_def=re.compile("\"")
137 pipe_def=re.compile("\|")
138
139 while 1:
140 line = f.readline()
141 if not line:
142 #suppress last line and last "};"
143 #myjson=myjson[:len(myjson)-2]
144 break
145 else:
146 #One must simple cote for comment in tags...
147 matchdef=simplecote_def.match(line__)
148 if (matchdef):
149 linesplit=line__[0:matchdef.span()[1]].split(':')
150 value=linesplit[1]
151 value=double_def.sub("|",value)
152 value=simple_def.sub("\"",value)
153 value=pipe_def.sub("'",value)
154 line__=linesplit[0]+" : "+value+",\n"
155 #g.write(line__)
156 #line without carriage return
157 myjson=myjson+line__[:-1]
158 line__=line_
159 line_=line
160 lineno = lineno + 1
161 #print(str(lineno)+" : "+line_)
162
163 #suppress last line and last "};"
164 if (re.match(r'\‍]',line__)):
165 line__="]"
166 myjson=myjson+line__
167 #g.write(line__)
168
169 #print(len(myjson),myjson[len(myjson)-100:])
170
171 # g.close
172 # try:
173 # g=open(outnodejson,'r')
174 # except Exception:
175 # print('unable to open '+outnodejson+' file for writing.')
176 # errors.write(traceback.format_exc() + '\n')
177 # errors.write("\n\n")
178 # exit(2)
179
180 #self.nodesjson=json.load(g)
181 self.nodesjson=json.loads(myjson)
182
183
184 def __getitem__(self,num):
185 return self.nodesjson[num]
186
187 def __len__(self):
188 return len(self.nodesjson)
189
190if __name__ == '__main__':
191 args = parse_args(sys.argv)
192 print('case file : ',args.oldcasefile)
193 print('node file : ',args.oldnodefile)
194 read_old_casefile(args.oldcasefile,args)
195 #print("args.projectNAME",args.projectNAME,"args.datapath",args.datapath,"args.sessionNAME=",args.sessionNAME)
196 print("")
197
198 Mynodes=OldJson(args.oldnodefile)
199 print("Test one (second) element : |", Mynodes[1]["comment"],"|")
200 # print("Test json.dumps of that element :",json.dumps(Mynodes[1], indent=4))
201 # print("Pass: ",os.getenv("passwordDB"))
202
203 metadata, conn, engine, pool, session = tvdb.SQLconnector(args)
204 for _t in metadata.tables:
205 print("Table: ", _t)
206
207 hashpass, salt=tvdb.passprotected(args.passwordtest)
208 id_users, uniqUser = tvdb.insert_table('users',{"name":args.usertest,"salt":salt,"password":hashpass})
209 # print("connections :",pool.checkedout())
210 tvdb.print_table("users")
211
212 id_projects, uniq = tvdb.insert_table('projects',{"name":args.projectNAME,"id_users":id_users,"description":"This is an old case "+args.projectNAME})
213 tvdb.print_table("projects")
214
215 config_default_file=open("TVWeb/app/static/js/config_default.json",'r')
216 json_configs=json.load(config_default_file)
217 config_default_file.close()
218 # json_configs_text=json.JSONEncoder().encode(json_configs)
219
220 id_sessions, uniqSession = tvdb.insert_table('sessions',{"name":args.sessionNAME,"id_projects":id_projects,"description":"This is an old case "+args.projectNAME+" session "+args.sessionNAME,"Number_of_active_users":1,"timeout":480,"config":json_configs})
221 tvdb.print_table("sessions")
222
223 if (uniqSession or uniqUser):
224 id_users_sessions, uniq = tvdb.insert_table('many_users_has_many_sessions',{"id_sessions":id_sessions,"id_users":id_users})
225
226 if (args.haveconnection):
227 id_connection, uniqConnection = tvdb.insert_table('connections',{"host_address":args.connecthost,
228 "auth_type":"ssh",
229 "container":"docker swarm",
230 "id_users":id_users,
231 "scheduler":"slurm",
232 "scheduler_file":"path_to_connection_f",
233 "config_files":{"site_config":"path_to_site_file","hostfile":"path_to_hostfile"}})
234 tvdb.print_table("connections")
235
236 id_tile_sets, uniqTileSet = tvdb.insert_table('tile_sets',{"name":args.testsets,"type_of_tiles":args.typeoftiles,"Dataset_path":args.datapath,"id_connections":id_connection})
237 else:
238 id_tile_sets, uniqTileSet = tvdb.insert_table('tile_sets',{"name":args.testsets,"type_of_tiles":args.typeoftiles,"Dataset_path":args.datapath})
239
240 tvdb.print_table("tile_sets")
241
242 if (uniqSession or uniqTileSet):
243 id_sessions_tilesets, uniq = tvdb.insert_table('many_sessions_has_many_tile_sets',{"id_sessions":id_sessions,"id_tile_sets":id_tile_sets})
244
245
246 now = datetime.datetime.now()
247 print ("Tiles creation : ",now)
248 try:
249 Mynodes[1]["tags"]
250 except :
251 for i in range(len(Mynodes)):
252 Mynodes[i]["tags"]=""
253 tiles=[]
254 for i in range(len(Mynodes)):
255 searchPort=re.search(r'port=\d+',Mynodes[i]["url"])
256 if (searchPort):
257 ConnectionPort=int(searchPort.group().replace('port=',''))
258 else:
259 ConnectionPort=0
260 tiles.append({"title" : Mynodes[i]["title"],
261 "comment" : Mynodes[i]["comment"],
262 "tags": Mynodes[i]["tags"],
263 "source" : {"connection" : ConnectionPort,
264 "url" : Mynodes[i]["url"]
265 },
266 "pos_px_x": -1,
267 "pos_px_y": -1,
268 'creation_date' : now
269 })
270
271 table_tiles = metadata.tables['tiles']
272 #myinsert=table_tiles.insert().returning(table_tiles.c.id)
273 #myinsert=table_tiles.insert(returning=[table_tiles.c.id])
274 myinsert=table_tiles.insert()
275 connexe=conn.execute(myinsert, tiles)
276 #connexe.inserted_primary_key() -> Can't call inserted_primary_key when returning() is used.
277 #connexe.first() -> sqlalchemy.exc.ResourceClosedError: This result object does not return rows. It has been closed automatically.
278 #connexe.fetchone() -> psycopg2.ProgrammingError: no results to fetch
279 #id_tiles=connexe.commit()
280 #id_tiles=connexe.fetchall()
281 #connexe.close()
282 #print("All tiles:",id_tiles)
283
284 select_st = table_tiles.select().where(table_tiles.c.creation_date==now)
285 res = conn.execute(select_st)
286 #print("All tiles:")
287 id_tiles=[]
288 many_tiles_has_many_tile_sets=[]
289 for _row in res:
290 #print(_row)
291 id_tiles.append(_row[0])
292 many_tiles_has_many_tile_sets.append({"id_tiles":_row[0], "id_tile_sets":id_tile_sets})
293 print("All tiles:",id_tiles)
294
295
296 table_tilesets_tiles = metadata.tables['many_tiles_has_many_tile_sets']
297 #myinsert = table_tilesets_tiles.insert().returning(table_tilesets_tiles.c.id_tiles).values(id_tiles=id_tiles,id_tile_sets=)
298 myinsert = table_tilesets_tiles.insert()
299 #connexe=conn.execute(myinsert)
300 connexe=conn.execute(myinsert, many_tiles_has_many_tile_sets)
301 #id_tilesets_tiles=connexe.fetchall()
302 #connexe.close()
303 #print("All tiles in this tile_set :",id_tilesets_tiles)
304
305 # #conn.close()
306 # #engine.disconnect()