You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
56 lines
2.0 KiB
56 lines
2.0 KiB
#!/usr/bin/python2 |
|
#### ampcal-db.py |
|
#### Jay Blanchard 2017 - python script to take an ampcal.dat file and add each line to the database. |
|
|
|
|
|
import MySQLdb |
|
import time |
|
from datetime import datetime,timedelta |
|
from warnings import filterwarnings |
|
|
|
|
|
def insertDB(exp, date, source, telescope, freq, cor): |
|
|
|
#basic SQL insert. If the below starts to fail this can be used (HOWEVER DUPLICATE CHECKING WILL HAVE TO BE DONE!) |
|
#sql = """INSERT INTO ampcal(exp, obs_date, source, telescope, frequency, correction) VALUES ('%s', '%s', '%s', '%s', '%s', '%s');""" %(exp, date, source, telescope, freq, cor) |
|
|
|
#somewhat complicated SQL here to try to avoid duplicates. |
|
sql = """INSERT INTO ampcal(exp, obs_date, source, telescope, frequency, correction) SELECT * FROM(SELECT '%s', '%s', '%s', '%s', '%s', '%s') AS tmp |
|
WHERE NOT EXISTS (SELECT * FROM ampcal WHERE exp='%s' AND source='%s' AND telescope='%s') |
|
LIMIT 1;""" % (exp, date, source, telescope, freq, cor, exp, source, telescope) |
|
|
|
# print sql |
|
try: |
|
cursor.execute(sql) |
|
db.commit() |
|
except (MySQLdb.Error, MySQLdb.Warning) as e: |
|
print(e) |
|
db.rollback() |
|
|
|
#MAIN STARTS HERE |
|
#open db con |
|
db = MySQLdb.connect("db0.jive.eu","ampcal-w","klsge[09w4hj","ampcal") |
|
cursor = db.cursor() |
|
|
|
#ignore warnings about SQL above... |
|
filterwarnings('ignore', category = MySQLdb.Warning) |
|
|
|
print("Inserting ampcal.dat into database") |
|
fin = open('ampcal.dat','rt') |
|
for line in fin: |
|
col = line.split() |
|
year = int(float(col[1])) |
|
rem = float(col[1]) - year |
|
base = datetime(year, 1, 1) |
|
date = base + timedelta(seconds=(base.replace(year=base.year + 1) - base).total_seconds() * rem) |
|
unix_date = int((time.mktime(date.timetuple())+date.microsecond/1000000.0)) |
|
try: # neeed to do this in a try for bad rows (easy way to avoid unreadable rows |
|
if (float(col[5]) < 1.0): #we ignore any corrections > 1 as they are likely a bad source |
|
insertDB(col[0],date, col[2], col[3], col[4], col[5]) |
|
except: |
|
pass; |
|
|
|
|
|
fin.close() |
|
#end - close db and file |
|
db.close()
|
|
|