queryps1inoldPS1.py

from astropy.io import ascii
from astropy.table import Table
import pandas as pd
import sys
import re
import numpy as np
import pylab
import json
import os
import requests
from astropy.table import Table, vstack

try: # Python 3.x
    from urllib.parse import quote as urlencode
    from urllib.request import urlretrieve
except ImportError:  # Python 2.x
    from urllib import pathname2url as urlencode
    from urllib import urlretrieve

try: # Python 3.x
    import http.client as httplib 
except ImportError:  # Python 2.x
    import httplib   


def ps1cone(ra,dec,radius,table="mean",release="dr1",format="csv",columns=None,
           baseurl="https://catalogs.mast.stsci.edu/api/v0.1/panstarrs", verbose=False,
           **kw):
    """Do a cone search of the PS1 catalog
    
    Parameters
    ----------
    ra (float): (degrees) J2000 Right Ascension
    dec (float): (degrees) J2000 Declination
    radius (float): (degrees) Search radius (<= 0.5 degrees)
    table (string): mean, stack, or detection
    release (string): dr1 or dr2
    format: csv, votable, json
    columns: list of column names to include (None means use defaults)
    baseurl: base URL for the request
    verbose: print info about request
    **kw: other parameters (e.g., 'nDetections.min':2)
    """
    
    data = kw.copy()
    data['ra'] = ra
    data['dec'] = dec
    data['radius'] = radius
    return ps1search(table=table,release=release,format=format,columns=columns,
                    baseurl=baseurl, verbose=verbose, **data)


def ps1search(table="mean",release="dr1",format="csv",columns=None,
           baseurl="https://catalogs.mast.stsci.edu/api/v0.1/panstarrs", verbose=False,
           **kw):
    """Do a general search of the PS1 catalog (possibly without ra/dec/radius)
    
    Parameters
    ----------
    table (string): mean, stack, or detection
    release (string): dr1 or dr2
    format: csv, votable, json
    columns: list of column names to include (None means use defaults)
    baseurl: base URL for the request
    verbose: print info about request
    **kw: other parameters (e.g., 'nDetections.min':2).  Note this is required!
    """
    
    data = kw.copy()
    if not data:
        raise ValueError("You must specify some parameters for search")
    checklegal(table,release)
    if format not in ("csv","votable","json"):
        raise ValueError("Bad value for format")
    url = "{baseurl}/{release}/{table}.{format}".format(**locals())
    if columns:
        # check that column values are legal
        # create a dictionary to speed this up
        dcols = {}
        for col in ps1metadata(table,release)['name']:
            dcols[col.lower()] = 1
        badcols = []
        for col in columns:
            if col.lower().strip() not in dcols:
                badcols.append(col)
        if badcols:
            raise ValueError('Some columns not found in table: {}'.format(', '.join(badcols)))
        # two different ways to specify a list of column values in the API
        # data['columns'] = columns
        data['columns'] = '[{}]'.format(','.join(columns))

    r = requests.get(url, params=data)

    if verbose:
        print(r.url)
    r.raise_for_status()
    if format == "json":
        return r.json()
    else:
        return r.text


def checklegal(table,release):
    """Checks if this combination of table and release is acceptable
    
    Raises a VelueError exception if there is problem
    """
    
    releaselist = ("dr1", "dr2")
    if release not in ("dr1","dr2"):
        raise ValueError("Bad value for release (must be one of {})".format(', '.join(releaselist)))
    if release=="dr1":
        tablelist = ("mean", "stack")
    else:
        tablelist = ("mean", "stack", "detection")
    if table not in tablelist:
        raise ValueError("Bad value for table (for {} must be one of {})".format(release, ", ".join(tablelist)))


def ps1metadata(table="mean",release="dr1",
           baseurl="https://catalogs.mast.stsci.edu/api/v0.1/panstarrs"):
    """Return metadata for the specified catalog and table
    
    Parameters
    ----------
    table (string): mean, stack, or detection
    release (string): dr1 or dr2
    baseurl: base URL for the request
    
    Returns an astropy table with columns name, type, description
    """
    
    checklegal(table,release)
    url = "{baseurl}/{release}/{table}/metadata".format(**locals())
    r = requests.get(url)
    r.raise_for_status()
    v = r.json()
    # convert to astropy table
    tab = Table(rows=[(x['name'],x['type'],x['description']) for x in v],
               names=('name','type','description'))
    return tab


def mastQuery(request):
    """Perform a MAST query.

    Parameters
    ----------
    request (dictionary): The MAST request json object

    Returns head,content where head is the response HTTP headers, and content is the returned data
    """
    
    server='mast.stsci.edu'

    # Grab Python Version 
    version = ".".join(map(str, sys.version_info[:3]))

    # Create Http Header Variables
    headers = {"Content-type": "application/x-www-form-urlencoded",
               "Accept": "text/plain",
               "User-agent":"python-requests/"+version}

    # Encoding the request as a json string
    requestString = json.dumps(request)
    requestString = urlencode(requestString)
    
    # opening the https connection
    conn = httplib.HTTPSConnection(server)

    # Making the query
    conn.request("POST", "/api/v0/invoke", "request="+requestString, headers)

    # Getting the response
    resp = conn.getresponse()
    head = resp.getheaders()
    content = resp.read().decode('utf-8')

    # Close the https connection
    conn.close()

    return head,content


def resolve(name):
    """Get the RA and Dec for an object using the MAST name resolver
    
    Parameters
    ----------
    name (str): Name of object

    Returns RA, Dec tuple with position"""

    resolverRequest = {'service':'Mast.Name.Lookup',
                       'params':{'input':name,
                                 'format':'json'
                                },
                      }
    headers,resolvedObjectString = mastQuery(resolverRequest)
    resolvedObject = json.loads(resolvedObjectString)
    # The resolver returns a variety of information about the resolved object, 
    # however for our purposes all we need are the RA and Dec
    try:
        objRa = resolvedObject['resolvedCoordinate'][0]['ra']
        objDec = resolvedObject['resolvedCoordinate'][0]['decl']
    except IndexError as e:
        raise ValueError("Unknown object '{}'".format(name))
    return (objRa, objDec)



#d = pd.read_csv('catalog/D5_release_ab_v3_2.csv',comment='#',delim_whitespace=True)
dg = pd.read_csv('rawstars/dougpv2_g_sample.txt',delim_whitespace=True,comment='#')
dg['RADEC']=dg['RA'].round(3).astype(str)+'_'+dg['DEC'].round(3).astype(str)
dr = pd.read_csv('rawstars/dougpv2_r_sample.txt',delim_whitespace=True,comment='#')
dr['RADEC']=dr['RA'].round(3).astype(str)+'_'+dr['DEC'].round(3).astype(str)
df = pd.merge(dg,dr,on='RADEC',suffixes=('_g','_r'))
di = pd.read_csv('rawstars/dougpv2_i_sample.txt',delim_whitespace=True,comment='#')
di['RADEC']=di['RA'].round(3).astype(str)+'_'+di['DEC'].round(3).astype(str)
df = pd.merge(df,di,on='RADEC',suffixes=('','_i'))
dz = pd.read_csv('rawstars/dougpv2_z_sample.txt',delim_whitespace=True,comment='#')
dz['RADEC']=dz['RA'].round(3).astype(str)+'_'+dz['DEC'].round(3).astype(str)
d = pd.merge(df,dz,on='RADEC',suffixes=('','_z'))

print(d)

pcols = d[['g','r','i','z']].columns
l = len(d)
tables = []
for i,row in d.iterrows():
    if row['g'] > 19: continue
    #if row['DEC_g'] < -29: continue
    #if i > 1000: continue
    print(i,'out of',l)

    ra = row['RA_g']
    dec = row['DEC_g']
    radius = 1*0.000277778
    constraints = {'nDetections.gt':5}

    # strip blanks and weed out blank and commented-out values
    columns = """objID,raMean,decMean,nDetections,ng,nr,ni,nz,
    gMeanPSFMag,gMeanPSFMagErr,rMeanPSFMag,rMeanPSFMagErr,iMeanPSFMag,iMeanPSFMagErr,zMeanPSFMag,zMeanPSFMagErr,gMeanApMag,gMeanApMagErr,rMeanApMag,rMeanApMagErr,iMeanApMag,iMeanApMagErr,zMeanApMag,zMeanApMagErr""".split(',')
    columns = [x.strip() for x in columns]
    columns = [x for x in columns if x and not x.startswith('#')]
    results = ps1cone(ra,dec,radius,release='dr2',columns=columns,verbose=True,**constraints)
    # print first few lines
    lines = results.split('\n')
    #print(len(lines),"rows in results -- first 5 rows:")
    #print('\n'.join(lines[:2]))

    try:
        tab = ascii.read(results)
    except:
        continue
    # improve the format
    for filter in 'griz':
        col = 'oldPS1_'+filter
        try:
            tab[col].format = ".4f"
            tab[col][tab[col] == -999.0] = np.nan
        except KeyError:
            print("{} not found".format(col))
        for pcol in pcols:
            tab['oldPS1_'+pcol] = row[pcol]
    print(tab)
    tables.append(tab)
bigtab = vstack(tables)
print(bigtab)
try:
    os.remove('newcatalog/PS1_in_oldPS1.fits')
except:
    pass
bigtab.write('newcatalog/PS1_in_oldPS1.fits')