<commit_before>"""
common.py

Shared functions for meerkat_frontend.
"""
from datetime import datetime, timedelta
from dateutil.parser import parse
from flask import abort, request
from meerkat_frontend import app
import authorise as auth
import requests
import logging
import json
import os

def api(url, api_key=False, params=None):
    """
    Returns JSON data from API request.
    Args:
        url (str): The Meerkat API url from which data is requested
        api_key (optional bool): Whethe or not we should include the api
        key. Defaults to False.
    Returns:
        dict: A python dictionary formed from the API reponse json string.
    """
    if(app.config['TESTING']):
        path = os.path.dirname(os.path.realpath(__file__))+"/apiData"+url
        with open(path+'.json') as data_file:
            return json.load(data_file)
    else:
        api_uri = ''.join([app.config['INTERNAL_API_ROOT'], url])
        app.logger.error(api_uri)
        try:
            if api_key:
                r = requests.get(
                    api_uri,
                    headers={'authorization': 'Bearer ' + auth.get_token()},
                    params=params
                )
            else:
                r = requests.get(
                    api_uri,
                    params=params
                )
            if r.status_code == 502:
                abort(500, "Can not access the api at " + api_uri)
        except requests.exceptions.RequestException as e:
            abort(500, e)
        try:
            output = r.json()
        except Exception as e:
            abort(500, r)
        return output

def hermes(url, method, data={}):
    """
    Makes a Hermes API request.

    Args:
       url (str): The Meerkat Hermes url for the desired function.
       method (str):  The desired HTML function: GET, POST or PUT.
       data (optional dict): The data to be sent to the url. Defaults
       to ```{}```.

    Returns:
       dict: a dictionary formed from the json data in the response.
    """

    # Add the API key and turn into JSON.
    data["api_key"] = app.config['HERMES_API_KEY']

    # Assemble the other request params.
    url = app.config['HERMES_ROOT'] + url
    headers = {'content-type': 'application/json'}

    logging.warning("Sending json: " + json.dumps(data) + "\nTo url: " + url)

    # Make the request and handle the response.
    try:
        r = requests.request(method, url, json=data, headers=headers)
        logging.warning(r)
    except requests.exceptions.RequestException:
        print("hei")
        abort(500, "request")
    return r.json()

def epi_week_to_date(epi_week, year=datetime.today().year):
    """Converts an epi_week to a datetime object.

       Args:
           epi_week (int): The epi week to convert.
           year (optional int): The year of the epi-week to convert. Defaults
           to current year.

       Returns:
           datetime: a datetime object for the end of the given epiweek."""
    api_request = "epi_week_start/{}/{}".format(year, epi_week)
    data = api(api_request)
    return parse(data["start_date"]) + timedelta(days=7)

def date_to_epi_week(day=datetime.today()):
    """Converts a datetime object to an epi_week.

       Args:
           day (optional datetime): The date for which to find the
                corressponding epi week. Defaults to today.

       Returns:
           int: the epi week number for the week that contains the given date.
    """
    api_request = "epi_week/{}".format(day.isoformat())
    data = api(api_request)
    return data["epi_week"]

def add_domain(path):
    """
    Add's the domain from the request to the begining of the specified path.
    Path shuld begin with a forward slash e.g. /index.html would become
    jordan.emro.info/index.html for the jordan site.

    Args:
        path (str): The path of the url that you want to prefix with
            the request's domain.
    Returns:
        string: The path prefixed with the request's domain.
    """
    url = path
    domain = '/'.join(request.url_root.split('/')[0:3])
    if path[0] == '/':
        url = domain + path
    return url
<commit_msg>Fixing internal api root in frontend.<commit_after>"""
common.py

Shared functions for meerkat_frontend.
"""
from datetime import datetime, timedelta
from dateutil.parser import parse
from flask import abort, request
from meerkat_frontend import app
import authorise as auth
import requests
import logging
import json
import os

def api(url, api_key=False, params=None):
    """
    Returns JSON data from API request.
    Args:
        url (str): The Meerkat API url from which data is requested
        api_key (optional bool): Whethe or not we should include the api
        key. Defaults to False.
    Returns:
        dict: A python dictionary formed from the API reponse json string.
    """
    if(app.config['TESTING']):
        path = os.path.dirname(os.path.realpath(__file__))+"/apiData"+url
        with open(path+'.json') as data_file:
            return json.load(data_file)
    else:
        api_uri = add_domain(''.join([app.config['INTERNAL_API_ROOT'], url]))
        app.logger.error(api_uri)
        try:
            if api_key:
                r = requests.get(
                    api_uri,
                    headers={'authorization': 'Bearer ' + auth.get_token()},
                    params=params
                )
            else:
                r = requests.get(
                    api_uri,
                    params=params
                )
            if r.status_code == 502:
                abort(500, "Can not access the api at " + api_uri)
        except requests.exceptions.RequestException as e:
            abort(500, e)
        try:
            output = r.json()
        except Exception as e:
            abort(500, r)
        return output

def hermes(url, method, data={}):
    """
    Makes a Hermes API request.

    Args:
       url (str): The Meerkat Hermes url for the desired function.
       method (str):  The desired HTML function: GET, POST or PUT.
       data (optional dict): The data to be sent to the url. Defaults
       to ```{}```.

    Returns:
       dict: a dictionary formed from the json data in the response.
    """

    # Add the API key and turn into JSON.
    data["api_key"] = app.config['HERMES_API_KEY']

    # Assemble the other request params.
    url = app.config['HERMES_ROOT'] + url
    headers = {'content-type': 'application/json'}

    logging.warning("Sending json: " + json.dumps(data) + "\nTo url: " + url)

    # Make the request and handle the response.
    try:
        r = requests.request(method, url, json=data, headers=headers)
        logging.warning(r)
    except requests.exceptions.RequestException:
        print("hei")
        abort(500, "request")
    return r.json()

def epi_week_to_date(epi_week, year=datetime.today().year):
    """Converts an epi_week to a datetime object.

       Args:
           epi_week (int): The epi week to convert.
           year (optional int): The year of the epi-week to convert. Defaults
           to current year.

       Returns:
           datetime: a datetime object for the end of the given epiweek."""
    api_request = "epi_week_start/{}/{}".format(year, epi_week)
    data = api(api_request)
    return parse(data["start_date"]) + timedelta(days=7)

def date_to_epi_week(day=datetime.today()):
    """Converts a datetime object to an epi_week.

       Args:
           day (optional datetime): The date for which to find the
                corressponding epi week. Defaults to today.

       Returns:
           int: the epi week number for the week that contains the given date.
    """
    api_request = "epi_week/{}".format(day.isoformat())
    data = api(api_request)
    return data["epi_week"]

def add_domain(path):
    """
    Add's the domain from the request to the begining of the specified path.
    Path shuld begin with a forward slash e.g. /index.html would become
    jordan.emro.info/index.html for the jordan site.

    Args:
        path (str): The path of the url that you want to prefix with
            the request's domain.
    Returns:
        string: The path prefixed with the request's domain.
    """
    url = path
    domain = '/'.join(request.url_root.split('/')[0:3])
    if path[0] == '/':
        url = domain + path
    return url
<|endoftext|>


<commit_before>import os
from .core import DataObject, CsvTable, GoogleSheets, FitsTable, NumpyBinary, FileObject
from ..spectra import SpectraData

__all__ = ['known_google_sheets', 'Database', 'SpectraData']

known_google_sheets = {
    'hosts': GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8', 1471095077),
    'sdss_remove': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 1379081675, header_start=1, include_names=['SDSS ID']),
    'sdss_recover': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 286645731, header_start=1, include_names=['SDSS ID']),
    'satellites_named': GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 1),
    'des_remove': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 124397139, include_names=['DES_OBJID']),
    'des_recover': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 797095192, include_names=['DES_OBJID']),
    'decals_remove': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 1574060112, include_names=['decals_objid']),
    'decals_recover': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 1549289503, include_names=['decals_objid']),
    'manual_targets_aat2018a': GoogleSheets('1Z8HISgp6ScJ0YZiFK5_TrGDZXY9t2OL3hkCSFJUiC6w', 0, include_names=['OBJID']),
    'manual_targets_aat2018b': GoogleSheets('1Z8HISgp6ScJ0YZiFK5_TrGDZXY9t2OL3hkCSFJUiC6w', 478933689, include_names=['OBJID']),
    'manual_targets_mmt2019a': GoogleSheets('1Z8HISgp6ScJ0YZiFK5_TrGDZXY9t2OL3hkCSFJUiC6w', 85066145, include_names=['OBJID']),
    'lowz_fields': GoogleSheets('1COd0BjZz0x_9O74Xi0ovVhNZgWeAWNIkVzrZ5UUZr9c', 1883640266),
}

class Database(object):
    """
    This class provide the interface between the filesystem and other parts of
    the SAGA package.

    Parameters
    ----------
    root_dir : str, optional
        path to the shared SAGA Dropbox root directory
        if you don't have access, set to None.

    Notes
    -----
    Use database[key].read() to access the file.
    Use database[key].path to overwrite the file path.
    For base catalogs, key should be in the form of ('base', host_id).

    Examples
    --------
    >>> import SAGA
    >>> saga_database = SAGA.Database('/path/to/SAGA/Dropbox')
    >>> saga_hosts = SAGA.HostCatalog(saga_database)
    >>> saga_objects = SAGA.ObjectCatalog(saga_database)

    If you don't have access to SAGA Dropbox, you can do:
    >>> saga_database = SAGA.Database()
    >>> saga_database['base', 32].local = 'base_catalogs/base_sql_nsa32.fits.gz'
    >>> saga_database['spectra_clean'].local = 'data/saga_spectra_clean.fits.gz'

    If you don't have internet, you can do things like
    >>> saga_database['hosts_named'].local = 'hosts_named.csv'
    >>> saga_database['hosts_no_flags'].local = 'hosts_no_flags.csv'

    If you have internet now, but want to download the file just in case:
    >>> saga_database['hosts_named'].download('hosts_named.csv')
    >>> saga_database['hosts_no_flags'].download('hosts_no_flags.csv')
    """

    def __init__(self, shared_dir=None, local_dir=None):

        self._shared_dir = shared_dir or os.curdir
        if not os.path.isdir(self._shared_dir):
            raise ValueError('cannot locate {}'.format(self._shared_dir))

        self._local_dir = local_dir or os.curdir
        if not os.path.isdir(self._local_dir):
            raise ValueError('cannot locate {}'.format(self._local_dir))

        self._tables = {
            'saga_spectra_May2017': DataObject(FitsTable(os.path.join(self._shared_dir, 'Products', 'saga_spectra_May2017.fits.gz'))),
            'saga_clean_specs': DataObject(FitsTable(os.path.join(self._local_dir, 'saga_clean_specs.fits.gz'))),
            'nsa_v1.0.1': DataObject(FitsTable('https://data.sdss.org/sas/dr14/sdss/atlas/v1/nsa_v1_0_1.fits'),
                                     FitsTable(os.path.join(self._local_dir, 'external_catalogs', 'nsa', 'nsa_v1_0_1.fits')),
                                     use_local_first=True),
            'nsa_v0.1.2': DataObject(FitsTable('http://sdss.physics.nyu.edu/mblanton/v0/nsa_v0_1_2.fits'),
                                     FitsTable(os.path.join(self._local_dir, 'external_catalogs', 'nsa', 'nsa_v0_1_2.fits')),
                                     use_local_first=True),
            'spectra_gama_dr2': DataObject(FitsTable('http://www.gama-survey.org/dr2/data/cat/SpecCat/v08/SpecObj.fits'),
                                           FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'GAMA', 'GAMA_SpecObj_dr2.fits')),
                                           use_local_first=True),
            'spectra_gama_dr3': DataObject(FitsTable('http://www.gama-survey.org/dr3/data/cat/SpecCat/v27/SpecObj.fits'),
                                           FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'GAMA', 'GAMA_SpecObj_dr3.fits')),
                                           use_local_first=True),
            'spectra_ozdes_dr1': DataObject(FitsTable('http://www.mso.anu.edu.au/ozdes/OzDES-DR1.fits'),
                                            FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'OzDES', 'OzDES-DR1.fits')),
                                            use_local_first=True),
            'spectra_2df': DataObject(FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', '2dF', '2dF_best.fit'))),
            'spectra_6df': DataObject(FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', '6dF', '6dF_DR3.fit'))),
            'spectra_lcrs': DataObject(FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'other', 'LCRS_shectman96.fits'))),
            'spectra_ukst': DataObject(FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'other', 'UKST_ratcliffe98.fits'))),
            'spectra_palomar': DataObject(GoogleSheets('12MM0KdnSb2RdieyUci6DSCnW-h7x0B-bLdxbTkAGhTI', 1276422948),
                                          CsvTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'other', 'SAGA_Palomar - Final.csv'))),
            'spectra_slackers': DataObject(FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'other', 'Slackers_unpub.fits'))),
            'spectra_2dflens': DataObject(
                FileObject('http://2dflens.swin.edu.au/2dflens_bestredshifts_goodz_withtypesandmags_final.dat.gz', format='ascii.fast_commented_header'),
                FileObject(os.path.join(self._shared_dir, 'Spectra', 'Final', '2dF', '2dflens_final.dat'), format='ascii.fast_commented_header'),
                use_local_first=True
            ),
            'spectra_halpha': DataObject(FileObject(os.path.join(self._shared_dir, 'Spectra', 'saga_halpha.dat'), format='ascii.fast_basic')),
        }

        self._tables['spectra_raw_all'] = DataObject(SpectraData(
            os.path.join(self._shared_dir, 'Spectra', 'Final'),
            {
                'gama': self._tables['spectra_gama_dr3'],
                '2df': self._tables['spectra_2df'],
                '2dflens': self._tables['spectra_2dflens'],
                '6df': self._tables['spectra_6df'],
                'ozdes': self._tables['spectra_ozdes_dr1'],
                'lcrs': self._tables['spectra_lcrs'],
                'ukst': self._tables['spectra_ukst'],
                'slackers': self._tables['spectra_slackers'],
                'palomar': self._tables['spectra_palomar'],
            },
            self._tables['spectra_halpha'],
        ))

        gmm_dir = os.path.join(self._shared_dir, 'AuxiliaryData', 'gmm')
        if os.path.isdir(gmm_dir):
            for fname in os.listdir(gmm_dir):
                if fname.startswith('gmm_parameters') and fname.endswith('.npz'):
                    self._tables[fname[:-4]] = DataObject(NumpyBinary(os.path.join(gmm_dir, fname)))

        for k, v in known_google_sheets.items():
            if k == 'hosts':
                self._tables[k] = DataObject(v, CsvTable(os.path.join(self._shared_dir, 'HostCatalogs', 'host_list.csv')), cache_in_memory=True)
            elif k == 'lowz_fields':
                self._tables[k] = DataObject(v, CsvTable(os.path.join(self._shared_dir, 'HostCatalogs', 'lowz_fields.csv')), cache_in_memory=True)
            else:
                self._tables[k] = DataObject(v, CsvTable(), cache_in_memory=True)

        self._file_path_pattern = {
            'base_v2':   os.path.join(self._local_dir, 'base_catalogs', 'base_v2_{}.fits.gz'),
            'base_v1':   os.path.join(self._local_dir, 'base_catalogs', 'base_v1_{}.fits.gz'),
            'base_v0p1': os.path.join(self._shared_dir, 'Paper1', 'base_catalogs', 'base_sql_{}.fits.gz'),
            'sdss_dr14': os.path.join(self._local_dir, 'external_catalogs', 'sdss_dr14', '{}.fits.gz'),
            'sdss_dr12': os.path.join(self._local_dir, 'external_catalogs', 'sdss_dr12', '{}.fits.gz'),
            'wise':      os.path.join(self._local_dir, 'external_catalogs', 'wise', '{}.fits.gz'),
            'des_dr1':   os.path.join(self._local_dir, 'external_catalogs', 'des_dr1', '{}_des_dr1.fits.gz'),
            'decals':    os.path.join(self._local_dir, 'external_catalogs', 'decals', '{}_decals.fits.gz'),
        }
        self._file_path_pattern['base'] = self._file_path_pattern['base_v2']
        self._file_path_pattern['sdss'] = self._file_path_pattern['sdss_dr14']
        self._file_path_pattern['des'] = self._file_path_pattern['des_dr1']

    def _set_file_path_pattern(self, key, value):
        self._file_path_pattern[key] = value
        keys_to_del = [k for k in self._tables if isinstance(k, tuple) and len(k) == 2 and k[0] == key]
        for k in keys_to_del:
            del self._tables[k]

    @property
    def base_file_path_pattern(self):
        return self._file_path_pattern['base']

    @base_file_path_pattern.setter
    def base_file_path_pattern(self, value):
        self._set_file_path_pattern('base', value)

    @property
    def sdss_file_path_pattern(self):
        return self._file_path_pattern['sdss']

    @sdss_file_path_pattern.setter
    def sdss_file_path_pattern(self, value):
        self._set_file_path_pattern('sdss', value)

    @property
    def wise_file_path_pattern(self):
        return self._file_path_pattern['wise']

    @wise_file_path_pattern.setter
    def wise_file_path_pattern(self, value):
        self._set_file_path_pattern('wise', value)

    @property
    def des_file_path_pattern(self):
        return self._file_path_pattern['des']

    @des_file_path_pattern.setter
    def des_file_path_pattern(self, value):
        self._set_file_path_pattern('des', value)

    @property
    def decals_file_path_pattern(self):
        return self._file_path_pattern['decals']

    @decals_file_path_pattern.setter
    def decals_file_path_pattern(self, value):
        self._set_file_path_pattern('decals', value)

    def __getitem__(self, key):
        if key in self._tables:
            return self._tables[key]

        if isinstance(key, tuple) and len(key) == 2 and key[0] in self._file_path_pattern:
            path = self._file_path_pattern[key[0]].format(key[1])
            self._tables[key] = DataObject(FitsTable(path))
            return self._tables[key]

        raise KeyError('cannot find {} in database'.format(key))

    def __setitem__(self, key, value):
        self._tables[key] = value

    def __delitem__(self, key):
        del self._tables[key]

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def keys(self):
        return self._tables.keys()

    def _ipython_key_completions_(self):
        return list(self.keys())
<commit_msg>added wigglez<commit_after>import os
from .core import DataObject, CsvTable, GoogleSheets, FitsTable, NumpyBinary, FileObject
from ..spectra import SpectraData

__all__ = ['known_google_sheets', 'Database', 'SpectraData']

known_google_sheets = {
    'hosts': GoogleSheets('1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8', 1471095077),
    'sdss_remove': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 1379081675, header_start=1, include_names=['SDSS ID']),
    'sdss_recover': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 286645731, header_start=1, include_names=['SDSS ID']),
    'satellites_named': GoogleSheets('1GJYuhqfKeuJr-IyyGF_NDLb_ezL6zBiX2aeZFHHPr_s', 1),
    'des_remove': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 124397139, include_names=['DES_OBJID']),
    'des_recover': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 797095192, include_names=['DES_OBJID']),
    'decals_remove': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 1574060112, include_names=['decals_objid']),
    'decals_recover': GoogleSheets('1Y3nO7VyU4jDiBPawCs8wJQt2s_PIAKRj-HSrmcWeQZo', 1549289503, include_names=['decals_objid']),
    'manual_targets_aat2018a': GoogleSheets('1Z8HISgp6ScJ0YZiFK5_TrGDZXY9t2OL3hkCSFJUiC6w', 0, include_names=['OBJID']),
    'manual_targets_aat2018b': GoogleSheets('1Z8HISgp6ScJ0YZiFK5_TrGDZXY9t2OL3hkCSFJUiC6w', 478933689, include_names=['OBJID']),
    'manual_targets_mmt2019a': GoogleSheets('1Z8HISgp6ScJ0YZiFK5_TrGDZXY9t2OL3hkCSFJUiC6w', 85066145, include_names=['OBJID']),
    'lowz_fields': GoogleSheets('1COd0BjZz0x_9O74Xi0ovVhNZgWeAWNIkVzrZ5UUZr9c', 1883640266),
}

class Database(object):
    """
    This class provide the interface between the filesystem and other parts of
    the SAGA package.

    Parameters
    ----------
    root_dir : str, optional
        path to the shared SAGA Dropbox root directory
        if you don't have access, set to None.

    Notes
    -----
    Use database[key].read() to access the file.
    Use database[key].path to overwrite the file path.
    For base catalogs, key should be in the form of ('base', host_id).

    Examples
    --------
    >>> import SAGA
    >>> saga_database = SAGA.Database('/path/to/SAGA/Dropbox')
    >>> saga_hosts = SAGA.HostCatalog(saga_database)
    >>> saga_objects = SAGA.ObjectCatalog(saga_database)

    If you don't have access to SAGA Dropbox, you can do:
    >>> saga_database = SAGA.Database()
    >>> saga_database['base', 32].local = 'base_catalogs/base_sql_nsa32.fits.gz'
    >>> saga_database['spectra_clean'].local = 'data/saga_spectra_clean.fits.gz'

    If you don't have internet, you can do things like
    >>> saga_database['hosts_named'].local = 'hosts_named.csv'
    >>> saga_database['hosts_no_flags'].local = 'hosts_no_flags.csv'

    If you have internet now, but want to download the file just in case:
    >>> saga_database['hosts_named'].download('hosts_named.csv')
    >>> saga_database['hosts_no_flags'].download('hosts_no_flags.csv')
    """

    def __init__(self, shared_dir=None, local_dir=None):

        self._shared_dir = shared_dir or os.curdir
        if not os.path.isdir(self._shared_dir):
            raise ValueError('cannot locate {}'.format(self._shared_dir))

        self._local_dir = local_dir or os.curdir
        if not os.path.isdir(self._local_dir):
            raise ValueError('cannot locate {}'.format(self._local_dir))

        self._tables = {
            'saga_spectra_May2017': DataObject(FitsTable(os.path.join(self._shared_dir, 'Products', 'saga_spectra_May2017.fits.gz'))),
            'saga_clean_specs': DataObject(FitsTable(os.path.join(self._local_dir, 'saga_clean_specs.fits.gz'))),
            'nsa_v1.0.1': DataObject(FitsTable('https://data.sdss.org/sas/dr14/sdss/atlas/v1/nsa_v1_0_1.fits'),
                                     FitsTable(os.path.join(self._local_dir, 'external_catalogs', 'nsa', 'nsa_v1_0_1.fits')),
                                     use_local_first=True),
            'nsa_v0.1.2': DataObject(FitsTable('http://sdss.physics.nyu.edu/mblanton/v0/nsa_v0_1_2.fits'),
                                     FitsTable(os.path.join(self._local_dir, 'external_catalogs', 'nsa', 'nsa_v0_1_2.fits')),
                                     use_local_first=True),
            'spectra_gama_dr2': DataObject(FitsTable('http://www.gama-survey.org/dr2/data/cat/SpecCat/v08/SpecObj.fits'),
                                           FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'GAMA', 'GAMA_SpecObj_dr2.fits')),
                                           use_local_first=True),
            'spectra_gama_dr3': DataObject(FitsTable('http://www.gama-survey.org/dr3/data/cat/SpecCat/v27/SpecObj.fits'),
                                           FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'GAMA', 'GAMA_SpecObj_dr3.fits')),
                                           use_local_first=True),
            'spectra_ozdes_dr1': DataObject(FitsTable('http://www.mso.anu.edu.au/ozdes/OzDES-DR1.fits'),
                                            FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'OzDES', 'OzDES-DR1.fits')),
                                            use_local_first=True),
            'spectra_2df': DataObject(FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', '2dF', '2dF_best.fit'))),
            'spectra_6df': DataObject(FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', '6dF', '6dF_DR3.fit'))),
            'spectra_wigglez': DataObject(FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'WiggleZ', 'Wigglez.fits'))),
            'spectra_lcrs': DataObject(FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'other', 'LCRS_shectman96.fits'))),
            'spectra_ukst': DataObject(FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'other', 'UKST_ratcliffe98.fits'))),
            'spectra_palomar': DataObject(GoogleSheets('12MM0KdnSb2RdieyUci6DSCnW-h7x0B-bLdxbTkAGhTI', 1276422948),
                                          CsvTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'other', 'SAGA_Palomar - Final.csv'))),
            'spectra_slackers': DataObject(FitsTable(os.path.join(self._shared_dir, 'Spectra', 'Final', 'other', 'Slackers_unpub.fits'))),
            'spectra_2dflens': DataObject(
                FileObject('http://2dflens.swin.edu.au/2dflens_bestredshifts_goodz_withtypesandmags_final.dat.gz', format='ascii.fast_commented_header'),
                FileObject(os.path.join(self._shared_dir, 'Spectra', 'Final', '2dF', '2dflens_final.dat'), format='ascii.fast_commented_header'),
                use_local_first=True
            ),
            'spectra_halpha': DataObject(FileObject(os.path.join(self._shared_dir, 'Spectra', 'saga_halpha.dat'), format='ascii.fast_basic')),
        }

        self._tables['spectra_raw_all'] = DataObject(SpectraData(
            os.path.join(self._shared_dir, 'Spectra', 'Final'),
            {
                'gama': self._tables['spectra_gama_dr3'],
                '2df': self._tables['spectra_2df'],
                '2dflens': self._tables['spectra_2dflens'],
                '6df': self._tables['spectra_6df'],
                'ozdes': self._tables['spectra_ozdes_dr1'],
                'lcrs': self._tables['spectra_lcrs'],
                'ukst': self._tables['spectra_ukst'],
                'slackers': self._tables['spectra_slackers'],
                'palomar': self._tables['spectra_palomar'],
            },
            self._tables['spectra_halpha'],
        ))

        gmm_dir = os.path.join(self._shared_dir, 'AuxiliaryData', 'gmm')
        if os.path.isdir(gmm_dir):
            for fname in os.listdir(gmm_dir):
                if fname.startswith('gmm_parameters') and fname.endswith('.npz'):
                    self._tables[fname[:-4]] = DataObject(NumpyBinary(os.path.join(gmm_dir, fname)))

        for k, v in known_google_sheets.items():
            if k == 'hosts':
                self._tables[k] = DataObject(v, CsvTable(os.path.join(self._shared_dir, 'HostCatalogs', 'host_list.csv')), cache_in_memory=True)
            elif k == 'lowz_fields':
                self._tables[k] = DataObject(v, CsvTable(os.path.join(self._shared_dir, 'HostCatalogs', 'lowz_fields.csv')), cache_in_memory=True)
            else:
                self._tables[k] = DataObject(v, CsvTable(), cache_in_memory=True)

        self._file_path_pattern = {
            'base_v2':   os.path.join(self._local_dir, 'base_catalogs', 'base_v2_{}.fits.gz'),
            'base_v1':   os.path.join(self._local_dir, 'base_catalogs', 'base_v1_{}.fits.gz'),
            'base_v0p1': os.path.join(self._shared_dir, 'Paper1', 'base_catalogs', 'base_sql_{}.fits.gz'),
            'sdss_dr14': os.path.join(self._local_dir, 'external_catalogs', 'sdss_dr14', '{}.fits.gz'),
            'sdss_dr12': os.path.join(self._local_dir, 'external_catalogs', 'sdss_dr12', '{}.fits.gz'),
            'wise':      os.path.join(self._local_dir, 'external_catalogs', 'wise', '{}.fits.gz'),
            'des_dr1':   os.path.join(self._local_dir, 'external_catalogs', 'des_dr1', '{}_des_dr1.fits.gz'),
            'decals':    os.path.join(self._local_dir, 'external_catalogs', 'decals', '{}_decals.fits.gz'),
        }
        self._file_path_pattern['base'] = self._file_path_pattern['base_v2']
        self._file_path_pattern['sdss'] = self._file_path_pattern['sdss_dr14']
        self._file_path_pattern['des'] = self._file_path_pattern['des_dr1']

    def _set_file_path_pattern(self, key, value):
        self._file_path_pattern[key] = value
        keys_to_del = [k for k in self._tables if isinstance(k, tuple) and len(k) == 2 and k[0] == key]
        for k in keys_to_del:
            del self._tables[k]

    @property
    def base_file_path_pattern(self):
        return self._file_path_pattern['base']

    @base_file_path_pattern.setter
    def base_file_path_pattern(self, value):
        self._set_file_path_pattern('base', value)

    @property
    def sdss_file_path_pattern(self):
        return self._file_path_pattern['sdss']

    @sdss_file_path_pattern.setter
    def sdss_file_path_pattern(self, value):
        self._set_file_path_pattern('sdss', value)

    @property
    def wise_file_path_pattern(self):
        return self._file_path_pattern['wise']

    @wise_file_path_pattern.setter
    def wise_file_path_pattern(self, value):
        self._set_file_path_pattern('wise', value)

    @property
    def des_file_path_pattern(self):
        return self._file_path_pattern['des']

    @des_file_path_pattern.setter
    def des_file_path_pattern(self, value):
        self._set_file_path_pattern('des', value)

    @property
    def decals_file_path_pattern(self):
        return self._file_path_pattern['decals']

    @decals_file_path_pattern.setter
    def decals_file_path_pattern(self, value):
        self._set_file_path_pattern('decals', value)

    def __getitem__(self, key):
        if key in self._tables:
            return self._tables[key]

        if isinstance(key, tuple) and len(key) == 2 and key[0] in self._file_path_pattern:
            path = self._file_path_pattern[key[0]].format(key[1])
            self._tables[key] = DataObject(FitsTable(path))
            return self._tables[key]

        raise KeyError('cannot find {} in database'.format(key))

    def __setitem__(self, key, value):
        self._tables[key] = value

    def __delitem__(self, key):
        del self._tables[key]

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def keys(self):
        return self._tables.keys()

    def _ipython_key_completions_(self):
        return list(self.keys())
<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function

import os
import sys
import stat
import errno
import logging
from tempfile import NamedTemporaryFile as ntf
from functools import wraps
import networkx as nx

from tldp.utils import execute, logtimings

logger = logging.getLogger(__name__)

preamble = '''#! /bin/bash
set -x
set -e
set -o pipefail
'''

postamble = '''
# -- end of file
'''

def depends(graph, *predecessors):
    '''decorator to be used for constructing build order graph'''
    def anon(f):
        for dep in predecessors:
            graph.add_edge(dep.__name__, f.__name__)

        @wraps(f)
        def method(self, *args, **kwargs):
            return f(self, *args, **kwargs)
        return method
    return anon

class SignatureChecker(object):

    @classmethod
    def signatureLocation(cls, f):
        f.seek(0)
        buf = f.read(1024).lower()
        for sig in cls.signatures:
            try:
                sindex = buf.index(sig.lower())
                logger.debug("YES FOUND signature %r in %s at %s; doctype %s.",
                             sig, f.name, sindex, cls)
                return sindex
            except ValueError:
                logger.debug("not found signature %r in %s for type %s",
                             sig, f.name, cls.__name__)
        return None

class BaseDoctype(object):

    def __repr__(self):
        return '<%s:%s>' % (self.__class__.__name__, self.source.stem,)

    def __init__(self, *args, **kwargs):
        self.source = kwargs.get('source', None)
        self.output = kwargs.get('output', None)
        self.config = kwargs.get('config', None)
        self.removals = list()
        assert self.source is not None
        assert self.output is not None
        assert self.config is not None

    def cleanup(self):
        if self.config.script:
            return
        stem = self.source.stem
        removals = getattr(self, 'removals', None)
        if removals:
            for fn in removals:
                logger.debug("%s cleaning up intermediate file %s", stem, fn)
                try:
                    os.unlink(fn)
                except OSError as e:
                    if e.errno is errno.ENOENT:
                        logger.error("%s missing file at cleanup %s", stem, fn)
                    else:
                        raise e

    def build_precheck(self):
        classname = self.__class__.__name__
        for tool, validator in self.required.items():
            thing = getattr(self.config, tool, None)
            logger.info("%s, tool = %s, thing = %s", classname, tool, thing)
            if thing is None:
                logger.error("%s missing required tool %s, skipping...",
                             classname, tool)
                return False
            assert validator(thing)
        return True

    def hook_build_success(self):
        self.cleanup()

    def hook_build_failure(self):
        self.cleanup()

    def shellscript(self, script, **kwargs):
        if self.config.build:
            return self.execute_shellscript(script, **kwargs)
        elif self.config.script:
            return self.dump_shellscript(script, **kwargs)
        else:
            etext = '%s in shellscript, neither --build nor --script'
            raise Exception(etext % (self.source.stem,))

    @logtimings(logger.debug)
    def dump_shellscript(self, script, preamble=preamble, postamble=postamble):
        source = self.source
        output = self.output
        config = self.config
        s = script.format(output=output, source=source, config=config)
        print('', file=sys.stdout)
        print(s, file=sys.stdout)
        return True

    @logtimings(logger.debug)
    def execute_shellscript(self, script, preamble=preamble, postamble=postamble):
        source = self.source
        output = self.output
        config = self.config

        logdir = output.logdir
        prefix = source.doctype.__name__ + '-'

        s = script.format(output=output, source=source, config=config)
        tf = ntf(dir=logdir, prefix=prefix, suffix='.sh', delete=False)
        if preamble:
            tf.write(preamble)
        tf.write(s)
        if postamble:
            tf.write(postamble)
        tf.close()

        mode = stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR
        os.chmod(tf.name, mode)

        cmd = [tf.name]
        result = execute(cmd, logdir=logdir)
        if result != 0:
            with open(tf.name) as f:
                for line in f:
                    logger.info("Script: %s", line.rstrip())
            return False
        return True

    @logtimings(logger.debug)
    def buildall(self):
        stem = self.source.stem
        order = nx.dag.topological_sort(self.graph)
        logger.debug("%s build order %r", self.source.stem, order)
        for dep in order:
            method = getattr(self, dep, None)
            assert method is not None
            classname = self.__class__.__name__
            logger.info("%s calling method %s.%s", stem, classname, dep)
            if not method():
                logger.error("%s reported method %s failure, skipping...",
                             stem, dep)
                return False
        return True

    @logtimings(logger.info)
    def generate(self):
        stem = self.source.stem
        classname = self.__class__.__name__

        # -- the output directory gets to prepare; must return True
        #
        # -- the processor gets to prepare; must return True
        #
        if not self.build_precheck():
            logger.warning("%s %s failed (%s), skipping to next build",
                           stem, 'build_precheck', classname)
            return False

        if not self.output.hook_prebuild():
            logger.warning("%s %s failed (%s), skipping to next build",
                           stem, 'hook_prebuild', classname)
            return False

        opwd = os.getcwd()
        os.chdir(self.output.dirname)

        # -- now, we can try to build everything; this is the BIG WORK!
        #
        result = self.buildall()

        if result:
            self.hook_build_success()  # -- processor
            self.output.hook_build_success()  # -- output document
        else:
            self.hook_build_failure()  # -- processor
            self.output.hook_build_failure()  # -- output document

        os.chdir(opwd)

        return result

#
# -- end of file
<commit_msg>lowering verbosity on reporting tool/data detection<commit_after>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function

import os
import sys
import stat
import errno
import logging
from tempfile import NamedTemporaryFile as ntf
from functools import wraps
import networkx as nx

from tldp.utils import execute, logtimings

logger = logging.getLogger(__name__)

preamble = '''#! /bin/bash
set -x
set -e
set -o pipefail
'''

postamble = '''
# -- end of file
'''

def depends(graph, *predecessors):
    '''decorator to be used for constructing build order graph'''
    def anon(f):
        for dep in predecessors:
            graph.add_edge(dep.__name__, f.__name__)

        @wraps(f)
        def method(self, *args, **kwargs):
            return f(self, *args, **kwargs)
        return method
    return anon

class SignatureChecker(object):

    @classmethod
    def signatureLocation(cls, f):
        f.seek(0)
        buf = f.read(1024).lower()
        for sig in cls.signatures:
            try:
                sindex = buf.index(sig.lower())
                logger.debug("YES FOUND signature %r in %s at %s; doctype %s.",
                             sig, f.name, sindex, cls)
                return sindex
            except ValueError:
                logger.debug("not found signature %r in %s for type %s",
                             sig, f.name, cls.__name__)
        return None

class BaseDoctype(object):

    def __repr__(self):
        return '<%s:%s>' % (self.__class__.__name__, self.source.stem,)

    def __init__(self, *args, **kwargs):
        self.source = kwargs.get('source', None)
        self.output = kwargs.get('output', None)
        self.config = kwargs.get('config', None)
        self.removals = list()
        assert self.source is not None
        assert self.output is not None
        assert self.config is not None

    def cleanup(self):
        if self.config.script:
            return
        stem = self.source.stem
        removals = getattr(self, 'removals', None)
        if removals:
            for fn in removals:
                logger.debug("%s cleaning up intermediate file %s", stem, fn)
                try:
                    os.unlink(fn)
                except OSError as e:
                    if e.errno is errno.ENOENT:
                        logger.error("%s missing file at cleanup %s", stem, fn)
                    else:
                        raise e

    def build_precheck(self):
        classname = self.__class__.__name__
        for tool, validator in self.required.items():
            thing = getattr(self.config, tool, None)
            logger.debug("%s, tool = %s, thing = %s", classname, tool, thing)
            if thing is None:
                logger.error("%s missing required tool %s, skipping...",
                             classname, tool)
                return False
            assert validator(thing)
        return True

    def hook_build_success(self):
        self.cleanup()

    def hook_build_failure(self):
        self.cleanup()

    def shellscript(self, script, **kwargs):
        if self.config.build:
            return self.execute_shellscript(script, **kwargs)
        elif self.config.script:
            return self.dump_shellscript(script, **kwargs)
        else:
            etext = '%s in shellscript, neither --build nor --script'
            raise Exception(etext % (self.source.stem,))

    @logtimings(logger.debug)
    def dump_shellscript(self, script, preamble=preamble, postamble=postamble):
        source = self.source
        output = self.output
        config = self.config
        s = script.format(output=output, source=source, config=config)
        print('', file=sys.stdout)
        print(s, file=sys.stdout)
        return True

    @logtimings(logger.debug)
    def execute_shellscript(self, script, preamble=preamble, postamble=postamble):
        source = self.source
        output = self.output
        config = self.config

        logdir = output.logdir
        prefix = source.doctype.__name__ + '-'

        s = script.format(output=output, source=source, config=config)
        tf = ntf(dir=logdir, prefix=prefix, suffix='.sh', delete=False)
        if preamble:
            tf.write(preamble)
        tf.write(s)
        if postamble:
            tf.write(postamble)
        tf.close()

        mode = stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR
        os.chmod(tf.name, mode)

        cmd = [tf.name]
        result = execute(cmd, logdir=logdir)
        if result != 0:
            with open(tf.name) as f:
                for line in f:
                    logger.info("Script: %s", line.rstrip())
            return False
        return True

    @logtimings(logger.debug)
    def buildall(self):
        stem = self.source.stem
        order = nx.dag.topological_sort(self.graph)
        logger.debug("%s build order %r", self.source.stem, order)
        for dep in order:
            method = getattr(self, dep, None)
            assert method is not None
            classname = self.__class__.__name__
            logger.info("%s calling method %s.%s", stem, classname, dep)
            if not method():
                logger.error("%s reported method %s failure, skipping...",
                             stem, dep)
                return False
        return True

    @logtimings(logger.info)
    def generate(self):
        stem = self.source.stem
        classname = self.__class__.__name__

        # -- the output directory gets to prepare; must return True
        #
        # -- the processor gets to prepare; must return True
        #
        if not self.build_precheck():
            logger.warning("%s %s failed (%s), skipping to next build",
                           stem, 'build_precheck', classname)
            return False

        if not self.output.hook_prebuild():
            logger.warning("%s %s failed (%s), skipping to next build",
                           stem, 'hook_prebuild', classname)
            return False

        opwd = os.getcwd()
        os.chdir(self.output.dirname)

        # -- now, we can try to build everything; this is the BIG WORK!
        #
        result = self.buildall()

        if result:
            self.hook_build_success()  # -- processor
            self.output.hook_build_success()  # -- output document
        else:
            self.hook_build_failure()  # -- processor
            self.output.hook_build_failure()  # -- output document

        os.chdir(opwd)

        return result

#
# -- end of file
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np

dsenames = pd.read_csv('crsp/dsenames.csv')

dsenames = dsenames[dsenames['NAMEENDT']>=20100101]

# SECSTAT == "R" indicates that the security is "Regular" (i.e. the security is past the "When-Issued" stage
# and the company is not going through bankruptcy proceedings)
dsenames = dsenames[dsenames['SECSTAT']=="R"]
# TRDSTAT == "A" indicates that the security is actively trading
dsenames = dsenames[dsenames['TRDSTAT']=="A"]

dsenames['SYMBOL_LEN'] = dsenames['TSYMBOL'].str.len()
dsenames['TICKER_LEN'] = dsenames['TICKER'].str.len()
# There are 239 cases where the length of tsymbol does not match up with the length of ticker
# For most of these cases, it is because tsymbol includes a suffix of some sort, while ticker does not

dsenames['LAST'] = dsenames['TSYMBOL'].str[-1]
# This grabs the last two elements of tsymbol
dsenames['LAST2'] = dsenames['TSYMBOL'].str[-2:]

bad_permnos = [14209, 15141, 91845]
# Here, we index out any securities whose records don't make sense
dsenames = dsenames[~dsenames['PERMNO'].isin(bad_permnos)]

# The 'S' suffix indicates the "Shares of Beneficial Interest, which do not trade like regular securities
# The 'V' and 'WI' suffixes indicate "When-Issued" shares, which have been authorized to trade, but have not actually begun trading
nasdaq = dsenames.PRIMEXCH == "Q"
beneficial = (dsenames['LAST'] == "S") & (dsenames['SYMBOL_LEN'] == 5)
whenissued_nasdaq = (dsenames['LAST'] == "V") & (dsenames['SYMBOL_LEN'] == 5) & nasdaq
whenissued_nonnasdaq = (dsenames['LAST2'] == "WI") & (dsenames['SYMBOL_LEN'] > 3) & ~nasdaq

# Here, we take out the BoWI shares from bot NASDAQ and non-NASDAQ securities
dsenames = dsenames[~(beneficial & whenissued_nasdaq & whenissued_nonnasdaq)]

# Flag = 0 is our base case (i.e. the ticker symbol has no suffix)
dsenames['FLAG'] = 0
# When the ticker has no suffix, the root is just the ticker symbol, and the suffix is an empty string
dsenames['SYM_ROOT'] = dsenames['TSYMBOL']
dsenames['SYM_SUFFIX'] = ""

# We're about to do major surgery, make a copy!
orig_dsenames = dsenames.copy()

# Now begins the subsetting based on cases of symbol suffixes

# Since we already have our nasdaq variable from earlier, we can create a few more
# booleans to help us conduct the subsetting in a vectorized way

# This first boolean vector will be True for a ticker symbol if its last character is the same as its share class
class_equal_last = dsenames.SHRCLS == dsenames.LAST

# This boolean will be True for all NASDAQ securities with a ticker symbol longer than 4 characters
## 4 is the maximum number of characters for a ticker symbol on the NASDAQ
nasdaq_long = nasdaq & (dsenames.SYMBOL_LEN > 4)

# The first flag will denote securities with ticker symbols that have share class suffixes
## e.g. a company on the NASDAQ that has Class A and Class B shares

flag1 = nasdaq_long & class_equal_last

# The second flag will denote two different special cases
## Suffixes Y and F denote shares in foreign companies
## Suffixes J and K denote voting and non-voting shares, respectively
flag2 = ~flag1 & nasdaq_long & dsenames.LAST.isin(["Y", "J", "F", "K"])

# The third flag will denote (NASDAQ) stocks that are currently going through a reverse stock split
## These securities usually keep this ticker symbol for about three weeks after the reverse stock split takes place
flag3 = ~flag1 & nasdaq_long & (dsenames.LAST == "D")

# The fourth flag will denote non-NASDAQ stocks that have share class suffixes
## For non-NASDAQ exchanges (e.g. the New York Stock Exchange),
## ticker symbols without suffixes are 3 characters or less
flag4 = ~nasdaq & (dsenames.SYMBOL_LEN > 3) & class_equal_last

# There is a fifth set of suffixed ticker symbols that do not fit into the above categories,
## but they do have a unifying manual adjustment
## We denote this set as funny

funny_permnos = [85254,29938,29946,93093,92118,83275,82924,82932,77158,46950,90655]

funny = dsenames.PERMNO.isin(funny_permnos) & (dsenames.SYMBOL_LEN - dsenames.TICKER_LEN == 1) & \
            dsenames.LAST.isin(["A","B","C","S"])

# Here, we assign the flags to each special case, as described above

dsenames.loc[flag4, "FLAG"] = 4
dsenames.loc[flag3, "FLAG"] = 3
dsenames.loc[flag2, "FLAG"] = 2
dsenames.loc[flag1, "FLAG"] = 1

# Here, we group together the symboled suffixes to make the final root-suffix separation cleaner
## sym5_with_suffix is the set of special cases with more than 4 characters in the symbol
sym5_with_suffix = flag1 | flag2 | flag3
symbol_with_suffix = flag4 | funny | sym5_with_suffix

# Finally, the big enchilada, the separation of each ticker symbol into its root and its symbol
## Since we are only dealing with suffixes of length 1, the root will consist of all but the last character,
## and the root will be the ticker symbol's last character

dsenames.loc[symbol_with_suffix, "SYM_ROOT"] = dsenames.loc[symbol_with_suffix, "TSYMBOL"].str[0:-1]
dsenames.loc[symbol_with_suffix, "SYM_SUFFIX"] = dsenames.loc[symbol_with_suffix, "TSYMBOL"].str[-1]

# There were a few wonky observations, so we do some additional manual adjustments

dsenames.loc[dsenames.PERMNO == 14461, "SYM_ROOT"] = dsenames.loc[dsenames.PERMNO == 14461, "TSYMBOL"].str[0:-1]
dsenames.loc[dsenames.PERMNO == 14461, "SYM_SUFFIX"] = dsenames.loc[dsenames.PERMNO == 14461, "TSYMBOL"].str[-1]
dsenames.loc[dsenames.PERMNO == 13914, "SYM_ROOT"] = dsenames.loc[dsenames.PERMNO == 13914, "TSYMBOL"]
dsenames.loc[dsenames.PERMNO == 13914, "SYM_SUFFIX"] = ""
dsenames.loc[dsenames.PERMNO == 92895, "SYM_ROOT"] = "SAPX"
dsenames.loc[dsenames.PERMNO == 92895, "SYM_SUFFIX"] = ""

# Finally, we want to ensure that, when the same information is recorded, the date range listed for the record reflects
# the entire range over which the security was actively trading
## For instance, if a security stopped trading for a six month period, it has two entries in this file
## We want both of those entries to include beginning date for the security's trading before the six month break
## and the end date for the security's trading after the six month break

# To do this, we first want to reset the index in the dsenames dataframe
dsenames = dsenames.reset_index(drop = True)

# When we say that we want to adjust the dates 'when the same information is recorded,' we make that adjustment based
# on the following seven variables in the data frame: Permno, the two components of the ticker symbol, the name of
#the company the CUSIP number (current and historical), and the primary exchange on which the security trades

# We first create a new data frame sorted on these 7 columns, which only includes said 7 columns

levels_sort = ["PERMNO", 'SYM_ROOT', 'SYM_SUFFIX', 'COMNAM', 'CUSIP', 'NCUSIP', 'PRIMEXCH']
dsenames_sort = dsenames.sort_values(by = levels_sort).loc[:, levels_sort]
dsenames = dsenames.sort_values(by = levels_sort)

# We create two new variables, begdate and enddate, to capture the full range of dates for which each security trades
## The default case, when a security only matches with itself based on the 7 sort levels, is that the beginning date
## is the same as the beginning effective name date, and the end date is the same as the end effective name date

dsenames['BEGDATE'] = dsenames.NAMEDT
dsenames['ENDDATE'] = dsenames.NAMEENDT

# We create a new dataframe that only includes the sort variables
dsenames_sort_squish = dsenames_sort.loc[:, levels_sort]

# Here, we create two copies of the dataframe:
## One without the first record, and one without the last

dsenames_nofirst = dsenames_sort_squish.iloc[1:].reset_index(drop = True)
dsenames_nolast = dsenames_sort_squish.iloc[:-1].reset_index(drop = True)

# We then create a boolean matrix based on whether the entries of each matrix match
compare_matrix = (dsenames_nofirst == dsenames_nolast)

# If the i-th record matches the next record for all 7 variables, then the i-th row of the compare matrix will
# be all true. We extract the index for subsetting purposes
same_as_below = compare_matrix.all(axis = 1)
same_as_below_index = same_as_below.index[same_as_below]

# In order to collapse the end dates, we will also need an index to indicate if a record is the same as the one above
## This is simply caputured by adding 1 to the first index we found
same_as_above_index = same_as_below_index + 1

# Finally, we loop through the first Int64Index we found to bring the earliest BEGDATE for a record down
# to all of its matches
## Doing this matching iteratively mitigates the issue of a particular security having more than 2 records match
## based on the 7 variables
for i in same_as_above_index:
    dsenames['BEGDATE'].iat[i] = dsenames['BEGDATE'].iat[i-1]

# Similar logic is used to bring the latest ENDDATE up - we just do it backwards
for i in same_as_below_index[::-1]:
    dsenames['ENDDATE'].iat[i] = dsenames['ENDDATE'].iat[i+1]

# Finally, we output a final dataframe that includes only the columns we sorted on and our new date variables
# Since the same information is recorded for these files now, we drop the duplicates
final_columns = levels_sort + ['BEGDATE', 'ENDDATE']
clean_dsenames = dsenames.drop_duplicates(subset = final_columns).loc[:, final_columns]<commit_msg>Cleanup of clean_dsenames.py<commit_after>import pandas as pd

dsenames = pd.read_csv('crsp/dsenames.csv')

dsenames = dsenames[dsenames['NAMEENDT'] >= 20100101]

# SECSTAT == "R" indicates that the security is "Regular" (i.e. the security is
# past the "When-Issued" stage and the company is not going through bankruptcy
# proceedings)
dsenames = dsenames[dsenames['SECSTAT'] == "R"]
# TRDSTAT == "A" indicates that the security is actively trading
dsenames = dsenames[dsenames['TRDSTAT'] == "A"]

dsenames['SYMBOL_LEN'] = dsenames['TSYMBOL'].str.len()
dsenames['TICKER_LEN'] = dsenames['TICKER'].str.len()
# There are 239 cases where the length of tsymbol does not match up with the
# length of ticker For most of these cases, it is because tsymbol includes a
# suffix of some sort, while ticker does not

dsenames['LAST'] = dsenames['TSYMBOL'].str[-1]
# This grabs the last two elements of tsymbol
dsenames['LAST2'] = dsenames['TSYMBOL'].str[-2:]

bad_permnos = [14209, 15141, 91845]
# Here, we index out any securities whose records don't make sense
dsenames = dsenames[~dsenames['PERMNO'].isin(bad_permnos)]

# The 'S' suffix indicates the "Shares of Beneficial Interest, which do not
# trade like regular securities The 'V' and 'WI' suffixes indicate
# "When-Issued" shares, which have been authorized to trade, but have not
# actually begun trading
nasdaq = dsenames.PRIMEXCH == "Q"
beneficial = (dsenames['LAST'] == "S") & (dsenames['SYMBOL_LEN'] == 5)
whenissued_nasdaq = ((dsenames['LAST'] == "V") &
                     (dsenames['SYMBOL_LEN'] == 5) & nasdaq)
whenissued_nonnasdaq = ((dsenames['LAST2'] == "WI") &
                        (dsenames['SYMBOL_LEN'] > 3) & ~nasdaq)

# Here, we take out the BoWI shares from bot NASDAQ and non-NASDAQ securities
dsenames = dsenames[~(beneficial & whenissued_nasdaq & whenissued_nonnasdaq)]

# Flag = 0 is our base case (i.e. the ticker symbol has no suffix)
dsenames['FLAG'] = 0
# When the ticker has no suffix, the root is just the ticker symbol, and the
# suffix is an empty string
dsenames['SYM_ROOT'] = dsenames['TSYMBOL']
dsenames['SYM_SUFFIX'] = ""

# We're about to do major surgery, make a copy!
orig_dsenames = dsenames.copy()

# Now begins the subsetting based on cases of symbol suffixes

# Since we already have our nasdaq variable from earlier, we can create a few
# more booleans to help us conduct the subsetting in a vectorized way

# This first boolean vector will be True for a ticker symbol if its last
# character is the same as its share class
class_equal_last = dsenames.SHRCLS == dsenames.LAST

# This boolean will be True for all NASDAQ securities with a ticker symbol
# longer than 4 characters. 4 is the maximum number of characters for a ticker
# symbol on the NASDAQ.
nasdaq_long = nasdaq & (dsenames.SYMBOL_LEN > 4)

# The first flag will denote securities with ticker symbols that have share
# class suffixes, e.g. a company on the NASDAQ that has Class A and Class B
# shares

flag1 = nasdaq_long & class_equal_last

# The second flag will denote two different special cases:
# - Suffixes Y and F denote shares in foreign companies
# - Suffixes J and K denote voting and non-voting shares, respectively
flag2 = ~flag1 & nasdaq_long & dsenames.LAST.isin(["Y", "J", "F", "K"])

# The third flag will denote (NASDAQ) stocks that are currently going through a
# reverse stock split. These securities usually keep this ticker symbol for
# about three weeks after the reverse stock split takes place.
flag3 = ~flag1 & nasdaq_long & (dsenames.LAST == "D")

# The fourth flag will denote non-NASDAQ stocks that have share class suffixes.
# For non-NASDAQ exchanges (e.g. the New York Stock Exchange), ticker symbols
# without suffixes are 3 characters or less.
flag4 = ~nasdaq & (dsenames.SYMBOL_LEN > 3) & class_equal_last

# There is a fifth set of suffixed ticker symbols that do not fit into the
# above categories, but they do have a unifying manual adjustment. We denote
# this set as "funny" (not "funny" ha ha).

funny_permnos = [85254, 29938, 29946, 93093, 92118, 83275, 82924, 82932, 77158,
                 46950, 90655]

funny = (dsenames.PERMNO.isin(funny_permnos) &
         (dsenames.SYMBOL_LEN - dsenames.TICKER_LEN == 1) &
         dsenames.LAST.isin(["A", "B", "C", "S"])
         )

# Here, we assign the flags to each special case, as described above

dsenames.loc[flag4, "FLAG"] = 4
dsenames.loc[flag3, "FLAG"] = 3
dsenames.loc[flag2, "FLAG"] = 2
dsenames.loc[flag1, "FLAG"] = 1

# Here, we group together the symboled suffixes to make the final root-suffix
# separation cleaner. `sym5_with_suffix` is the set of special cases with more
# than 4 characters in the symbol
sym5_with_suffix = flag1 | flag2 | flag3
symbol_with_suffix = flag4 | funny | sym5_with_suffix

# Finally, the big enchilada, the separation of each ticker symbol into its
# root and its symbol. Since we are only dealing with suffixes of length 1, the
# root will consist of all but the last character, and the root will be the
# ticker symbol's last character

dsenames.loc[symbol_with_suffix, "SYM_ROOT"] = \
    dsenames.loc[symbol_with_suffix, "TSYMBOL"].str[0:-1]
dsenames.loc[symbol_with_suffix, "SYM_SUFFIX"] = \
    dsenames.loc[symbol_with_suffix, "TSYMBOL"].str[-1]

# There were a few wonky observations, so we do some additional manual
# adjustments

dsenames.loc[dsenames.PERMNO == 14461, "SYM_ROOT"] = \
    dsenames.loc[dsenames.PERMNO == 14461, "TSYMBOL"].str[0:-1]
dsenames.loc[dsenames.PERMNO == 14461, "SYM_SUFFIX"] = \
    dsenames.loc[dsenames.PERMNO == 14461, "TSYMBOL"].str[-1]
dsenames.loc[dsenames.PERMNO == 13914, "SYM_ROOT"] = \
    dsenames.loc[dsenames.PERMNO == 13914, "TSYMBOL"]
dsenames.loc[dsenames.PERMNO == 13914, "SYM_SUFFIX"] = ""
dsenames.loc[dsenames.PERMNO == 92895, "SYM_ROOT"] = "SAPX"
dsenames.loc[dsenames.PERMNO == 92895, "SYM_SUFFIX"] = ""

# Finally, we want to ensure that, when the same information is recorded, the
# date range listed for the record reflects the entire range over which the
# security was actively trading.

# For instance, if a security stopped trading for a six month period, it has
# two entries in this file. We want both of those entries to include beginning
# date for the security's trading before the six month break and the end date
# for the security's trading after the six month break.

# To do this, we first want to reset the index in the dsenames dataframe
dsenames = dsenames.reset_index(drop=True)

# When we say that we want to adjust the dates 'when the same information is
# recorded,' we make that adjustment based on the following seven variables in
# the data frame:
# - Permno, the two components of the ticker symbol, the name of the company
# the CUSIP number (current and historical), and
# - the primary exchange on which the security trades

# We first create a new data frame sorted on these 7 columns, which only
# includes said 7 columns

levels_sort = ["PERMNO", 'SYM_ROOT', 'SYM_SUFFIX', 'COMNAM', 'CUSIP', 'NCUSIP',
               'PRIMEXCH']
dsenames_sort = dsenames.sort_values(by=levels_sort).loc[:, levels_sort]
dsenames = dsenames.sort_values(by=levels_sort)

# We create two new variables, begdate and enddate, to capture the full range
# of dates for which each security trades. The default case, when a security
# only matches with itself based on the 7 sort levels, is that the beginning
# date is the same as the beginning effective name date, and the end date is
# the same as the end effective name date.

dsenames['BEGDATE'] = dsenames.NAMEDT
dsenames['ENDDATE'] = dsenames.NAMEENDT

# We create a new dataframe that only includes the sort variables
dsenames_sort_squish = dsenames_sort.loc[:, levels_sort]

# Here, we create two copies of the dataframe:
# 1. One without the first record, and
# 2. one without the last

dsenames_nofirst = dsenames_sort_squish.iloc[1:].reset_index(drop=True)
dsenames_nolast = dsenames_sort_squish.iloc[:-1].reset_index(drop=True)

# We then create a boolean matrix based on whether the entries of each matrix
# match
compare_matrix = (dsenames_nofirst == dsenames_nolast)

# If the i-th record matches the next record for all 7 variables, then the i-th
# row of the compare matrix will be all true. We extract the index for
# subsetting purposes
same_as_below = compare_matrix.all(axis=1)
same_as_below_index = same_as_below.index[same_as_below]

# In order to collapse the end dates, we will also need an index to indicate if
# a record is the same as the one above.  This is simply caputured by adding 1
# to the first index we found
same_as_above_index = same_as_below_index + 1

# Finally, we loop through the first Int64Index we found to bring the earliest
# `BEGDATE` for a record down to all of its matches.  Doing this matching
# iteratively mitigates the issue of a particular security having more than 2
# records match based on the 7 variables.
for i in same_as_above_index:
    dsenames['BEGDATE'].iat[i] = dsenames['BEGDATE'].iat[i-1]

# Similar logic is used to bring the latest ENDDATE up - we just do it
# backwards
for i in same_as_below_index[::-1]:
    dsenames['ENDDATE'].iat[i] = dsenames['ENDDATE'].iat[i+1]

# Finally, we output a final dataframe that includes only the columns we sorted
# on and our new date variables.  Since the same information is recorded for
# these files now, we drop the duplicates
final_columns = levels_sort + ['BEGDATE', 'ENDDATE']
clean_dsenames = \
    dsenames.drop_duplicates(subset=final_columns).loc[:, final_columns]
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import random
import subprocess
import wikipedia
from twython import Twython

ApiKey = ''
ApiSecret = ''
AccessToken = ''
AccessTokenSecret = ''
cred_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                         '.auth')
twitter_allowed_char = 260

def get_api_token():
    ''' Obtain Twitter app's API token from file .auth

    Returns list
    '''
    f = open(cred_file, 'rb')
    c = f.read()
    t = c.splitlines()
    return t[0:4]

def get_today_str():
    ''' Obtain current date in 'Month_Date' format, e.g., March 3

    Returns str
    '''
    d = subprocess.check_output(["date", "+%B_%d"])
    d_str = d.strip()
    return d_str.decode('utf-8')

def get_events_list(date_str):
    ''' Obtain the list of events for date_str

    Returns list
    '''
    page = wikipedia.WikipediaPage(title=str(date_str))
    # Use some heuristics to take into account of recent wikipedia page format
    # changes
    maybe_events = [page.section(page.sections[i]) for i in
        range(page.sections.index('Events'), page.sections.index('Births'))]
    # Drop empty string
    events = '\n'.join([e.strip() for e in maybe_events if e.strip()])
    events_list = events.splitlines()
    return events_list

def get_tweet_str():
    ''' Obtain the string to tweet. It does sanity checks.

    Returns str
    '''
    trials = 10
    today_str = get_today_str()
    tweet_str = 'Nothing to tweet today. #' + today_str
    events_list = get_events_list(today_str)
    list_size = len(events_list)
    if list_size == 0:
        return tweet_str

    while trials > 0:
        i = random.randrange(0, list_size)
        entry_str = events_list[i]
        if len(entry_str) + len(' #' + today_str) <= twitter_allowed_char:
            tweet_str = entry_str + ' #' + today_str
            return tweet_str
        else:
            trials = trials - 1

    return tweet_str

def do_tweet(str):
    ''' Tweet str to Twitter

    '''
    [ApiKey, ApiSecret, AccessToken, AccessTokenSecret] = get_api_token()
    api = Twython(ApiKey, ApiSecret, AccessToken, AccessTokenSecret)
    api.update_status(status=str)
    print("Tweeted: ", str)

if __name__ == '__main__':
    tweet_str = get_tweet_str()
    print(tweet_str)
    do_tweet(tweet_str)
<commit_msg>Use with keyword when opening a file This can be treated as a bugfix.<commit_after>#!/usr/bin/env python

import os
import sys
import random
import subprocess
import wikipedia
from twython import Twython

ApiKey = ''
ApiSecret = ''
AccessToken = ''
AccessTokenSecret = ''
cred_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                         '.auth')
twitter_allowed_char = 260

def get_api_token():
    ''' Obtain Twitter app's API token from file .auth

    Returns list
    '''
    with open(cred_file, 'rb') as f:
        c = f.read()
        t = c.splitlines()
        return t[0:4]

def get_today_str():
    ''' Obtain current date in 'Month_Date' format, e.g., March 3

    Returns str
    '''
    d = subprocess.check_output(["date", "+%B_%d"])
    d_str = d.strip()
    return d_str.decode('utf-8')

def get_events_list(date_str):
    ''' Obtain the list of events for date_str

    Returns list
    '''
    page = wikipedia.WikipediaPage(title=str(date_str))
    # Use some heuristics to take into account of recent wikipedia page format
    # changes
    maybe_events = [page.section(page.sections[i]) for i in
        range(page.sections.index('Events'), page.sections.index('Births'))]
    # Drop empty string
    events = '\n'.join([e.strip() for e in maybe_events if e.strip()])
    events_list = events.splitlines()
    return events_list

def get_tweet_str():
    ''' Obtain the string to tweet. It does sanity checks.

    Returns str
    '''
    trials = 10
    today_str = get_today_str()
    tweet_str = 'Nothing to tweet today. #' + today_str
    events_list = get_events_list(today_str)
    list_size = len(events_list)
    if list_size == 0:
        return tweet_str

    while trials > 0:
        i = random.randrange(0, list_size)
        entry_str = events_list[i]
        if len(entry_str) + len(' #' + today_str) <= twitter_allowed_char:
            tweet_str = entry_str + ' #' + today_str
            return tweet_str
        else:
            trials = trials - 1

    return tweet_str

def do_tweet(str):
    ''' Tweet str to Twitter

    '''
    [ApiKey, ApiSecret, AccessToken, AccessTokenSecret] = get_api_token()
    api = Twython(ApiKey, ApiSecret, AccessToken, AccessTokenSecret)
    api.update_status(status=str)
    print("Tweeted: ", str)

if __name__ == '__main__':
    tweet_str = get_tweet_str()
    print(tweet_str)
    do_tweet(tweet_str)
<|endoftext|>


<commit_before>import requests
from enum import Enum

from messagebird.serde import json_serialize

try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin

class ResponseFormat(Enum):
    text = 1
    binary = 2

class HttpClient(object):
    """Used for sending simple HTTP requests."""

    def __init__(self, endpoint, access_key, user_agent):
        self.__supported_status_codes = [200, 201, 204, 401, 404, 405, 422]

        self.endpoint = endpoint
        self.access_key = access_key
        self.user_agent = user_agent

    def request(self, path, method='GET', params=None, format=ResponseFormat.text):
        """Builds a request and gets a response."""
        if params is None:
            params = {}
        url = urljoin(self.endpoint, path)
        headers = {
            'Accept': 'application/json',
            'Authorization': 'AccessKey ' + self.access_key,
            'User-Agent': self.user_agent,
            'Content-Type': 'application/json'
        }

        method_switcher = {
            'DELETE': lambda: requests.delete(url, verify=True, headers=headers, data=json_serialize(params)),
            'GET': lambda: requests.get(url, verify=True, headers=headers, params=params),
            'PATCH': lambda: requests.patch(url, verify=True, headers=headers, data=json_serialize(params)),
            'POST': lambda: requests.post(url, verify=True, headers=headers, data=json_serialize(params)),
            'PUT': lambda: requests.put(url, verify=True, headers=headers, data=json_serialize(params))
        }
        response = method_switcher.get(method, str(method) + ' is not a supported HTTP method')
        if isinstance(response, str):
            raise ValueError(response)
        response = response()

        if response.status_code not in self.__supported_status_codes:
            response.raise_for_status()

        response_switcher = {
            ResponseFormat.text: response.text,
            ResponseFormat.binary: response.content
        }
        return response_switcher.get(format)
<commit_msg>Changed allowed methods check<commit_after>import requests
from enum import Enum

from messagebird.serde import json_serialize

try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin

class ResponseFormat(Enum):
    text = 1
    binary = 2

class HttpClient(object):
    """Used for sending simple HTTP requests."""

    def __init__(self, endpoint, access_key, user_agent):
        self.__supported_status_codes = [200, 201, 204, 401, 404, 405, 422]

        self.endpoint = endpoint
        self.access_key = access_key
        self.user_agent = user_agent

    def request(self, path, method='GET', params=None, format=ResponseFormat.text):
        """Builds a request and gets a response."""
        if params is None:
            params = {}
        url = urljoin(self.endpoint, path)
        headers = {
            'Accept': 'application/json',
            'Authorization': 'AccessKey ' + self.access_key,
            'User-Agent': self.user_agent,
            'Content-Type': 'application/json'
        }

        method_switcher = {
            'DELETE': lambda: requests.delete(url, verify=True, headers=headers, data=json_serialize(params)),
            'GET': lambda: requests.get(url, verify=True, headers=headers, params=params),
            'PATCH': lambda: requests.patch(url, verify=True, headers=headers, data=json_serialize(params)),
            'POST': lambda: requests.post(url, verify=True, headers=headers, data=json_serialize(params)),
            'PUT': lambda: requests.put(url, verify=True, headers=headers, data=json_serialize(params))
        }
        if method not in method_switcher:
            raise ValueError(str(method) + ' is not a supported HTTP method')

        response = method_switcher[method]()

        if response.status_code not in self.__supported_status_codes:
            response.raise_for_status()

        response_switcher = {
            ResponseFormat.text: response.text,
            ResponseFormat.binary: response.content
        }
        return response_switcher.get(format)
<|endoftext|>


<commit_before>import requests

try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin

class HttpClient(object):
    """Used for sending simple HTTP requests."""

    def __init__(self, endpoint, access_key, user_agent):
        self.__supported_status_codes = [200, 201, 204, 401, 404, 405, 422]

        self.endpoint = endpoint
        self.access_key = access_key
        self.user_agent = user_agent

    def request(self, path, method='GET', params=None):
        """Builds a request and gets a response."""
        if params is None: params = {}
        url = urljoin(self.endpoint, path)

        headers = {
            'Accept': 'application/json',
            'Authorization': 'AccessKey ' + self.access_key,
            'User-Agent': self.user_agent,
            'Content-Type': 'application/json'
        }

        if method == 'GET':
            response = requests.get(url, verify=True, headers=headers, params=params)
        else:
            response = requests.post(url, verify=True, headers=headers, data=json.dumps(params))

        if response.status_code in self.__supported_status_codes:
            response_text = response.text
        else:
            response.raise_for_status()

        return response_text<commit_msg>Fix support for HTTP request with methods other than GET and POST.<commit_after>import json
import requests

try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin

class HttpClient(object):
    """Used for sending simple HTTP requests."""

    def __init__(self, endpoint, access_key, user_agent):
        self.__supported_status_codes = [200, 201, 204, 401, 404, 405, 422]

        self.endpoint = endpoint
        self.access_key = access_key
        self.user_agent = user_agent

    def request(self, path, method='GET', params=None):
        """Builds a request and gets a response."""
        if params is None: params = {}
        url = urljoin(self.endpoint, path)

        headers = {
            'Accept': 'application/json',
            'Authorization': 'AccessKey ' + self.access_key,
            'User-Agent': self.user_agent,
            'Content-Type': 'application/json'
        }

        if method == 'DELETE':
            response = requests.delete(url, verify=True, headers=headers, data=json.dumps(params))
        elif method == 'GET':
            response = requests.get(url, verify=True, headers=headers, params=params)
        elif method == 'PATCH':
            response = requests.patch(url, verify=True, headers=headers, data=json.dumps(params))
        elif method == 'POST':
            response = requests.post(url, verify=True, headers=headers, data=json.dumps(params))
        elif method == 'PUT':
            response = requests.put(url, verify=True, headers=headers, data=json.dumps(params))
        else:
            raise ValueError(str(method) + ' is not a supported HTTP method')

        if response.status_code in self.__supported_status_codes:
            response_text = response.text
        else:
            response.raise_for_status()

        return response_text
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#----------model import----------

import time
import random
import redis.exceptions
from datetime import datetime

import tbdmConfig
from tbdmSetting import tbdmDatabase
from tbdmLogging import tbdmLogger

#----------model import----------

#----------global variables----------

"""
@param:
    tbdmDb         tbdmDatabase instance.
    logger        tbdmLogger instance.
    mongoCli    MongoDB connections. This can be maintained by the manager in future,
                        'cause pymongo(3.3.0) instances are thread-safe (but not fork-safe).
    redis0Cli    Redis connections to db0. This can be maintained by the manager in future,
                        'cause py-redis(2.10.5) instances are thread-safe.
"""
#Workers maintain their own connections
tbdmDb = tbdmDatabase()

#----------global variables----------

#----------class definition----------
class redisCheck():
    """
    Scrape pipeline for TBDM Project.
    !! This class must be instantiated before use.
    @pythonVersion: 3.5.2
    @methods:
    tasker         Acquires tasks for this worker and sets up a
                file lock for recovering.
    postman        Checks for last fail records and sends them back
                to redis after task acquiring.
    worker         Does the work and record what have been done.
    @author: X.Huang, P.Liu
    @maintenance: X.Huang, P.Liu
    @creation: 2016-11-8
    @modified: 2016-11-8
    @version: 0.01-alpha
    """
    redisCli = tbdmDb.tbdmRedis(addrOwner = 'xhuang', auth = True)

    def reporter(self):
        result = {0 : 0, 1 : 0, 2 : 0, "today" : 0, "nearnd" : {}, "total" : 0}
        nowstamp = time.time()
        todayLeft = (datetime.strptime("23:59:59", "%H:%M:%S") -
                    datetime.strptime(datetime.fromtimestamp(nowstamp).strftime("%H:%M:%S"), "%H:%M:%S")).seconds
        with self.redisCli.pipeline() as poppipe:
            while True:
                try:
                    #Starting transaction
                    poppipe.watch('juList')
                    tasklist = poppipe.zrange('juList', 0, -1)
                    poppipe.execute()
                    if tasklist:
                        for item in tasklist:
                                score = int(item.decode().split('/')[2])
                                if (score == 0):
                                    result[0] += 1
                                elif (score <= nowstamp + 3600):
                                    result[1] += 1
                                elif (score <= nowstamp + 7200):
                                    result[2] += 1
                                if (score <= nowstamp + todayLeft):
                                    result["today"] += 1
                                else:
                                    if (score in result["nearnd"].keys()):
                                            result["nearnd"][score] += 1
                                    else:
                                        if (len(result["nearnd"]) < 8):
                                            result["nearnd"][score] = 1
                                        elif (score < max(result["nearnd"].keys())):
                                            result["nearnd"].pop()
                                            result["nearnd"][score] = 1
                                result["total"] += 1
                    else:
                        print("Redis returned empty list, exiting.")
                        #Redis returned no task
                    break
                except redis.exceptions.WatchError as _Ewatch:
                    print("Redis watch error, retry fetching.")
                    time.sleep(random.uniform(3, 5))
                    continue
                except Exception as _Eall:
                    print(_Eall)

        print("=====REPORT=====\nTasks: " + str(result["total"]))
        print("Tasks not fetched yet: " + str(result[0]))
        print("Tasks within 1 hour:   " + str(result[1]))
        print("Tasks within 2 hour:   " + str(result[2]))
        print("Tasks within today:    " + str(result["today"]))
        print("\nEarliest task after today: ")
        for k, v in result["nearnd"].items():
            print(datetime.fromtimestamp(k).strftime("%Y-%m-%d %H:%M:%S") + " : " + str(v))
        print("=====ENDREP=====")
        return

#----------class definition----------

#----------function definition----------

#----------function definition----------

#----------main function----------

if __name__ == "__main__":
    if (tbdmConfig.WHO_IAM == ''):
        # Check for Config
        print("Tell me who you are in tbdmConfig before starting RedisCheck.")
    else:
        rcheck = redisCheck()
        rcheck.reporter()

#----------main function----------
<commit_msg>* tbdmRedisCheck.py Function Enhance.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#----------model import----------

import time
import random
import redis.exceptions
from datetime import datetime

import tbdmConfig
from tbdmSetting import tbdmDatabase
from tbdmLogging import tbdmLogger

#----------model import----------

#----------global variables----------

"""
@param:
    tbdmDb         tbdmDatabase instance.
    logger        tbdmLogger instance.
    mongoCli    MongoDB connections. This can be maintained by the manager in future,
                        'cause pymongo(3.3.0) instances are thread-safe (but not fork-safe).
    redis0Cli    Redis connections to db0. This can be maintained by the manager in future,
                        'cause py-redis(2.10.5) instances are thread-safe.
"""
#Workers maintain their own connections
tbdmDb = tbdmDatabase()

#----------global variables----------

#----------class definition----------
class redisCheck():
    """
    Scrape pipeline for TBDM Project.
    !! This class must be instantiated before use.
    @pythonVersion: 3.5.2
    @methods:
    tasker         Acquires tasks for this worker and sets up a
                file lock for recovering.
    postman        Checks for last fail records and sends them back
                to redis after task acquiring.
    worker         Does the work and record what have been done.
    @author: X.Huang, P.Liu
    @maintenance: X.Huang, P.Liu
    @creation: 2016-11-8
    @modified: 2016-11-8
    @version: 0.01-alpha
    """
    redisCli = tbdmDb.tbdmRedis(addrOwner = 'xhuang', auth = True)

    def reporter(self):
        result = {0 : 0, 1 : 0, 2 : 0, "today" : 0, "nearnd" : {}, "total" : 0}
        nowstamp = time.time()
        todayLeft = (datetime.strptime("23:59:59", "%H:%M:%S") -
                    datetime.strptime(datetime.fromtimestamp(nowstamp).strftime("%H:%M:%S"), "%H:%M:%S")).seconds
        with self.redisCli.pipeline() as poppipe:
            while True:
                try:
                    #Starting transaction
                    poppipe.watch('juList')
                    tasklist = poppipe.zrange('juList', 0, -1)
                    poppipe.execute()
                    if tasklist:
                        for item in tasklist:
                                score = int(item.decode().split('/')[2])
                                if (score == 0):
                                    result[0] += 1
                                elif (score <= nowstamp + 3600):
                                    result[1] += 1
                                elif (score <= nowstamp + 7200):
                                    result[2] += 1
                                if (score <= nowstamp + todayLeft):
                                    result["today"] += 1
                                else:
                                    if (score in result["nearnd"].keys()):
                                            result["nearnd"][score] += 1
                                    else:
                                        if (len(result["nearnd"]) < 12):
                                            result["nearnd"][score] = 1
                                        elif (score < max(result["nearnd"].keys())):
                                            result["nearnd"].pop()
                                            result["nearnd"][score] = 1
                                result["total"] += 1
                    else:
                        print("Redis returned empty list, exiting.")
                        #Redis returned no task
                    break
                except redis.exceptions.WatchError as _Ewatch:
                    print("Redis watch error, retry fetching.")
                    time.sleep(random.uniform(3, 5))
                    continue
                except Exception as _Eall:
                    print(_Eall)

        print("=====REPORT=====\nTasks: " + str(result["total"]))
        print("Tasks not fetched yet: " + str(result[0]))
        print("Tasks within 1 hour:   " + str(result[1]))
        print("Tasks within 2 hour:   " + str(result[2]))
        print("Tasks within today:    " + str(result["today"]))
        print("\nEarliest task after today: ")
        for k, v in result["nearnd"].items():
            print(datetime.fromtimestamp(k).strftime("%Y-%m-%d %H:%M:%S") + " : " + str(v))
        print("=====ENDREP=====")
        return

#----------class definition----------

#----------function definition----------

#----------function definition----------

#----------main function----------

if __name__ == "__main__":
    if (tbdmConfig.WHO_IAM == ''):
        # Check for Config
        print("Tell me who you are in tbdmConfig before starting RedisCheck.")
    else:
        rcheck = redisCheck()
        rcheck.reporter()

#----------main function----------
<|endoftext|>


<commit_before>import yaml.nodes

__all__ = [
    'construct_from_mapping',
    'construct_from_sequence',
    'construct_from_string',
    'construct_from_integer',
    'construct_from_any',
    'construct_from_none',
    'construct_from_value',
    'construct_from_args_kwargs',
]

def construct_from_mapping(cls, loader, node):
    mapping = loader.construct_mapping(node, True)
    return cls(**mapping)

def construct_from_sequence(cls, loader, node):
    sequence = loader.construct_sequence(node, True)
    return cls(*sequence)

def construct_from_string(cls, loader, node):
    scalar = str(loader.construct_scalar(node))
    return cls(scalar)

def construct_from_integer(cls, loader, node):
    scalar = int(loader.construct_scalar(node))
    return cls(scalar)

def construct_from_none(cls, loader, node):
    assert loader.construct_yaml_null(node) is None
    return cls()

def construct_from_value(cls, loader, node):
    if isinstance(node, yaml.nodes.ScalarNode):
        value = loader.construct_scalar(node)
    elif isinstance(node, yaml.nodes.SequenceNode):
        value = loader.construct_sequence(node, True)
    elif isinstance(node, yaml.nodes.MappingNode):
        value = loader.construct_mapping(node, True)
    else:
        raise ValueError(node)
    return cls(value)

def construct_from_args_kwargs(cls, loader, node):
    mapping = loader.construct_mapping(node, True)
    args = mapping.pop("=", ())
    return cls(*args, **mapping)

def construct_from_any(cls, loader, node):
    if isinstance(node, yaml.nodes.ScalarNode):
        if loader.construct_yaml_null(node) is None:
            return cls()
        else:
            value = loader.construct_scalar(node)
            return cls(value)
    elif isinstance(node, yaml.nodes.SequenceNode):
        return construct_from_sequence(cls, loader, node)
    elif isinstance(node, yaml.nodes.MappingNode):
        return construct_from_args_kwargs(cls, loader, node)
    else:
        raise ValueError(node)
<commit_msg>better value constructor<commit_after>import yaml.nodes

__all__ = [
    'construct_from_mapping',
    'construct_from_sequence',
    'construct_from_string',
    'construct_from_integer',
    'construct_from_any',
    'construct_from_none',
    'construct_from_value',
    'construct_from_args_kwargs',
]

def construct_from_mapping(cls, loader, node):
    mapping = loader.construct_mapping(node, True)
    return cls(**mapping)

def construct_from_sequence(cls, loader, node):
    sequence = loader.construct_sequence(node, True)
    return cls(*sequence)

def construct_from_string(cls, loader, node):
    scalar = str(loader.construct_scalar(node))
    return cls(scalar)

def construct_from_integer(cls, loader, node):
    scalar = int(loader.construct_scalar(node))
    return cls(scalar)

def construct_from_none(cls, loader, node):
    assert loader.construct_yaml_null(node) is None
    return cls()

def construct_from_value(cls, loader, node):
    if isinstance(node, yaml.nodes.ScalarNode):
        tag = loader.resolve(yaml.nodes.ScalarNode, node.value, (True, False))
        subnode = yaml.nodes.ScalarNode(tag, node.value, node.start_mark, node.end_mark, node.style)
        value = loader.construct_object(subnode, True)
    elif isinstance(node, yaml.nodes.SequenceNode):
        value = loader.construct_sequence(node, True)
    elif isinstance(node, yaml.nodes.MappingNode):
        value = loader.construct_mapping(node, True)
    else:
        raise ValueError(node)
    return cls(value)

def construct_from_args_kwargs(cls, loader, node):
    mapping = loader.construct_mapping(node, True)
    args = mapping.pop("=", ())
    return cls(*args, **mapping)

def construct_from_any(cls, loader, node):
    if isinstance(node, yaml.nodes.ScalarNode):
        if loader.construct_yaml_null(node) is None:
            return cls()
        else:
            value = loader.construct_scalar(node)
            return cls(value)
    elif isinstance(node, yaml.nodes.SequenceNode):
        return construct_from_sequence(cls, loader, node)
    elif isinstance(node, yaml.nodes.MappingNode):
        return construct_from_args_kwargs(cls, loader, node)
    else:
        raise ValueError(node)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2016 Gerard Kok
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# work in progress, do not use

import os
import shutil
import subprocess
import re

from autopkglib import Processor, ProcessorError

__all__ = ["SVNUpdater"]

class SVNUpdater(Processor):
    """Keeps an svn working copy up to date."""
    description = __doc__
    input_variables = {
        "source": {
            "required": True,
            "description": "svn repository url",
        },
        "working_copy_dir": {
            "required": False,
            "description": "path to working copy",
        },
    }
    output_variables = {
        "updated": {
            "description": "if the working copy has been updated"
        },
        "revision": {
            "description": "the revision the working copy is updated to"
        }
    }

    REVISION_RE = re.compile('^Revision:\s+(\d+)')

    def get_working_copy_dir(self):
        if 'working_copy_dir' in self.env:
            return self.env['working_copy_dir']
        else:
            return os.path.join(self.env.get('RECIPE_CACHE_DIR'), 'downloads')

    def is_working_copy(path):
        return os.path.isdir(path) and os.path.isdir(os.path.join(path, '.svn'))

    def create_dir(path):
        shutil.rmtree(path)
        os.makedirs(path)

    def run_svn_cmd(params, working_copy_dir):
        svn_cmd = ['/usr/bin/svn', '--non-interactive']
        svn_cmd.extend(params)
        p = subprocess.Popen(svn_cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=working_copy_dir)
        return p.communicate()

    def get_latest_rev(self):
        (out, err) = run_svn_cmd(['info', '-r', 'HEAD'])
        match = REVISION_RE.search(out)
        return match.group(match.lastindex or 0)

    def get_current_ref(self):
        (out, err) = run_svn_cmd(['info'])
        match = REVISION_RE.search(out)
        return match.group(match.lastindex or 0)

    def update_working_copy(self):
        run_svn_cmd(['update'])

    def checkout_working_copy(self):
        run_svn_cmd(['checkout'])

    def main(self):
        source = self.env['source']
        working_copy_dir = self.get_working_copy_dir()
        if is_working_copy(working_copy_dir):
            latest_rev = self.get_latest_rev()
            current_rev = self.get_current_rev()
            if latest_rev > current_rev:
                self.update_working_copy()
                self.env['updated'] = True
            else:
                self.env['updated'] = False
        else:
            create_dir(working_copy_dir)
            self.checkout_working_copy()
            self.env['updated'] = True

if __name__ == '__main__':
    processor = SVNUpdater()
    processor.execute_shell()
    <commit_msg>move global functions outside class declaration<commit_after>#!/usr/bin/env python
#
# Copyright 2016 Gerard Kok
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# work in progress, do not use

import os
import shutil
import subprocess
import re

from autopkglib import Processor, ProcessorError

__all__ = ["SVNUpdater"]

def is_working_copy(path):
    return os.path.isdir(path) and os.path.isdir(os.path.join(path, '.svn'))

def create_dir(path):
    shutil.rmtree(path)
    os.makedirs(path)

def run_svn_cmd(params, working_copy_dir):
    svn_cmd = ['/usr/bin/svn', '--non-interactive']
    svn_cmd.extend(params)
    p = subprocess.Popen(svn_cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        cwd=working_copy_dir)
    return p.communicate()

class SVNUpdater(Processor):
    """Keeps an svn working copy up to date."""
    description = __doc__
    input_variables = {
        "source": {
            "required": True,
            "description": "svn repository url",
        },
        "working_copy_dir": {
            "required": False,
            "description": "path to working copy",
        },
    }
    output_variables = {
        "updated": {
            "description": "if the working copy has been updated"
        },
        "revision": {
            "description": "the revision the working copy is updated to"
        }
    }

    REVISION_RE = re.compile('^Revision:\s+(\d+)')

    def get_working_copy_dir(self):
        if 'working_copy_dir' in self.env:
            return self.env['working_copy_dir']
        else:
            return os.path.join(self.env.get('RECIPE_CACHE_DIR'), 'downloads')

    def get_latest_rev(self):
        (out, err) = run_svn_cmd(['info', '-r', 'HEAD'])
        match = REVISION_RE.search(out)
        return match.group(match.lastindex or 0)

    def get_current_ref(self):
        (out, err) = run_svn_cmd(['info'])
        match = REVISION_RE.search(out)
        return match.group(match.lastindex or 0)

    def update_working_copy(self):
        run_svn_cmd(['update'])

    def checkout_working_copy(self):
        run_svn_cmd(['checkout'])

    def main(self):
        source = self.env['source']
        working_copy_dir = self.get_working_copy_dir()
        if is_working_copy(working_copy_dir):
            latest_rev = self.get_latest_rev()
            current_rev = self.get_current_rev()
            if latest_rev > current_rev:
                self.update_working_copy()
                self.env['updated'] = True
            else:
                self.env['updated'] = False
        else:
            create_dir(working_copy_dir)
            self.checkout_working_copy()
            self.env['updated'] = True

if __name__ == '__main__':
    processor = SVNUpdater()
    processor.execute_shell()
    <|endoftext|>