<commit_before>#!/usr/bin/env python
import argparse
import logging
import os

from foremast import (configs, consts, iam, s3,
securitygroup, elb, dns, slacknotify, app, pipeline)
import gogoutils

LOG = logging.getLogger(__name__)
logging.basicConfig(format=consts.LOGGING_FORMAT)
logging.getLogger("foremast").setLevel(logging.DEBUG)

class ForemastRunner:
    def __init__(self, group=None, repo=None, env=None,
                region=None, email=None):
        """ Setups the Runner for all Foremast modules

        Args:
            group (str): Gitlab group of the application
            repo (str): The application repository name
            env (str): Deployment environment, dev/stage/prod etc
            region (str): AWS region (us-east-1)
        """
        self.group = group
        self.repo = repo
        self.git_project = "{}/{}".format(group, repo)
        parsed = gogoutils.Parser(self.git_project)
        generated = gogoutils.Generator(*parsed.parse_url())
        self.app = generated.app
        self.trigger_job = generated.jenkins()['name']
        self.git_short = generated.gitlab()['main']
        self.gitlab_token_path = os.environ["HOME"]+ "/.aws/git.token"
        self.raw_path = "./raw.properties"
        self.json_path = self.raw_path + ".json"
        self.env = env
        self.region = region
        self.email= email
        self.configs = None

    def write_configs(self):
        print('===========================')
        print('Read application.json')
        print('===========================')
        if not self.gitlab_token_path:
            raise SystemExit('Must provide private token file as well.')
        self.configs = configs.process_git_configs(git_short=self.git_short,
                                      token_file=self.gitlab_token_path)

        configs.write_variables(app_configs=self.configs,
                        out_file=self.raw_path,
                        git_short=self.git_short)

    def create_app(self):
        print('===========================')
        print('Creating Spinnaker App')
        print('===========================')
        spinnakerapp = app.SpinnakerApp(app=self.app,
                                        email=self.email,
                                        project=self.group,
                                        repo=self.repo)
        spinnakerapp.create_app()

    def create_pipeline(self):
        spinnakerpipeline = pipeline.SpinnakerPipeline(app=self.app,
                                                       trigger_job=self.trigger_job,
                                                       prop_path=self.json_path,
                                                       base=None,
                                                       token_file=self.gitlab_token_path)
        spinnakerpipeline.create_pipeline()

    def create_iam(self):
        print('===========================')
        print('Create IAM')
        print('===========================')
        iam.create_iam_resources(env=self.env, app=self.app)

    def create_s3(self):
        print('===========================')
        print('Create S3')
        print('===========================')
        s3.init_properties(env=self.env, app=self.app)

    def create_secgroups(self):
        print('===========================')
        print('Create Security Group')
        print('===========================')
        sgobj = securitygroup.SpinnakerSecurityGroup(app=self.app,
                                                    env=self.env,
                                                    region=self.region,
                                                    prop_path=self.json_path)
        sgobj.create_security_group()

    def create_elb(self):
        print('===========================')
        print('Create ELB')
        print('===========================')
        elbobj = elb.SpinnakerELB(app=self.app,
                                env=self.env,
                                region=self.region,
                                prop_path=self.json_path)
        elbobj.create_elb()

    def create_dns(self):
        print('===========================')
        print('Create DNS')
        print('===========================')
        elb_type = self.configs[self.env]['elb']['subnet_purpose']
        dnsobj = dns.SpinnakerDns(app=self.app,
                                     env=self.env,
                                     region=self.region,
                                     elb_subnet=elb_subnet)
        dnsobj.create_elb_dns()

    def slack_notify(self):
        print('===========================')
        print('Sending slack notification')
        print('===========================')
        if self.env.startswith("prod"):
            notify = slacknotify.SlackNotification(app=self.app,
                                                  env=self.env,
                                                  prop_path=self.json_path)
            notify.post_message()
        else:
            LOG.info("No slack message sent, not production environment")

    def cleanup(self):
        os.remove(self.raw_path)

    def prepare_infrastructure(self):
        """ This runs everything necessary to prepare the infrastructure in a specific env """
        self.write_configs()
        self.create_iam()
        self.create_s3()
        self.create_secgroups()
        if not self.configs[self.env]['app']['eureka_enabled']:
            LOG.info("No Eureka, running ELB and DNS setup")
            self.create_elb()
            self.create_dns()
        else:
            LOG.info("Eureka Enabled, skipping ELB and DNS setup")
        self.slack_notify()
        self.cleanup()

    def prepare_app_pipeline(self):
        self.create_app()
        self.write_configs()
        self.create_pipeline()
        self.cleanup

    def prepare_onetime_pipeline(self):
        return

if __name__ == "__main__":
    #foremastrunner = ForemastRunner(group="forrest", repo="edge", env='dev')
    foremastrunner = ForemastRunner(group="forrest", repo="edge")
    foremastrunner.prepare_app_pipeline()

<commit_msg>pipeline and app creating working, added banner. Still need to test infra builds<commit_after>#!/usr/bin/env python
import argparse
import logging
import os

from foremast import (configs, consts, iam, s3,
securitygroup, elb, dns, slacknotify, app, pipeline)
import gogoutils

LOG = logging.getLogger(__name__)
logging.basicConfig(format=consts.LOGGING_FORMAT)
logging.getLogger("foremast").setLevel(logging.INFO)

class ForemastRunner:
    def __init__(self, group=None, repo=None, env=None,
                region=None, email=None):
        """ Setups the Runner for all Foremast modules

        Args:
            group (str): Gitlab group of the application
            repo (str): The application repository name
            env (str): Deployment environment, dev/stage/prod etc
            region (str): AWS region (us-east-1)
        """
        self.group = group
        self.repo = repo
        self.git_project = "{}/{}".format(group, repo)
        parsed = gogoutils.Parser(self.git_project)
        generated = gogoutils.Generator(*parsed.parse_url())
        self.app = generated.app
        self.trigger_job = generated.jenkins()['name']
        self.git_short = generated.gitlab()['main']
        self.gitlab_token_path = os.environ["HOME"]+ "/.aws/git.token"
        self.raw_path = "./raw.properties"
        self.json_path = self.raw_path + ".json"
        self.env = env
        self.region = region
        self.email= email
        self.configs = None

    def write_configs(self):
        self.banner("Generating Configs")
        if not self.gitlab_token_path:
            raise SystemExit('Must provide private token file as well.')
        self.configs = configs.process_git_configs(git_short=self.git_short,
                                      token_file=self.gitlab_token_path)

        configs.write_variables(app_configs=self.configs,
                        out_file=self.raw_path,
                        git_short=self.git_short)

    def create_app(self):
        self.banner("Creating Spinnaker App")
        spinnakerapp = app.SpinnakerApp(app=self.app,
                                        email=self.email,
                                        project=self.group,
                                        repo=self.repo)
        spinnakerapp.create_app()

    def create_pipeline(self, onetime=None):
        self.banner("Creating Pipeline")
        if not onetime:
            spinnakerpipeline = pipeline.SpinnakerPipeline(app=self.app,
                                                       trigger_job=self.trigger_job,
                                                       prop_path=self.json_path,
                                                       base=None,
                                                       token_file=self.gitlab_token_path)
        else:
             spinnakerpipeline = pipeline.SpinnakerPipelineOnetime(app=self.app,
                                                       trigger_job=self.trigger_job,
                                                       prop_path=self.json_path,
                                                       base=None,
                                                       token_file=self.gitlab_token_path,
                                                       onetime=onetime)

        spinnakerpipeline.create_pipeline()

    def create_iam(self):
        self.banner("Creating IAM")
        iam.create_iam_resources(env=self.env, app=self.app)

    def create_s3(self):
        self.banner("Creating S3")
        s3.init_properties(env=self.env, app=self.app)

    def create_secgroups(self):
        self.banner("Creating Security Group")
        sgobj = securitygroup.SpinnakerSecurityGroup(app=self.app,
                                                    env=self.env,
                                                    region=self.region,
                                                    prop_path=self.json_path)
        sgobj.create_security_group()

    def create_elb(self):
        self.banner("Creating ELB")
        elbobj = elb.SpinnakerELB(app=self.app,
                                env=self.env,
                                region=self.region,
                                prop_path=self.json_path)
        elbobj.create_elb()

    def create_dns(self):
        self.banner("Creating DNS")
        elb_type = self.configs[self.env]['elb']['subnet_purpose']
        dnsobj = dns.SpinnakerDns(app=self.app,
                                     env=self.env,
                                     region=self.region,
                                     elb_subnet=elb_subnet)
        dnsobj.create_elb_dns()

    def slack_notify(self):
        self.banner("Sending slack notification")
        if self.env.startswith("prod"):
            notify = slacknotify.SlackNotification(app=self.app,
                                                  env=self.env,
                                                  prop_path=self.json_path)
            notify.post_message()
        else:
            LOG.info("No slack message sent, not production environment")

    def banner(self, notice):
        """ Prints a banner for seperating tasks """
        print("\n===============================")
        print(notice)
        print("===============================\n")

    def cleanup(self):
        os.remove(self.raw_path)

    def prepare_infrastructure(self):
        """ This runs everything necessary to prepare the infrastructure in a specific env """
        self.write_configs()
        self.create_iam()
        self.create_s3()
        self.create_secgroups()
        if not self.configs[self.env]['app']['eureka_enabled']:
            LOG.info("No Eureka, running ELB and DNS setup")
            self.create_elb()
            self.create_dns()
        else:
            LOG.info("Eureka Enabled, skipping ELB and DNS setup")
        self.slack_notify()
        self.cleanup()

    def prepare_app_pipeline(self):
        self.create_app()
        self.write_configs()
        self.create_pipeline()
        self.cleanup

    def prepare_onetime_pipeline(self, onetime=None):
        self.write_configs()
        self.create_pipeline(onetime=onetime)
        self.cleanup()

if __name__ == "__main__":
    #foremastrunner = ForemastRunner(group="forrest", repo="edge", env='dev')
    foremastrunner = ForemastRunner(group="forrest", repo="edge")
    foremastrunner.prepare_onetime_pipeline(onetime='dev')

<|endoftext|>


<commit_before>#!/usr/bin/python
# This will take a bedgraph and expand it into a weird sort of bed file, where we have
# CHR LOC DNASE_1 ... DNASE_N
# N is the size of the window around LOC, and DNASE_i are the DNASE values corresponding the window locations
# The output of this script shall be the input to the feature extraction part of the larger code
import gzip
import sys

# arbitrary!
WIN_SIZE = 25
WIN_EDGE = WIN_SIZE/2

def overlap(A1,A2,B1,B2):
    if A1 <= B1 and B1 <= A2:
        return True
    elif B1 <= A1 and A1 <= B2:
        return True
    else:
        return False

def in_range(x,A,B):
    if x < A or x > B:
        return False
    else:
        return True

def get_peak_range(pre_line):
    line = pre_line.strip()
    colon = line.find(':')
    hyphen = line.find('-')
    start = int(line[(colon+1):hyphen])
    end = int(line[(hyphen+1):])
    return (start,end)

def get_flanks(i,buffer,WIN_EDGE):
# OUTSTANDING QUESTION: what to do when there is insufficient buffer to do this?
# Also WHY is that happening?
    flank_list = [element.split()[2] for element in buffer[(i-WIN_EDGE):(i+WIN_EDGE+1)]]
#    if not len(flank_list)==(2*WIN_EDGE+1):
#        print buffer[(i-WIN_EDGE):(i+WIN_EDGE+1)]
#        print WIN_EDGE
#        print i-WIN_EDGE
#        print buffer[i+WIN_EDGE+1]
#        print buffer[i-WIN_EDGE]
#        print len(flank_list)
#        sys.exit()
    flanks = ' '.join(flank_list)
    return flanks

def process_peak(peak_num,peak_start,peak_end,buffer,outfile,new_peaks_file):
    # what we have: one line per location (good!), with CHR LOC VAL PEAK_NUM
    # peak_end is the last location in the peak, not the upper bound
    peak_length = peak_end-peak_start+1
    buffer_length = len(buffer)
    # this will be the total signal in the peak... can rewrite the dnase_peaks file to include this, i guess
    peak_total = 0
    for i in range(buffer_length):
        element = buffer[i]
        [chr,loc,val] = element.split()
        if in_range(int(loc),peak_start,peak_end):
            peak_total = peak_total + float(val)
            flanks = get_flanks(i,buffer,WIN_EDGE)
            loc_line = chr+' '+str(loc)+' '+str(peak_num)+' '+flanks+'\n'
            outfile.write(loc_line)
    new_peaks_file.write(chr+'\t'+str(peak_start)+'\t'+str(peak_end)+'\t'+str(peak_total)+'\n')

# each line of this file contains peak range
peaks_file = open('../k562_dnase_peaks','r')
new_peaks_file = open('../k562_dnase_peaks_totals','w')
bedgraph = gzip.open(sys.argv[1],'r')
out_file = open('../processed_signal_in_peaks.bed','w')

first_peak = peaks_file.readline()
[peak_start,peak_end] = get_peak_range(first_peak)

peak_buffer = []

line_num = 0
peak_num = 1
for line in bedgraph:
    # file's pretty big
    if line_num%100000==0:
        print line_num
    line_num = line_num+1

    chro = line.split()[0]
    start = int(line.split()[1])
    end = int(line.split()[2])
    if overlap(start,end,(peak_start-WIN_EDGE),(peak_end+WIN_EDGE)):
        length = end-start
        val = float(line.split()[3])
        # creating a buffer of all the locs (expanded - one loc per line) in the peak
        for n in range(length):
            temp_loc = start+n
            temp_line = chro+' '+str(temp_loc)+' '+str(val)
            peak_buffer.append(temp_line)
    else:
        if len(peak_buffer)>0:
            process_peak(peak_num,peak_start,peak_end,peak_buffer,out_file,new_peaks_file)
            peak_buffer = []
            peak_num = peak_num + 1
        if end>peak_end:
            # get a new peak
            new_peak = peaks_file.readline()
            # avoid EOF
            if len(new_peak)>0:
                [peak_start,peak_end] = get_peak_range(new_peak)

<commit_msg>solution to the problem: if there isn't enough space for the window around the peak, shrink the peak!<commit_after>#!/usr/bin/python
# This will take a bedgraph and expand it into a weird sort of bed file, where we have
# CHR LOC DNASE_1 ... DNASE_N
# N is the size of the window around LOC, and DNASE_i are the DNASE values corresponding the window locations
# The output of this script shall be the input to the feature extraction part of the larger code
import gzip
import sys

# arbitrary!
WIN_SIZE = 25
WIN_EDGE = WIN_SIZE/2

def overlap(A1,A2,B1,B2):
    if A1 <= B1 and B1 <= A2:
        return True
    elif B1 <= A1 and A1 <= B2:
        return True
    else:
        return False

def in_range(x,A,B):
    if x < A or x > B:
        return False
    else:
        return True

def get_peak_range(pre_line):
    line = pre_line.strip()
    colon = line.find(':')
    hyphen = line.find('-')
    start = int(line[(colon+1):hyphen])
    end = int(line[(hyphen+1):])
    return (start,end)

def get_flanks(i,buff,WIN_EDGE):
# OUTSTANDING QUESTION: what to do when there is insufficient buffer to do this?
# Also WHY is that happening?
    flank_list = [element.split()[2] for element in buff[(i-WIN_EDGE):(i+WIN_EDGE+1)]]
#    if not len(flank_list)==(2*WIN_EDGE+1):
#        print buff[(i-WIN_EDGE):(i+WIN_EDGE+1)]
#        print WIN_EDGE
#        print i-WIN_EDGE
#        print buff[i+WIN_EDGE+1]
#        print buff[i-WIN_EDGE]
#        print len(flank_list)
#        sys.exit()
    flanks = ' '.join(flank_list)
    return flanks

def process_peak(peak_num,peak_start,peak_end,buff,outfile,new_peaks_file):
    # what we have: one line per location (good!), with CHR LOC VAL PEAK_NUM
    # peak_end is the last location in the peak, not the upper bound
    peak_length = peak_end-peak_start+1
    buff_length = len(buff)
    # what if there isn't enough buffer? NOTE: THIS MODIFIES THE PEAK CALLS, SO WE NEED TO USE THE OUTPUT OF THIS SCRIPT TO DEFINE OUR PEAKS FROM NOW ON... (the new_peaks_file)..
    buff_start = int(buff[0].split()[1])
    buff_end = int(buff[-1].split()[1])
    if (peak_start-buff_start)<WIN_EDGE:
        # move up the start of the peak
        peak_start = buff_start+WIN_EDGE
    if (buff_end-peak_end)<WIN_EDGE:
        # move in the end of the peak
        peak_end = buff_end-WIN_EDGE
    # this will be the total signal in the peak... can rewrite the dnase_peaks file to include this, i guess
    peak_total = 0
    for i in range(buff_length):
        element = buff[i]
        [chr,loc,val] = element.split()
        if in_range(int(loc),peak_start,peak_end):
            flanks = get_flanks(i,buff,WIN_EDGE)
            peak_total = peak_total + float(val)
            loc_line = chr+' '+str(loc)+' '+str(peak_num)+' '+flanks+'\n'
            outfile.write(loc_line)
    new_peaks_file.write(chr+'\t'+str(peak_start)+'\t'+str(peak_end)+'\t'+str(peak_total)+'\n')

# each line of this file contains peak range
peaks_file = open(sys.argv[2],'r')
new_peaks_file = open(sys.argv[2]+'.totals','w')
bedgraph = gzip.open(sys.argv[1],'r')
out_file = open(sys.argv[1]+'.processed','w')

first_peak = peaks_file.readline()
[peak_start,peak_end] = get_peak_range(first_peak)

peak_buff = []

line_num = 0
peak_num = 1
for line in bedgraph:
    # file's pretty big
    if line_num%100000==0:
        print line_num
    line_num = line_num+1

    chro = line.split()[0]
    start = int(line.split()[1])
    end = int(line.split()[2])
    if overlap(start,end,(peak_start-WIN_EDGE),(peak_end+WIN_EDGE)):
        length = end-start
        val = float(line.split()[3])
        # creating a buffer of all the locs (expanded - one loc per line) in the peak
        for n in range(length):
            temp_loc = start+n
            temp_line = chro+' '+str(temp_loc)+' '+str(val)
            peak_buff.append(temp_line)
    else:
        if len(peak_buff)>0:
            process_peak(peak_num,peak_start,peak_end,peak_buff,out_file,new_peaks_file)
            peak_buff = []
            peak_num = peak_num + 1
        if end>peak_end:
            # get a new peak
            new_peak = peaks_file.readline()
            # avoid EOF
            if len(new_peak)>0:
                [peak_start,peak_end] = get_peak_range(new_peak)

<|endoftext|>


<commit_before>"""DES Survey"""

import pdb

import numpy as np
from astropy import units, io, utils

from frb.surveys import dlsurvey
from frb.surveys import catalog_utils

# Dependencies
try:
    from pyvo.dal import sia
except ImportError:
    print("Warning:  You need to install pyvo to retrieve DES images")
    _svc = None
else:
    _DEF_ACCESS_URL = "https://datalab.noao.edu/sia/des_dr1"
    _svc = sia.SIAService(_DEF_ACCESS_URL)

# Define the data model for DES data
photom = {}
photom['DES'] = {}
DES_bands = ['g', 'r', 'i', 'z', 'Y']
for band in DES_bands:
    photom['DES']['DES_{:s}'.format(band)] = 'mag_auto_{:s}'.format(band.lower())
    photom['DES']['DES_{:s}_err'.format(band)] = 'magerr_auto_{:s}'.format(band.lower())
photom['DES']['DES_ID'] = 'coadd_object_id'
photom['DES']['ra'] = 'ra'
photom['DES']['dec'] = 'dec'
photom['DES']['DES_tile'] = 'tilename'

# DES-WISE
photom['DES-WISE'] = {}
DES_WISE_bands = ['W1', 'W2', 'W3', 'W4']
for band in DES_WISE_bands:
    photom['DES-WISE']['{:s}'.format(band)] = '{:s}mpro'.format(band.lower())
    photom['DES-WISE']['{:s}_err'.format(band)] = '{:s}sigmpro'.format(band.lower())
photom['DES-WISE']['DES_ID'] = 'coadd_object_id'
photom['DES-WISE']['DES_ra'] = 'des_ra'
photom['DES-WISE']['DES_dec'] = 'des_dec'
photom['DES-WISE']['WISE_ra'] = 'ra'
photom['DES-WISE']['WISE_dec'] = 'dec'

class DES_Survey(dlsurvey.DL_Survey):
    """
    Class to handle queries on the DECaL survey

    Child of DL_Survey which uses datalab to access NOAO

    Args:
        coord (SkyCoord): Coordiante for surveying around
        radius (Angle): Search radius around the coordinate

    """

    def __init__(self, coord, radius, **kwargs):
        dlsurvey.DL_Survey.__init__(self, coord, radius, **kwargs)
        self.survey = 'DES'
        self.bands = ['g', 'r', 'i', 'z', 'y']
        self.svc = sia.SIAService("https://datalab.noao.edu/sia/des_dr1")
        self.qc_profile = "default"
        self.database = "des_dr1.main"

    def _parse_cat_band(self,band):
        """
        Internal method to generate the bands for grabbing
        a cutout image

        For DES, nothing much is necessary.

        Args:
            band (str): Band desired

        Returns:
            list, list, str:  Table columns, Column values, band string for cutout

        """
        table_cols = ['proctype','prodtype']
        col_vals = ['Stack','image']

        return table_cols, col_vals, band

    def get_catalog(self, query=None, query_fields=None, print_query=False):
        """
        Grab a catalog of sources around the input coordinate to the search radius

        Args:
            query: Not used
            query_fields (list, optional): Over-ride list of items to query
            print_query (bool): Print the SQL query generated

        Returns:
            astropy.table.Table:  Catalog of sources returned.  Includes WISE
            photometry for matched sources.
        """
        # Main DES query
        main_cat = super(DES_Survey, self).get_catalog(query_fields=query_fields, print_query=print_query)
        if len(main_cat) == 0:
            return main_cat
        main_cat = catalog_utils.clean_cat(main_cat, photom['DES'])

        # WISE
        wise_query = self._gen_cat_query(qtype='wise')
        wise_cat = super(DES_Survey, self).get_catalog(query=wise_query, print_query=print_query)
        wise_cat = catalog_utils.clean_cat(wise_cat, photom['DES-WISE'], fill_mask=-999.)

        # Match em up
        if len(wise_cat) > 0:
            idx = catalog_utils.match_ids(wise_cat['DES_ID'], main_cat['DES_ID'])
            # Fill me
            for band in DES_WISE_bands:
                main_cat['{:s}'.format(band)] = -999.
                main_cat['{:s}'.format(band)][idx] = wise_cat['{:s}'.format(band)]
                main_cat['{:s}_err'.format(band)] = -999.
                main_cat['{:s}_err'.format(band)][idx] = wise_cat['{:s}_err'.format(band)]

        # Finish
        self.catalog = main_cat
        self.validate_catalog()
        return self.catalog

    def _gen_cat_query(self,query_fields=None, qtype='main'):
        """
        Generate SQL Query for catalog search

        self.query is modified in place

        Args:
            query_fields (list):  Override the default list for the SQL query

        """
        if query_fields is None:
            query_fields = []
            # Main query
            if qtype == 'main':
                for key,value in photom['DES'].items():
                    query_fields += [value]
                database = self.database
            elif qtype == 'wise':
                for key,value in photom['DES-WISE'].items():
                    query_fields += [value]
                database = "des_dr1.des_allwise"
            else:
                raise IOError("Bad qtype")

        self.query = dlsurvey._default_query_str(query_fields, database,self.coord,self.radius)
        # Return
        return self.query

    def _select_best_img(self,imgTable,verbose,timeout=120):
        """
        Select the best band for a cutout

        Args:
            imgTable: Table of images
            verbose (bool):  Print status
            timeout (int or float):  How long to wait before timing out, in seconds

        Returns:
            HDU: header data unit for the downloaded image

        """
        row = imgTable[np.argmax(imgTable['exptime'].data.data.astype('float'))] # pick image with longest exposure time
        url = row['access_url'].decode()
        if verbose:
            print ('downloading deepest stacked image...')

        imagedat = io.fits.open(utils.data.download_file(url,cache=True,show_progress=False,timeout=timeout))
        return imagedat

<commit_msg>DES and WISE cats may have different lengths.<commit_after>"""DES Survey"""

import pdb

import numpy as np
from astropy import units, io, utils

from frb.surveys import dlsurvey
from frb.surveys import catalog_utils

# Dependencies
try:
    from pyvo.dal import sia
except ImportError:
    print("Warning:  You need to install pyvo to retrieve DES images")
    _svc = None
else:
    _DEF_ACCESS_URL = "https://datalab.noao.edu/sia/des_dr1"
    _svc = sia.SIAService(_DEF_ACCESS_URL)

# Define the data model for DES data
photom = {}
photom['DES'] = {}
DES_bands = ['g', 'r', 'i', 'z', 'Y']
for band in DES_bands:
    photom['DES']['DES_{:s}'.format(band)] = 'mag_auto_{:s}'.format(band.lower())
    photom['DES']['DES_{:s}_err'.format(band)] = 'magerr_auto_{:s}'.format(band.lower())
photom['DES']['DES_ID'] = 'coadd_object_id'
photom['DES']['ra'] = 'ra'
photom['DES']['dec'] = 'dec'
photom['DES']['DES_tile'] = 'tilename'

# DES-WISE
photom['DES-WISE'] = {}
DES_WISE_bands = ['W1', 'W2', 'W3', 'W4']
for band in DES_WISE_bands:
    photom['DES-WISE']['{:s}'.format(band)] = '{:s}mpro'.format(band.lower())
    photom['DES-WISE']['{:s}_err'.format(band)] = '{:s}sigmpro'.format(band.lower())
photom['DES-WISE']['DES_ID'] = 'coadd_object_id'
photom['DES-WISE']['DES_ra'] = 'des_ra'
photom['DES-WISE']['DES_dec'] = 'des_dec'
photom['DES-WISE']['WISE_ra'] = 'ra'
photom['DES-WISE']['WISE_dec'] = 'dec'

class DES_Survey(dlsurvey.DL_Survey):
    """
    Class to handle queries on the DECaL survey

    Child of DL_Survey which uses datalab to access NOAO

    Args:
        coord (SkyCoord): Coordiante for surveying around
        radius (Angle): Search radius around the coordinate

    """

    def __init__(self, coord, radius, **kwargs):
        dlsurvey.DL_Survey.__init__(self, coord, radius, **kwargs)
        self.survey = 'DES'
        self.bands = ['g', 'r', 'i', 'z', 'y']
        self.svc = sia.SIAService("https://datalab.noao.edu/sia/des_dr1")
        self.qc_profile = "default"
        self.database = "des_dr1.main"

    def _parse_cat_band(self,band):
        """
        Internal method to generate the bands for grabbing
        a cutout image

        For DES, nothing much is necessary.

        Args:
            band (str): Band desired

        Returns:
            list, list, str:  Table columns, Column values, band string for cutout

        """
        table_cols = ['proctype','prodtype']
        col_vals = ['Stack','image']

        return table_cols, col_vals, band

    def get_catalog(self, query=None, query_fields=None, print_query=False):
        """
        Grab a catalog of sources around the input coordinate to the search radius

        Args:
            query: Not used
            query_fields (list, optional): Over-ride list of items to query
            print_query (bool): Print the SQL query generated

        Returns:
            astropy.table.Table:  Catalog of sources returned.  Includes WISE
            photometry for matched sources.
        """
        # Main DES query
        main_cat = super(DES_Survey, self).get_catalog(query_fields=query_fields, print_query=print_query)
        if len(main_cat) == 0:
            return main_cat
        main_cat = catalog_utils.clean_cat(main_cat, photom['DES'])

        # WISE
        wise_query = self._gen_cat_query(qtype='wise')
        wise_cat = super(DES_Survey, self).get_catalog(query=wise_query, print_query=print_query)
        wise_cat = catalog_utils.clean_cat(wise_cat, photom['DES-WISE'], fill_mask=-999.)

        # Match em up
        if len(wise_cat) > 0:
            idx = catalog_utils.match_ids(wise_cat['DES_ID'], main_cat['DES_ID'],require_in_match=False)
            # Fill me
            for band in DES_WISE_bands:
                main_cat['{:s}'.format(band)] = -999.
                main_cat['{:s}'.format(band)][idx] = wise_cat['{:s}'.format(band)]
                main_cat['{:s}_err'.format(band)] = -999.
                main_cat['{:s}_err'.format(band)][idx] = wise_cat['{:s}_err'.format(band)]

        # Finish
        self.catalog = main_cat
        self.validate_catalog()
        return self.catalog

    def _gen_cat_query(self,query_fields=None, qtype='main'):
        """
        Generate SQL Query for catalog search

        self.query is modified in place

        Args:
            query_fields (list):  Override the default list for the SQL query

        """
        if query_fields is None:
            query_fields = []
            # Main query
            if qtype == 'main':
                for key,value in photom['DES'].items():
                    query_fields += [value]
                database = self.database
            elif qtype == 'wise':
                for key,value in photom['DES-WISE'].items():
                    query_fields += [value]
                database = "des_dr1.des_allwise"
            else:
                raise IOError("Bad qtype")

        self.query = dlsurvey._default_query_str(query_fields, database,self.coord,self.radius)
        # Return
        return self.query

    def _select_best_img(self,imgTable,verbose,timeout=120):
        """
        Select the best band for a cutout

        Args:
            imgTable: Table of images
            verbose (bool):  Print status
            timeout (int or float):  How long to wait before timing out, in seconds

        Returns:
            HDU: header data unit for the downloaded image

        """
        row = imgTable[np.argmax(imgTable['exptime'].data.data.astype('float'))] # pick image with longest exposure time
        url = row['access_url'].decode()
        if verbose:
            print ('downloading deepest stacked image...')

        imagedat = io.fits.open(utils.data.download_file(url,cache=True,show_progress=False,timeout=timeout))
        return imagedat

<|endoftext|>


<commit_before>from flask import Flask, render_template, request, Response
import uuid
import os
import datetime
import json

from database_handler import DatabaseHandler
from models.house_record import HouseRecord

app = Flask(__name__)
app.debug = True

upload_folder = '/opt/stack/folsom'
app.config['upload_folder'] = upload_folder

handler = DatabaseHandler('sqlite:///house_record.db')

#users
users = ['admin', 'user', 'anna', 'jake']

# Routes
@app.route("/")
def index():
    return render_template('base.html')

@app.route('/login')
# GET renders HTML login form
def login():
    return render_template('partials/login.html')

@app.route('/upload', methods=['POST'])
def upload():
    file = request.files['csv']
    if file.content_type == 'text/csv':
        filename = file.filename
        try:
            file.save(os.path.join(app.config['upload_folder'], filename))
        except Exception as e:
            data = json.dumps({"message": "error when saving file on disk"})
            resp = Response(data, status=511, mimetype='application/json')
            return resp
        i = 0
        records = []
        my_csv = open(app.config['upload_folder']+"/"+filename, 'r')
        for line in my_csv.readlines():
            values = line.replace('"', "").split(',')[:11]
            if values[0] == 'id':
                continue
            h = HouseRecord()
            h.id = str(uuid.uuid4())
            h.user_id = i % len(users) + 1
            h.date_posted = datetime.datetime.strptime(values[1].split('T')[0],
                                                       "%Y%m%d").date()
            h.price = values[2]
            h.beds = values[3]
            h.baths = values[4]
            h.sqft_house = values[5]
            h.sqft_lot = values[6]
            h.floors = values[7]
            h.condition = values[10]
            i += 1
            records.append(h)

        handler.save_records(records)
        data = json.dumps({"message": "data stored"})
        resp = Response(data, status=200, mimetype='application/json')
        #send messages to rabbitmq queue
        #for record in records:
        #    message = json.dumps({"uuid": record.id, "action": "default"})
        return resp
    else:
        data = json.dumps({"message": "Only csv is supported"})
        resp = Response(data, status=501, mimetype='application/json')
        return resp

@app.route('/viewall')
def view():
    try:
        results = handler.get_houserecords()
        houserecords = []
        for result in results:
            data ={"id":result.id,
                   "date_posted": result.date_posted,
                   "price": result.price,
                   "beds": result.beds,
                   "baths": result.baths,
                   "sqft_house": result.sqft_house,
                   "sqft_lot": result.sqft_lot,
                   "floors": result.floors,
                       "condition": result.condition}
            houserecords.append(data)
        print houserecords
        data = json.dumps({"message": houserecords})
        resp = Response(data, status=200, mimetype='application/json')
        return resp
    except Exception as e:
        resp = Response( status=500)
        return resp

if __name__ == "__main__":
    app.run(host='192.168.33.12', port=8181, debug=True)
<commit_msg>updating imports in server.py after database module was extracted<commit_after>from flask import Flask, render_template, request, Response
import uuid
import os
import datetime
import json

from database.database_handler import DatabaseHandler
from database.models.house_record import HouseRecord

app = Flask(__name__)
app.debug = True

upload_folder = '/opt/stack/folsom'
app.config['upload_folder'] = upload_folder

handler = DatabaseHandler('sqlite:///house_record.db')

#users
users = ['admin', 'user', 'anna', 'jake']

# Routes
@app.route("/")
def index():
    return render_template('base.html')

@app.route('/login')
# GET renders HTML login form
def login():
    return render_template('partials/login.html')

@app.route('/upload', methods=['POST'])
def upload():
    file = request.files['csv']
    if file.content_type == 'text/csv':
        filename = file.filename
        try:
            file.save(os.path.join(app.config['upload_folder'], filename))
        except Exception as e:
            data = json.dumps({"message": "error when saving file on disk"})
            resp = Response(data, status=511, mimetype='application/json')
            return resp
        i = 0
        records = []
        my_csv = open(app.config['upload_folder']+"/"+filename, 'r')
        for line in my_csv.readlines():
            values = line.replace('"', "").split(',')[:11]
            if values[0] == 'id':
                continue
            h = HouseRecord()
            h.id = str(uuid.uuid4())
            h.user_id = i % len(users) + 1
            h.date_posted = datetime.datetime.strptime(values[1].split('T')[0],
                                                       "%Y%m%d").date()
            h.price = values[2]
            h.beds = values[3]
            h.baths = values[4]
            h.sqft_house = values[5]
            h.sqft_lot = values[6]
            h.floors = values[7]
            h.condition = values[10]
            i += 1
            records.append(h)

        handler.save_records(records)
        data = json.dumps({"message": "data stored"})
        resp = Response(data, status=200, mimetype='application/json')
        #send messages to rabbitmq queue
        #for record in records:
        #    message = json.dumps({"uuid": record.id, "action": "default"})
        return resp
    else:
        data = json.dumps({"message": "Only csv is supported"})
        resp = Response(data, status=501, mimetype='application/json')
        return resp

@app.route('/viewall')
def view():
    try:
        results = handler.get_houserecords()
        houserecords = []
        for result in results:
            data ={"id":result.id,
                   "date_posted": result.date_posted,
                   "price": result.price,
                   "beds": result.beds,
                   "baths": result.baths,
                   "sqft_house": result.sqft_house,
                   "sqft_lot": result.sqft_lot,
                   "floors": result.floors,
                       "condition": result.condition}
            houserecords.append(data)
        print houserecords
        data = json.dumps({"message": houserecords})
        resp = Response(data, status=200, mimetype='application/json')
        return resp
    except Exception as e:
        resp = Response( status=500)
        return resp

if __name__ == "__main__":
    app.run(host='192.168.33.12', port=8181, debug=True)
<|endoftext|>


<commit_before>from flask import Flask, redirect, render_template, request, url_for
from flask.ext.pymongo import PyMongo
import markdown2
import re

app = Flask(__name__)
mongo = PyMongo(app)

WIKIPART = re.compile(r'([A-Z][a-z0-9_]+)')
WIKIWORD = re.compile(r'([A-Z][a-z0-9_]+(?:[A-Z][a-z0-9_]+)+)')

@app.route('/', methods=['GET'])
def redirect():
    return redirect(url_for('HomePage'))

@app.template_filter()
def totitle(value):
    return ' '.join(WIKIPART.findall(value))

@app.template_filter()
def wikify(value):
    parts = WIKIWORD.split(value)
    for i, part in enumerate(parts):
        if WIKIWORD.match(part):
            name = totitle(part)
            parts[i] = '[%s](%s)' % (name, url_for('show_page', pagepath=part))
    return markdown2.markdown(''.join(parts))

@app.route('/<path:pagepath>')
def show_page(pagepath):
    page = mongo.db.pages.find_one_or_404({'_id': pagepath})
    return render_template('page.html',
        page=page,
        pagepath=pagepath)

@app.route('/edit/<path:pagepath>', methods=['GET'])
def edit_page(pagepath):
    page = mongo.db.pages.find_one_or_404({'_id': pagepath})
    return render_template('edit.html',
        page=page,
        pagepath=pagepath)

@app.route('/edit/<path:pagepath>', methods=['POST'])
def save_page(pagepath):
    if 'cancel' not in request.form:
        mongo.db.pages.update(
            {'_id': pagepath},
            {'$set': {'body': request.form['body']}},
            safe=True, upsert=True)
    return redirect(url_for('show_page', pagepath=pagepath))

@app.errorhandler(404)
def new_page(error):
    pagepath = request.path.lstrip('/')
    if pagepath.startswith('uploads'):
        filename = pagepath[len('uploads'):].lstrip('/')
        return render_template('upload.html', filename=filename)
    else:
        return render_template('edit.html', page=None, pagepath=pagepath)

@app.route('/uploads/<path:filename>')
def get_upload(filename):
    return mongo.send_file(filename)

@app.route('/uploads/<path:filename>', methods=['POST'])
def save_upload(filename):
    if request.files.get('file'):
        mongo.save_file(filename, request.files['file'])
        return redirect(url_for('get_upload', filename=filename))
    return render_template('upload.html', filename=filename)

if __name__ == '__main__':
    import doctest
    doctest.testmod()
    app.run(debug=True)
<commit_msg>don't shadow 'redirect' from flask<commit_after>from flask import Flask, redirect, render_template, request, url_for
from flask.ext.pymongo import PyMongo
import markdown2
import re

app = Flask(__name__)
mongo = PyMongo(app)

WIKIPART = re.compile(r'([A-Z][a-z0-9_]+)')
WIKIWORD = re.compile(r'([A-Z][a-z0-9_]+(?:[A-Z][a-z0-9_]+)+)')

@app.route('/', methods=['GET'])
def redirect_to_homepage():
    return redirect(url_for('HomePage'))

@app.template_filter()
def totitle(value):
    return ' '.join(WIKIPART.findall(value))

@app.template_filter()
def wikify(value):
    parts = WIKIWORD.split(value)
    for i, part in enumerate(parts):
        if WIKIWORD.match(part):
            name = totitle(part)
            parts[i] = '[%s](%s)' % (name, url_for('show_page', pagepath=part))
    return markdown2.markdown(''.join(parts))

@app.route('/<path:pagepath>')
def show_page(pagepath):
    page = mongo.db.pages.find_one_or_404({'_id': pagepath})
    return render_template('page.html',
        page=page,
        pagepath=pagepath)

@app.route('/edit/<path:pagepath>', methods=['GET'])
def edit_page(pagepath):
    page = mongo.db.pages.find_one_or_404({'_id': pagepath})
    return render_template('edit.html',
        page=page,
        pagepath=pagepath)

@app.route('/edit/<path:pagepath>', methods=['POST'])
def save_page(pagepath):
    if 'cancel' not in request.form:
        mongo.db.pages.update(
            {'_id': pagepath},
            {'$set': {'body': request.form['body']}},
            safe=True, upsert=True)
    return redirect(url_for('show_page', pagepath=pagepath))

@app.errorhandler(404)
def new_page(error):
    pagepath = request.path.lstrip('/')
    if pagepath.startswith('uploads'):
        filename = pagepath[len('uploads'):].lstrip('/')
        return render_template('upload.html', filename=filename)
    else:
        return render_template('edit.html', page=None, pagepath=pagepath)

@app.route('/uploads/<path:filename>')
def get_upload(filename):
    return mongo.send_file(filename)

@app.route('/uploads/<path:filename>', methods=['POST'])
def save_upload(filename):
    if request.files.get('file'):
        mongo.save_file(filename, request.files['file'])
        return redirect(url_for('get_upload', filename=filename))
    return render_template('upload.html', filename=filename)

if __name__ == '__main__':
    import doctest
    doctest.testmod()
    app.run(debug=True)
<|endoftext|>


<commit_before>from decimal import Decimal
from helpers import get_datetime, get_response

TICKER_URL = 'https://bitstamp.net/api/ticker/'

def get_current_price():
    data = get_response(TICKER_URL)
    price = data['last']
    return Decimal(price)

def get_current_bid():
    data = get_response(TICKER_URL)
    price = data['bid']
    return Decimal(price)

def get_current_ask():
    data = get_response(TICKER_URL)
    price = data['ask']
    return Decimal(price)

print get_current_price()
<commit_msg>Tidy up<commit_after>from decimal import Decimal
from helpers import get_datetime, get_response

TICKER_URL = 'https://bitstamp.net/api/ticker/'

def get_current_price():
    data = get_response(TICKER_URL)
    price = data['last']
    return Decimal(price)

def get_current_bid():
    data = get_response(TICKER_URL)
    price = data['bid']
    return Decimal(price)

def get_current_ask():
    data = get_response(TICKER_URL)
    price = data['ask']
    return Decimal(price)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import unittest

import intelmq.lib.test as test
from intelmq.bots.experts.cymru_whois.expert import CymruExpertBot

EXAMPLE_INPUT = {"__type": "Event",
                 "source.ip": "93.184.216.34",  # example.com
                 "time.observation": "2015-01-01T00:00:00+00:00",
                 }
EXAMPLE_OUTPUT = {"__type": "Event",
                  "source.ip": "93.184.216.34",
                  "source.geolocation.cc": "EU",
                  "source.registry": "RIPE",
                  "source.network": "93.184.216.0/24",
                  "source.allocated": "2008-06-02T00:00:00+00:00",
                  "source.asn": 15133,
                  "source.as_name": "EDGECAST - MCI Communications Services, Inc. d/b/a Verizon Business, US",
                  "time.observation": "2015-01-01T00:00:00+00:00",
                  }
EXAMPLE_INPUT6 = {"__type": "Event",
                  "destination.ip": "2001:500:88:200::8",  # iana.org
                  "time.observation": "2015-01-01T00:00:00+00:00",
                  }
EXAMPLE_OUTPUT6 = {"__type": "Event",
                   "destination.ip": "2001:500:88:200::8",  # iana.org
                   "destination.registry": "ARIN",
                   "destination.allocated": "2010-02-18T00:00:00+00:00",
                   "destination.as_name": "ICANN-DC - ICANN, US",
                   "destination.geolocation.cc": "US",
                   "time.observation": "2015-01-01T00:00:00+00:00",
                   "destination.asn": 16876,
                   "destination.network": "2001:500:88::/48",
                   }
UNICODE_INPUT = {"__type": "Event",
                 "destination.ip": "200.236.128.1",
                 "time.observation": "2015-01-01T00:00:00+00:00",
                 }
UNICODE_OUTPUT = {"__type": "Event",
                  "destination.ip": "200.236.128.1",
                  "time.observation": "2015-01-01T00:00:00+00:00",
                  "destination.registry": "LACNIC",
                  "destination.allocated": "2000-02-15T00:00:00+00:00",
                  "destination.as_name": "Fundação de Desenvolvimento da Pesquisa, BR",
                  "destination.geolocation.cc": "BR",
                  "destination.asn": 10417,
                  "destination.network": "200.236.128.0/18",
                  }
EMPTY_INPUT = {"__type": "Event",
               "source.ip": "198.105.125.77",  # no result
               "time.observation": "2015-01-01T00:00:00+00:00",
               }
NO_ASN_INPUT = {"__type": "Event",
                "source.ip": "212.92.127.126",
                "time.observation": "2015-01-01T00:00:00+00:00",
                }
NO_ASN_OUTPUT = {"__type": "Event",
                 "source.ip": "212.92.127.126",
                 "time.observation": "2015-01-01T00:00:00+00:00",
                 "source.asn": 23456,
                 "source.geolocation.cc": 'RU',
                 "source.ip": '212.92.127.126',
                 "source.network": '212.92.127.0/24',
                 "source.registry": 'RIPE',
                 }

@test.skip_redis()
@test.skip_internet()
class TestCymruExpertBot(test.BotTestCase, unittest.TestCase):
    """
    A TestCase for AbusixExpertBot.
    """

    @classmethod
    def set_bot(cls):
        cls.bot_reference = CymruExpertBot
        cls.use_cache = True

    def test_ipv4_lookup(self):
        self.input_message = EXAMPLE_INPUT
        self.run_bot()
        self.assertMessageEqual(0, EXAMPLE_OUTPUT)

    def test_ipv6_lookup(self):
        self.input_message = EXAMPLE_INPUT6
        self.run_bot()
        self.assertMessageEqual(0, EXAMPLE_OUTPUT6)

    def test_unicode_as_name(self):
        self.input_message = UNICODE_INPUT
        self.run_bot()
        self.assertMessageEqual(0, UNICODE_OUTPUT)

    def test_empty_result(self):
        self.input_message = EMPTY_INPUT
        self.run_bot()
        self.assertMessageEqual(0, EMPTY_INPUT)

    def test_missing_asn(self):
        """
        No information for ASN.

        https://github.com/certtools/intelmq/issues/635
        """
        self.input_message = NO_ASN_INPUT
        self.run_bot()
        self.assertMessageEqual(0, NO_ASN_OUTPUT)

if __name__ == '__main__':  # pragma: no cover
    unittest.main()
<commit_msg>TST: Skip cymru bot, it's offline<commit_after># -*- coding: utf-8 -*-

import unittest

import intelmq.lib.test as test
from intelmq.bots.experts.cymru_whois.expert import CymruExpertBot

EXAMPLE_INPUT = {"__type": "Event",
                 "source.ip": "93.184.216.34",  # example.com
                 "time.observation": "2015-01-01T00:00:00+00:00",
                 }
EXAMPLE_OUTPUT = {"__type": "Event",
                  "source.ip": "93.184.216.34",
                  "source.geolocation.cc": "EU",
                  "source.registry": "RIPE",
                  "source.network": "93.184.216.0/24",
                  "source.allocated": "2008-06-02T00:00:00+00:00",
                  "source.asn": 15133,
                  "source.as_name": "EDGECAST - MCI Communications Services, Inc. d/b/a Verizon Business, US",
                  "time.observation": "2015-01-01T00:00:00+00:00",
                  }
EXAMPLE_INPUT6 = {"__type": "Event",
                  "destination.ip": "2001:500:88:200::8",  # iana.org
                  "time.observation": "2015-01-01T00:00:00+00:00",
                  }
EXAMPLE_OUTPUT6 = {"__type": "Event",
                   "destination.ip": "2001:500:88:200::8",  # iana.org
                   "destination.registry": "ARIN",
                   "destination.allocated": "2010-02-18T00:00:00+00:00",
                   "destination.as_name": "ICANN-DC - ICANN, US",
                   "destination.geolocation.cc": "US",
                   "time.observation": "2015-01-01T00:00:00+00:00",
                   "destination.asn": 16876,
                   "destination.network": "2001:500:88::/48",
                   }
UNICODE_INPUT = {"__type": "Event",
                 "destination.ip": "200.236.128.1",
                 "time.observation": "2015-01-01T00:00:00+00:00",
                 }
UNICODE_OUTPUT = {"__type": "Event",
                  "destination.ip": "200.236.128.1",
                  "time.observation": "2015-01-01T00:00:00+00:00",
                  "destination.registry": "LACNIC",
                  "destination.allocated": "2000-02-15T00:00:00+00:00",
                  "destination.as_name": "Fundação de Desenvolvimento da Pesquisa, BR",
                  "destination.geolocation.cc": "BR",
                  "destination.asn": 10417,
                  "destination.network": "200.236.128.0/18",
                  }
EMPTY_INPUT = {"__type": "Event",
               "source.ip": "198.105.125.77",  # no result
               "time.observation": "2015-01-01T00:00:00+00:00",
               }
NO_ASN_INPUT = {"__type": "Event",
                "source.ip": "212.92.127.126",
                "time.observation": "2015-01-01T00:00:00+00:00",
                }
NO_ASN_OUTPUT = {"__type": "Event",
                 "source.ip": "212.92.127.126",
                 "time.observation": "2015-01-01T00:00:00+00:00",
                 "source.asn": 23456,
                 "source.geolocation.cc": 'RU',
                 "source.ip": '212.92.127.126',
                 "source.network": '212.92.127.0/24',
                 "source.registry": 'RIPE',
                 }

@test.skip_redis()
@test.skip_internet()
class TestCymruExpertBot(test.BotTestCase, unittest.TestCase):
    """
    A TestCase for AbusixExpertBot.
    """

    @classmethod
    def set_bot(cls):
        cls.bot_reference = CymruExpertBot
        cls.use_cache = True

    @unittest.skip("Team Cymru is currently offline.")
    def test_ipv4_lookup(self):
        self.input_message = EXAMPLE_INPUT
        self.run_bot()
        self.assertMessageEqual(0, EXAMPLE_OUTPUT)

    @unittest.skip("Team Cymru is currently offline.")
    def test_ipv6_lookup(self):
        self.input_message = EXAMPLE_INPUT6
        self.run_bot()
        self.assertMessageEqual(0, EXAMPLE_OUTPUT6)

    @unittest.skip("Team Cymru is currently offline.")
    def test_unicode_as_name(self):
        self.input_message = UNICODE_INPUT
        self.run_bot()
        self.assertMessageEqual(0, UNICODE_OUTPUT)

    @unittest.skip("Team Cymru is currently offline.")
    def test_empty_result(self):
        self.input_message = EMPTY_INPUT
        self.run_bot()
        self.assertMessageEqual(0, EMPTY_INPUT)

    @unittest.skip("Team Cymru is currently offline.")
    def test_missing_asn(self):
        """
        No information for ASN.

        https://github.com/certtools/intelmq/issues/635
        """
        self.input_message = NO_ASN_INPUT
        self.run_bot()
        self.assertMessageEqual(0, NO_ASN_OUTPUT)

if __name__ == '__main__':  # pragma: no cover
    unittest.main()
<|endoftext|>


<commit_before>from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db import models
from django.utils import simplejson
from django.core.serializers.json import DjangoJSONEncoder
from django.conf import settings

from jsonfield import JSONField
from modeldict import ModelDict

from gargoyle.manager import gargoyle
from gargoyle.models import Switch

import datetime
import random

CONTROL_GROUP = 'control'

CONTROL_STATE = 0
ENABLED_STATE = 1
GARGOYLE_STATE = 2
TRACK_STATE = 2

STATES = (
    (CONTROL_STATE, 'Control'),
    (ENABLED_STATE, 'Enabled'),
    (GARGOYLE_STATE, 'Gargoyle'),
    (TRACK_STATE, 'Track'),
)

class Experiment(models.Model):
    name = models.CharField(primary_key=True, max_length=128)
    description = models.TextField(default="", blank=True, null=True)
    alternatives = JSONField(default="{}", blank=True)
    relevant_goals = models.TextField(default = "", null=True, blank=True)
    switch_key = models.CharField(default = "", max_length=50, null=True, blank=True)

    state = models.IntegerField(default=CONTROL_STATE, choices=STATES)

    start_date = models.DateTimeField(default=datetime.datetime.now, blank=True, null=True, db_index=True)
    end_date = models.DateTimeField(blank=True, null=True)

    def is_displaying_alternatives(self):
        if self.state == CONTROL_STATE:
            return False
        elif self.state == ENABLED_STATE:
            return True
        elif self.state == GARGOYLE_STATE:
            return True
        elif self.state == TRACK_STATE:
            return True
        else:
            raise Exception("Invalid experiment state %s!" % experiment.state)

    def is_accepting_new_users(self, request):
        if self.state == CONTROL_STATE:
            return False
        elif self.state == ENABLED_STATE:
            return True
        elif self.state == GARGOYLE_STATE:
            return gargoyle.is_active(self.switch_key, request)
        elif self.state == TRACK_STATE:
            return False
        else:
            raise Exception("Invalid experiment state %s!" % experiment.state)

    def ensure_alternative_exists(self, alternative):
        if alternative not in self.alternatives:
            self.alternatives[alternative] = {}
            self.alternatives[alternative]['enabled'] = True
            self.save()

    def random_alternative(self):
        return random.choice(self.alternatives.keys())

    def __unicode__(self):
        return self.name

    def to_dict(self):
        data = {
            'name': self.name,
            'edit_url': reverse('experiments:results', kwargs={'name': self.name}),
            'start_date': self.start_date,
            'end_date': self.end_date,
            'state': self.state,
            'switch_key': self.switch_key,
            'description': self.description,
            'relevant_goals': self.relevant_goals,
        }
        return data

    def to_dict_serialized(self):
        return simplejson.dumps(self.to_dict(), cls=DjangoJSONEncoder)

    def save(self, *args, **kwargs):
        # Create new switch
        if self.switch_key and getattr(settings, 'EXPERIMENTS_SWITCH_AUTO_CREATE', True):
            try:
                Switch.objects.get(key=self.switch_key)
            except Switch.DoesNotExist:
                Switch.objects.create(key=self.switch_key, label=getattr(settings, 'EXPERIMENTS_SWITCH_LABEL', "Experiment: %s") % self.name, description=self.description)

        if not self.switch_key and self.state == 2:
            self.state = 0

        super(Experiment, self).save(*args, **kwargs)

    def delete(self, *args, **kwargs):
        # Delete existing switch
        if getattr(settings, 'EXPERIMENTS_SWITCH_AUTO_DELETE', True):
            try:
                Switch.objects.get(key=Experiment.objects.get(name=self.name).switch_key).delete()
            except Switch.DoesNotExist:
                pass

        super(Experiment, self).delete(*args, **kwargs)

class ExperimentManager(ModelDict):
    def __init__(self, *args, **kwargs):
        self._registry = {}
        super(ExperimentManager, self).__init__(*args, **kwargs)

    def __getitem__(self, key):
        experiment = super(ExperimentManager, self).__getitem__(key)
        return experiment

class Enrollment(models.Model):
    """ A participant in a split testing experiment """
    user = models.ForeignKey(User, null=True)
    experiment = models.ForeignKey(Experiment)
    enrollment_date = models.DateField(db_index=True, auto_now_add=True)
    alternative = models.CharField(max_length=50)
    goals = JSONField(default="[]", blank=True)

    class Meta:
        unique_together = ('user', 'experiment')

    def to_dict(self):
        data = {
            'user': self.user,
            'experiment': self.experiment,
            'enrollment_date': self.enrollment_date,
            'alternative': self.alternative,
            'goals': self.goals,
        }
        return data
<commit_msg>Remove useless code from ExperimentManager<commit_after>from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db import models
from django.utils import simplejson
from django.core.serializers.json import DjangoJSONEncoder
from django.conf import settings

from jsonfield import JSONField
from modeldict import ModelDict

from gargoyle.manager import gargoyle
from gargoyle.models import Switch

import datetime
import random

CONTROL_GROUP = 'control'

CONTROL_STATE = 0
ENABLED_STATE = 1
GARGOYLE_STATE = 2
TRACK_STATE = 2

STATES = (
    (CONTROL_STATE, 'Control'),
    (ENABLED_STATE, 'Enabled'),
    (GARGOYLE_STATE, 'Gargoyle'),
    (TRACK_STATE, 'Track'),
)

class Experiment(models.Model):
    name = models.CharField(primary_key=True, max_length=128)
    description = models.TextField(default="", blank=True, null=True)
    alternatives = JSONField(default="{}", blank=True)
    relevant_goals = models.TextField(default = "", null=True, blank=True)
    switch_key = models.CharField(default = "", max_length=50, null=True, blank=True)

    state = models.IntegerField(default=CONTROL_STATE, choices=STATES)

    start_date = models.DateTimeField(default=datetime.datetime.now, blank=True, null=True, db_index=True)
    end_date = models.DateTimeField(blank=True, null=True)

    def is_displaying_alternatives(self):
        if self.state == CONTROL_STATE:
            return False
        elif self.state == ENABLED_STATE:
            return True
        elif self.state == GARGOYLE_STATE:
            return True
        elif self.state == TRACK_STATE:
            return True
        else:
            raise Exception("Invalid experiment state %s!" % experiment.state)

    def is_accepting_new_users(self, request):
        if self.state == CONTROL_STATE:
            return False
        elif self.state == ENABLED_STATE:
            return True
        elif self.state == GARGOYLE_STATE:
            return gargoyle.is_active(self.switch_key, request)
        elif self.state == TRACK_STATE:
            return False
        else:
            raise Exception("Invalid experiment state %s!" % experiment.state)

    def ensure_alternative_exists(self, alternative):
        if alternative not in self.alternatives:
            self.alternatives[alternative] = {}
            self.alternatives[alternative]['enabled'] = True
            self.save()

    def random_alternative(self):
        return random.choice(self.alternatives.keys())

    def __unicode__(self):
        return self.name

    def to_dict(self):
        data = {
            'name': self.name,
            'edit_url': reverse('experiments:results', kwargs={'name': self.name}),
            'start_date': self.start_date,
            'end_date': self.end_date,
            'state': self.state,
            'switch_key': self.switch_key,
            'description': self.description,
            'relevant_goals': self.relevant_goals,
        }
        return data

    def to_dict_serialized(self):
        return simplejson.dumps(self.to_dict(), cls=DjangoJSONEncoder)

    def save(self, *args, **kwargs):
        # Create new switch
        if self.switch_key and getattr(settings, 'EXPERIMENTS_SWITCH_AUTO_CREATE', True):
            try:
                Switch.objects.get(key=self.switch_key)
            except Switch.DoesNotExist:
                Switch.objects.create(key=self.switch_key, label=getattr(settings, 'EXPERIMENTS_SWITCH_LABEL', "Experiment: %s") % self.name, description=self.description)

        if not self.switch_key and self.state == 2:
            self.state = 0

        super(Experiment, self).save(*args, **kwargs)

    def delete(self, *args, **kwargs):
        # Delete existing switch
        if getattr(settings, 'EXPERIMENTS_SWITCH_AUTO_DELETE', True):
            try:
                Switch.objects.get(key=Experiment.objects.get(name=self.name).switch_key).delete()
            except Switch.DoesNotExist:
                pass

        super(Experiment, self).delete(*args, **kwargs)

class ExperimentManager(ModelDict):
    pass

class Enrollment(models.Model):
    """ A participant in a split testing experiment """
    user = models.ForeignKey(User, null=True)
    experiment = models.ForeignKey(Experiment)
    enrollment_date = models.DateField(db_index=True, auto_now_add=True)
    alternative = models.CharField(max_length=50)
    goals = JSONField(default="[]", blank=True)

    class Meta:
        unique_together = ('user', 'experiment')

    def to_dict(self):
        data = {
            'user': self.user,
            'experiment': self.experiment,
            'enrollment_date': self.enrollment_date,
            'alternative': self.alternative,
            'goals': self.goals,
        }
        return data
<|endoftext|>


<commit_before>from __future__ import print_function, unicode_literals

import datetime
import json
import traceback

import progressbar

from django.contrib.auth.models import User
from django.core.management.base import BaseCommand
from django.db.models import Q

from inthe_am.taskmanager.models import TaskStore, TaskStoreStatistic
from inthe_am.taskmanager.lock import (
    get_lock_name_for_store,
    get_lock_redis,
    redis_lock,
)

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument(
            'subcommand',
            nargs=1,
            choices=[
                'list',
                'lock',
                'unlock',
                'search',
                'update_statistics',
                'gc_large_repos',
                'squash',
                'delete_old_accounts',
            ],
            type=str,
        )
        parser.add_argument(
            'username',
            nargs='?',
            type=str
        )
        parser.add_argument(
            '--minutes',
            type=int,
            default=5,
        )
        parser.add_argument(
            '--force',
            action='store_true',
            default=False,
        )
        parser.add_argument(
            '--repack-size',
            type=int,
            default=int(5e7)
        )
        parser.add_argument(
            '--squash-size',
            type=int,
            default=int(1e8)
        )

    def handle(self, *args, **options):
        subcommand = options['subcommand'][0]
        username = options['username']
        minutes = options['minutes']
        repack_size = options['repack_size']
        squash_size = options['squash_size']

        if subcommand == 'lock':
            store = TaskStore.objects.get(user__username=username)
            store.set_lock_state(lock=True, seconds=minutes*60)
            print('{} locked'.format(store))
        elif subcommand == 'unlock':
            store = TaskStore.objects.get(user__username=username)
            store.set_lock_state(lock=False)
            print('{} unlocked'.format(store))
        elif subcommand == 'search':
            users = User.objects.filter(
                Q(email__contains=username) |
                Q(username__contains=username) |
                Q(first_name__contains=username) |
                Q(last_name__contains=username)
            )
            for user in users:
                print(user.username)
        elif subcommand == 'list':
            redis = get_lock_redis()
            for key in redis.keys('*.lock'):
                value = datetime.datetime.fromtimestamp(
                    int(float(redis.get(key)))
                )
                if value > datetime.datetime.utcnow():
                    print('{}: {}'.format(key, value))
        elif subcommand == 'update_statistics':
            run_id = 'update_statistics_{date}'.format(
                date=datetime.datetime.now().strftime('%Y%m%dT%H%M%SZ')
            )
            print("Run ID: {}".format(run_id))
            with progressbar.ProgressBar(
                max_value=TaskStore.objects.count(),
                widgets=[
                    ' [', progressbar.Timer(), '] ',
                    progressbar.Bar(),
                    ' (', progressbar.ETA(), ') ',
                ]
            ) as bar:
                for idx, store in enumerate(
                    TaskStore.objects.order_by('-last_synced')
                ):
                    TaskStoreStatistic.objects.create(
                        store=store,
                        measure=TaskStoreStatistic.MEASURE_SIZE,
                        value=store.get_repository_size(),
                        run_id=run_id,
                    )
                    bar.update(idx)
        elif subcommand == 'gc_large_repos':
            for store in TaskStore.objects.order_by('-last_synced'):
                try:
                    last_size_measurement = store.statistics.filter(
                        measure=TaskStoreStatistic.MEASURE_SIZE
                    ).latest('created')
                except TaskStoreStatistic.DoesNotExist:
                    continue
                if last_size_measurement.value > squash_size:
                    print("> Squashing {store}...".format(store=store))
                    try:
                        store.squash()
                        store.gc()
                        final_size = store.get_repository_size()
                        print(
                            ">> {diff} MB recovered".format(
                                diff=int(
                                    (last_size_measurement.value - final_size)
                                    / 1e6
                                )
                            )
                        )
                    except Exception as e:
                        print("> FAILED: %s" % e)
                        traceback.print_exc()
                elif last_size_measurement.value > repack_size:
                    print("> Repacking {store}...".format(store=store))
                    try:
                        store.gc()
                        final_size = store.get_repository_size()
                        print(
                            ">> {diff} MB recovered".format(
                                diff=int(
                                    (last_size_measurement.value - final_size)
                                    / 1e6
                                )
                            )
                        )
                    except Exception as e:
                        print("> FAILED: %s" % e)
                        traceback.print_exc()
        elif subcommand == 'squash':
            store = TaskStore.objects.get(user__username=username)

            starting_size = store.get_repository_size()

            store.squash(force=options['force'])
            store.gc()
            ending_size = store.get_repository_size()

            print(
                ">> {diff} MB recovered".format(
                    diff=int(
                        (starting_size - ending_size)
                        / 1e6
                    )
                )
            )
        elif subcommand == 'delete_old_accounts':
            now = datetime.datetime.now()
            min_action_recency = now - datetime.timedelta(days=370)
            for store in TaskStore.objects.filter(
                last_synced__lt=min_action_recency,
                user__last_login__lt=min_action_recency
            ).distinct('pk').objects.order_by('-last_synced'):
                print('> %s' % store.path)
                print('>> %s' % (now - store.last_synced).days)
                print('>> %s' % (now - store.user.last_login).days)
<commit_msg>Properly access model manager.<commit_after>from __future__ import print_function, unicode_literals

import datetime
import json
import traceback

import progressbar

from django.contrib.auth.models import User
from django.core.management.base import BaseCommand
from django.db.models import Q

from inthe_am.taskmanager.models import TaskStore, TaskStoreStatistic
from inthe_am.taskmanager.lock import (
    get_lock_name_for_store,
    get_lock_redis,
    redis_lock,
)

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument(
            'subcommand',
            nargs=1,
            choices=[
                'list',
                'lock',
                'unlock',
                'search',
                'update_statistics',
                'gc_large_repos',
                'squash',
                'delete_old_accounts',
            ],
            type=str,
        )
        parser.add_argument(
            'username',
            nargs='?',
            type=str
        )
        parser.add_argument(
            '--minutes',
            type=int,
            default=5,
        )
        parser.add_argument(
            '--force',
            action='store_true',
            default=False,
        )
        parser.add_argument(
            '--repack-size',
            type=int,
            default=int(5e7)
        )
        parser.add_argument(
            '--squash-size',
            type=int,
            default=int(1e8)
        )

    def handle(self, *args, **options):
        subcommand = options['subcommand'][0]
        username = options['username']
        minutes = options['minutes']
        repack_size = options['repack_size']
        squash_size = options['squash_size']

        if subcommand == 'lock':
            store = TaskStore.objects.get(user__username=username)
            store.set_lock_state(lock=True, seconds=minutes*60)
            print('{} locked'.format(store))
        elif subcommand == 'unlock':
            store = TaskStore.objects.get(user__username=username)
            store.set_lock_state(lock=False)
            print('{} unlocked'.format(store))
        elif subcommand == 'search':
            users = User.objects.filter(
                Q(email__contains=username) |
                Q(username__contains=username) |
                Q(first_name__contains=username) |
                Q(last_name__contains=username)
            )
            for user in users:
                print(user.username)
        elif subcommand == 'list':
            redis = get_lock_redis()
            for key in redis.keys('*.lock'):
                value = datetime.datetime.fromtimestamp(
                    int(float(redis.get(key)))
                )
                if value > datetime.datetime.utcnow():
                    print('{}: {}'.format(key, value))
        elif subcommand == 'update_statistics':
            run_id = 'update_statistics_{date}'.format(
                date=datetime.datetime.now().strftime('%Y%m%dT%H%M%SZ')
            )
            print("Run ID: {}".format(run_id))
            with progressbar.ProgressBar(
                max_value=TaskStore.objects.count(),
                widgets=[
                    ' [', progressbar.Timer(), '] ',
                    progressbar.Bar(),
                    ' (', progressbar.ETA(), ') ',
                ]
            ) as bar:
                for idx, store in enumerate(
                    TaskStore.objects.order_by('-last_synced')
                ):
                    TaskStoreStatistic.objects.create(
                        store=store,
                        measure=TaskStoreStatistic.MEASURE_SIZE,
                        value=store.get_repository_size(),
                        run_id=run_id,
                    )
                    bar.update(idx)
        elif subcommand == 'gc_large_repos':
            for store in TaskStore.objects.order_by('-last_synced'):
                try:
                    last_size_measurement = store.statistics.filter(
                        measure=TaskStoreStatistic.MEASURE_SIZE
                    ).latest('created')
                except TaskStoreStatistic.DoesNotExist:
                    continue
                if last_size_measurement.value > squash_size:
                    print("> Squashing {store}...".format(store=store))
                    try:
                        store.squash()
                        store.gc()
                        final_size = store.get_repository_size()
                        print(
                            ">> {diff} MB recovered".format(
                                diff=int(
                                    (last_size_measurement.value - final_size)
                                    / 1e6
                                )
                            )
                        )
                    except Exception as e:
                        print("> FAILED: %s" % e)
                        traceback.print_exc()
                elif last_size_measurement.value > repack_size:
                    print("> Repacking {store}...".format(store=store))
                    try:
                        store.gc()
                        final_size = store.get_repository_size()
                        print(
                            ">> {diff} MB recovered".format(
                                diff=int(
                                    (last_size_measurement.value - final_size)
                                    / 1e6
                                )
                            )
                        )
                    except Exception as e:
                        print("> FAILED: %s" % e)
                        traceback.print_exc()
        elif subcommand == 'squash':
            store = TaskStore.objects.get(user__username=username)

            starting_size = store.get_repository_size()

            store.squash(force=options['force'])
            store.gc()
            ending_size = store.get_repository_size()

            print(
                ">> {diff} MB recovered".format(
                    diff=int(
                        (starting_size - ending_size)
                        / 1e6
                    )
                )
            )
        elif subcommand == 'delete_old_accounts':
            now = datetime.datetime.now()
            min_action_recency = now - datetime.timedelta(days=370)
            for store in TaskStore.objects.filter(
                last_synced__lt=min_action_recency,
                user__last_login__lt=min_action_recency
            ).distinct('pk').order_by('-last_synced'):
                print('> %s' % store.path)
                print('>> %s' % (now - store.last_synced).days)
                print('>> %s' % (now - store.user.last_login).days)
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
A module of classes useful for defining object models that are serialized
and deserialized via JSON.

One specific use case would be when using a REST client. In python,
dicts and lists are easy to use, and the stock json library is quite good
and creating and using them. However, if you want to define actual object
Models, possibly with some logic in them, it becomes cumbersome having to
insert each property on deserialization and extracting only set properties
on serialization.

These classes attempt to solve this problem.

Sample Usage:

class Model1(jsonmodels.JsonModel):
    # These are simple types (like strings and ints) that JSON can already handle ok.
    Field1 = jsonmodels.JsonProperty('Field1')  # You can use the same name
    Field2 = jsonmodels.JsonProperty('field_2') # Or a different name

class Model2(jsonmodels.JsonModel):
    # If you need a nested Model:
    Nested = jsonmodels.JsonModelProperty('Nested', Model1)

    # Or even a list of nested Models..
    ListOfNested = jsonmodels.JsonModelListProperty('Nested2', Model1)

some_json_thingy = .... # A dict object returned from something like json.loads(...)

# Assuming the dict in some_json_thingy represents a Model2, you can...
model = Model2.from_json_entity(some_json_thing)

# Only the properties it recognizes will be populated and unkonwn values will be ignored.

print model.Nested.Field1 # Assuming it was there, this will print the Field1 of the Nested model.

# Go ahead and use the setter properties
model.ListOfNested = [Model2(), Model2(Nested=Model1())]

# And you can send it off via JSON again!

json.dumps(model.to_json_entity()) # A string fit for whatever

'''
import inspect
import logging

logger = logging.getLogger(__name__)

class JsonProperty(property):
    '''
    Base class implementation for declaring any property in a JsonModel.

    Some subclasses will be used to implement more intelligent things (like nesting)
    or complex types
    '''

    def __init__(self, json_name):
        '''
        json_name: What name should be used when dealing with JSON entities?
            Allows the implementation to use a different name if the default is not desirable
        '''
        property.__init__(self, self.__get, self.__set)
        self.__name = json_name

    @property
    def name(self):
        '''
        The json property name
        '''
        return self.__name

    @property
    def default(self):
        '''
        What should the default value be (before a set happens)?
        '''
        return None

    def __get(self, obj):
        '''
        Installed to the parent property class as the getter

        the 'obj' here is the instance of the JsonModel
        '''
        return obj.json_properties.get(self.name)
    def __set(self, obj, value):
        '''
        Installed to the parent property class as the setter

        the 'obj' here is the instance of the JsonModel
        '''
        obj.json_properties[self.name] = value

    def to_json_entity(self, value):
        '''
        Default implementation. Simple types can be used as-is
        '''
        return value

    def from_json_entity(self, value):
        '''
        Default implementation. Simple types can be used as-is
        '''
        return value

class JsonModelProperty(JsonProperty):
    '''
    A more complex version of JsonProperty that handles a JsonModel as the property type (nesting)
    '''
    def __init__(self, name, model_class):
        super(JsonModelProperty, self).__init__(name)
        self.__model_class = model_class

    def to_json_entity(self, value):
        '''
        Tell the model to turn itself to a json object
        '''
        return value.to_json_entity()

    def from_json_entity(self, value):
        '''
        Use the class' method to convert the json object into an instance of itself
        '''
        return self.__model_class.from_json_entity(value)

class JsonModelListProperty(JsonProperty):
    '''
    A more complex version of JsonProperty that handles a list of JsonModel as the property type
    '''
    def __init__(self, name, model_class):
        super(JsonModelListProperty, self).__init__(name)
        self.__model_class = model_class

    def to_json_entity(self, value):
        '''
        Convert to a list of json objects
        '''
        if value is None:
            return None
        return [v.to_json_entity() for v in value]

    def from_json_entity(self, value):
        '''
        Convert from a list of json objects into model instances
        '''
        if value is None:
            return None
        return [self.__model_class.from_json_entity(v) for v in value]

class JsonModelDictProperty(JsonProperty):
    '''
    A more complex version of JsonProperty that handles a dictionary.

    The assumption is that only simple types are used as keys since the hashing of objects is not
    trivial to define over an API. So, only the 'type' of the value need be specified.

    The assumption is also that the key values can be used as-is
    '''
    def __init__(self, name, model_class):
        super(JsonModelDictProperty, self).__init__(name)
        self.__model_class = model_class

    def to_json_entity(self, value):
        '''
        Convert from a dict of key -> models, to dict -> json entities
        '''
        if value is None:
            return None

        result = {}
        for k, v in value.iteritems():
            # v is of type self.__model_class
            assert isinstance(v, self.__model_class)
            obj = v.to_json_entity()
            result[k] = obj
        return result

    def from_json_entity(self, value):
        '''
        Convert from a dict of json objects into key -> model instances
        '''
        if value is None:
            return None
        result = {}
        for k, v in value.iteritems():
            result[k] = self.__model_class.from_json_entity(v)
        return result

class JsonModel(object):
    '''
    Base class for any model definition that will need to be (de)serialized via JSON.

    Implementations will look like:

    class MyModel(JsonModel):

        Field1 = JsonProperty('Field1')
        Field2 = JsonModelProperty('Field2', SomeOtherModel)

    This defines a simple model with a couple fields that will understand how to
    prepare for json via to_json_entity and how to create from json via from_json_entity
    '''

    def __init__(self, **property_initializers):
        '''
        property_initializers: keyword's to set as initial values for the json_properties
        '''
        self.__props = {}
        for prop, value in property_initializers.iteritems():
            setattr(self, prop, value)

    @property
    def json_properties(self):
        return self.__props

    def to_json_entity(self):
        '''
        Return this instance as a dictionary ready for JSON conversion

        Does *not* serialize to JSON at this level. The intent is to convert this instance into an
        object ready for json consumption but because of nesting it needs to happen at the top level.
        '''
        result = {}

        for name, prop in inspect.getmembers(self.__class__, lambda x: isinstance(x, JsonProperty)):
            if not self.__props.has_key(name):
                continue
            result[name] = prop.to_json_entity(self.__props[name])
        return result

    @classmethod
    def from_json_entity(cls, obj):
        '''
        Create a new instance of this class (or subclass) based on the json object 'obj'.
        '''
        if obj is None:
            return None

        assert isinstance(obj, dict)

        instance = cls()
        # For each property defined, see if there is a value in obj
        for name, prop in inspect.getmembers(cls, lambda x: isinstance(x, JsonProperty)):
            if not obj.has_key(name):
                continue
            instance.json_properties[prop.name] = prop.from_json_entity(obj[name])
        return instance

<commit_msg>Now allow the property name and the field name to not match<commit_after>#!/usr/bin/env python

'''
A module of classes useful for defining object models that are serialized
and deserialized via JSON.

One specific use case would be when using a REST client. In python,
dicts and lists are easy to use, and the stock json library is quite good
and creating and using them. However, if you want to define actual object
Models, possibly with some logic in them, it becomes cumbersome having to
insert each property on deserialization and extracting only set properties
on serialization.

These classes attempt to solve this problem.

Sample Usage:

class Model1(jsonmodels.JsonModel):
    # These are simple types (like strings and ints) that JSON can already handle ok.
    Field1 = jsonmodels.JsonProperty('Field1')  # You can use the same name
    Field2 = jsonmodels.JsonProperty('field_2') # Or a different name

class Model2(jsonmodels.JsonModel):
    # If you need a nested Model:
    Nested = jsonmodels.JsonModelProperty('Nested', Model1)

    # Or even a list of nested Models..
    ListOfNested = jsonmodels.JsonModelListProperty('Nested2', Model1)

some_json_thingy = .... # A dict object returned from something like json.loads(...)

# Assuming the dict in some_json_thingy represents a Model2, you can...
model = Model2.from_json_entity(some_json_thing)

# Only the properties it recognizes will be populated and unkonwn values will be ignored.

print model.Nested.Field1 # Assuming it was there, this will print the Field1 of the Nested model.

# Go ahead and use the setter properties
model.ListOfNested = [Model2(), Model2(Nested=Model1())]

# And you can send it off via JSON again!

json.dumps(model.to_json_entity()) # A string fit for whatever

'''
import inspect
import logging

logger = logging.getLogger(__name__)

class JsonProperty(property):
    '''
    Base class implementation for declaring any property in a JsonModel.

    Some subclasses will be used to implement more intelligent things (like nesting)
    or complex types
    '''

    def __init__(self, json_name):
        '''
        json_name: What name should be used when dealing with JSON entities?
            Allows the implementation to use a different name if the default is not desirable
        '''
        property.__init__(self, self.__get, self.__set)
        self.__name = json_name

    @property
    def name(self):
        '''
        The json property name
        '''
        return self.__name

    @property
    def default(self):
        '''
        What should the default value be (before a set happens)?
        '''
        return None

    def __get(self, obj):
        '''
        Installed to the parent property class as the getter

        the 'obj' here is the instance of the JsonModel
        '''
        return obj.json_properties.get(self.name)
    def __set(self, obj, value):
        '''
        Installed to the parent property class as the setter

        the 'obj' here is the instance of the JsonModel
        '''
        obj.json_properties[self.name] = value

    def to_json_entity(self, value):
        '''
        Default implementation. Simple types can be used as-is
        '''
        return value

    def from_json_entity(self, value):
        '''
        Default implementation. Simple types can be used as-is
        '''
        return value

class JsonModelProperty(JsonProperty):
    '''
    A more complex version of JsonProperty that handles a JsonModel as the property type (nesting)
    '''
    def __init__(self, name, model_class):
        super(JsonModelProperty, self).__init__(name)
        self.__model_class = model_class

    def to_json_entity(self, value):
        '''
        Tell the model to turn itself to a json object
        '''
        return value.to_json_entity()

    def from_json_entity(self, value):
        '''
        Use the class' method to convert the json object into an instance of itself
        '''
        return self.__model_class.from_json_entity(value)

class JsonModelListProperty(JsonProperty):
    '''
    A more complex version of JsonProperty that handles a list of JsonModel as the property type
    '''
    def __init__(self, name, model_class):
        super(JsonModelListProperty, self).__init__(name)
        self.__model_class = model_class

    def to_json_entity(self, value):
        '''
        Convert to a list of json objects
        '''
        if value is None:
            return None
        return [v.to_json_entity() for v in value]

    def from_json_entity(self, value):
        '''
        Convert from a list of json objects into model instances
        '''
        if value is None:
            return None
        return [self.__model_class.from_json_entity(v) for v in value]

class JsonModelDictProperty(JsonProperty):
    '''
    A more complex version of JsonProperty that handles a dictionary.

    The assumption is that only simple types are used as keys since the hashing of objects is not
    trivial to define over an API. So, only the 'type' of the value need be specified.

    The assumption is also that the key values can be used as-is
    '''
    def __init__(self, name, model_class):
        super(JsonModelDictProperty, self).__init__(name)
        self.__model_class = model_class

    def to_json_entity(self, value):
        '''
        Convert from a dict of key -> models, to dict -> json entities
        '''
        if value is None:
            return None

        result = {}
        for k, v in value.iteritems():
            # v is of type self.__model_class
            assert isinstance(v, self.__model_class)
            obj = v.to_json_entity()
            result[k] = obj
        return result

    def from_json_entity(self, value):
        '''
        Convert from a dict of json objects into key -> model instances
        '''
        if value is None:
            return None
        result = {}
        for k, v in value.iteritems():
            result[k] = self.__model_class.from_json_entity(v)
        return result

class JsonModel(object):
    '''
    Base class for any model definition that will need to be (de)serialized via JSON.

    Implementations will look like:

    class MyModel(JsonModel):

        Field1 = JsonProperty('Field1')
        Field2 = JsonModelProperty('Field2', SomeOtherModel)

    This defines a simple model with a couple fields that will understand how to
    prepare for json via to_json_entity and how to create from json via from_json_entity
    '''

    def __init__(self, **property_initializers):
        '''
        property_initializers: keyword's to set as initial values for the json_properties
        '''
        self.__props = {}
        for prop, value in property_initializers.iteritems():
            setattr(self, prop, value)

    @property
    def json_properties(self):
        return self.__props

    def to_json_entity(self):
        '''
        Return this instance as a dictionary ready for JSON conversion

        Does *not* serialize to JSON at this level. The intent is to convert this instance into an
        object ready for json consumption but because of nesting it needs to happen at the top level.
        '''
        result = {}

        for name, prop in inspect.getmembers(self.__class__, lambda x: isinstance(x, JsonProperty)):
            if not self.__props.has_key(name):
                continue
            result[prop.name] = prop.to_json_entity(self.__props[name])
        return result

    @classmethod
    def from_json_entity(cls, obj):
        '''
        Create a new instance of this class (or subclass) based on the json object 'obj'.
        '''
        if obj is None:
            return None

        instance = cls()
        # For each property defined, see if there is a value in obj
        for name, prop in inspect.getmembers(cls, lambda x: isinstance(x, JsonProperty)):
            if not obj.has_key(prop.name):
                continue
            instance.json_properties[name] = prop.from_json_entity(obj[prop.name])
        return instance

<|endoftext|>


<commit_before>#! /bin/python
# -*- coding: utf-8 -*-

import os, json, time, hashlib

class memory():
    def __init__(self, cache_path=None, cache_name=None, cache_extension="cache", expired=3600):
        extension = cache_extension.lstrip(".") or "cache"
        self.extension = "." + extension

        self.expired = expired
        self.cache_name = cache_name or hashlib.md5(".").hexdigest()

        if cache_path and os.path.exists(cache_path) == True:
            self.cache_path = cache_path
        else:
            self.cache_path = os.path.abspath("./cache")

    def __get_cache(self):
        path = self.cache_path+"/"+self.cache_name+self.extension
        try:
            fp = open(path, "r+")
            data = fp.read()
            return (fp, data)
        except IOError as e:
            fp = open(path, "w+")
            data = "{}"
            return (fp, data)

    def __set_cache(self, data):
        path = self.cache_path+"/"+self.cache_name+self.extension
        fp = open(path, "w+")
        fp.write(data)

    def get_hash(self):
        return self.cache_name

    def set_exipired(self, expired=3600):
        self.expired = expired or self.expired
        return self.expired

    def set_cache_name(self, name):
        self.cache_name = hashlib.md5(name).hexdigest()
        return self.cache_name

    def get_cache_name(self):
        return self.cache_name

    def set_cache_dir(self, dir):
        if os.path.exists(dir) == True:
            self.cache_path = dir
        return self.cache_path

    def get(self, key):
        fp, data = self.__get_cache()
        data = json.loads(data)
        if data.has_key(key):
            item = data[key]
            if item["time"] + item["expired"] < int(time.time()):
                '''data expired'''
                del data[key]
                self.__set_cache(json.dumps(data))
                return None
            else:
                return item
        return None

    def set(self, key, value, expired=None):
        expired = expired or self.expired
        fp, data = self.__get_cache()

        try:
            data = json.loads(data)
        except:
            data = {}

        store_value = {
            "time": int(time.time()),
            "expired": expired,
            "data": value
        }
        data[key] = store_value
        m = json.dumps(data)
        self.__set_cache(m.strip(""))

if __name__ == "__main__":
    memory = memory(cache_extension=".cache")
    memory.set("key", "value")
    print memory.get("123")
<commit_msg>Update memory<commit_after>#! /bin/python
# -*- coding: utf-8 -*-

import os, json, time, hashlib

class memory():
    def __init__(self, cache_path=None, cache_name=None, cache_extension="cache", expired=3600):
        extension = cache_extension.lstrip(".") or "cache"
        self.extension = "." + extension

        self.expired = expired
        self.cache_name = cache_name or hashlib.md5(".").hexdigest()

        if cache_path and os.path.exists(cache_path) == True:
            self.cache_path = cache_path
        else:
            self.cache_path = os.path.abspath("./cache")

    def __get_cache(self):
        path = self.cache_path+"/"+self.cache_name+self.extension
        try:
            fp = open(path, "r+")
            data = fp.read()
            return (fp, data)
        except IOError as e:
            fp = open(path, "w+")
            data = "{}"
            return (fp, data)

    def __set_cache(self, data):
        path = self.cache_path+"/"+self.cache_name+self.extension
        fp = open(path, "w+")
        fp.write(data)

    def set_exipired(self, expired=3600):
        self.expired = expired or self.expired
        return self.expired

    def get_cache_name(self):
        return self.cache_name

    def set_cache_name(self, name):
        self.cache_name = hashlib.md5(name).hexdigest()
        return self.cache_name

    def get_cache_name(self):
        return self.cache_name

    def set_extension(self, extension):
        extension = extension.lstrip(".") or "cache"
        self.extension = "." + extension

    def get_extension(self):
        return self.extension

    def set_cache_dir(self, dir):
        if os.path.exists(dir) == True:
            self.cache_path = dir
        return self.cache_path

    def clear_expired(self):
        fp, data = self.__get_cache()
        data = json.loads(data)
        for key, value in data.items():
            if value["time"] + value["expired"] < int(time.time()):
                del data[key]
        self.__set_cache(json.dumps(data))

    def get(self, key):
        fp, data = self.__get_cache()
        data = json.loads(data)
        if data.has_key(key):
            item = data[key]
            if item["time"] + item["expired"] < int(time.time()):
                '''data expired'''
                del data[key]
                self.__set_cache(json.dumps(data))
                return None
            else:
                return item
        return None

    def set(self, key, value, expired=None):
        expired = expired or self.expired
        fp, data = self.__get_cache()

        try:
            data = json.loads(data)
        except:
            data = {}

        store_value = {
            "time": int(time.time()),
            "expired": expired,
            "data": value
        }
        data[key] = store_value
        m = json.dumps(data)
        self.__set_cache(m.strip(""))

if __name__ == "__main__":
    memory = memory(cache_extension=".cache")
    memory.set("key", "value", 1)
    memory.clear_expired()
    print memory.get("key")
<|endoftext|>