<commit_before>import os
import tempfile

from attest import Assert, Tests, TestBase, test

import pyhis

TWDB_WSDL_URL = 'http://his.crwr.utexas.edu/TWDB_Sondes/cuahsi_1_0.asmx?WSDL'

#TWDB_WSDL_URL = 'file://' + os.path.abspath('./twdb_wsdl.xml')
USGS_WSDL_URL = 'file://' + os.path.abspath('./usgs_wsdl.xml')

TEST_CACHE_DATABASE_PATH = os.path.join(tempfile.gettempdir(),
                                        'pyhis_test_cache.db')

TEST_SHAPEFILE_PATH = 'shapefiles/pyhis_test.shp'

class TWDBTestBase(TestBase):
    """Base class for using TWDB dataset as a service"""

    @test
    def get_site(self):
        site = self.service.get_site('TWDBSondes', 'Aransas95_D1')
        assert Assert(site.latitude) == 28.06666667
        assert Assert(site.longitude) == -97.20333333
        assert Assert(site.name) == 'Upper Copano Bay'
        assert Assert(site.service.url) == TWDB_WSDL_URL

    @test
    def check_sites(self):
        assert Assert(len(self.service.get_all_sites())) == 74
        assert Assert(len(self.service.sites)) == 74

    @test
    def check_variables(self):
        assert Assert(len(self.service.sites['Aransas95_D1'].timeseries)) == 5
        assert Assert(len(self.service.sites['Christmas92_D1'].timeseries)) == 5

    @test
    def check_dataframe(self):
        df = self.service.sites['Aransas95_D1'].dataframe
        assert Assert(len(df['SAL001'])) == 725
        df = self.service.sites['Christmas92_D1'].dataframe
        assert Assert(len(df['SAL001'])) == 1044

class TWDBFreshCacheTests(TWDBTestBase):
    """Run tests with cache backend"""

    def __context__(self):
        if os.path.exists(TEST_CACHE_DATABASE_PATH):
            os.remove(TEST_CACHE_DATABASE_PATH)
        pyhis.cache.init_cache(TEST_CACHE_DATABASE_PATH)

        self.service = pyhis.Service(
            TWDB_WSDL_URL,
            use_cache=True)
        yield
        del self.service

class TWDBCacheTests(TWDBTestBase):
    """Run tests with cache backend"""

    def __context__(self):
        pyhis.cache.init_cache(TEST_CACHE_DATABASE_PATH)
        self.service = pyhis.Service(
            TWDB_WSDL_URL,
            use_cache=True)
        yield
        del self.service

    @test
    def get_series_and_quantity_check_for_updates(self):
        """
        tests that cache.get_timeseries_dict_for_site()
        actually checks for updates when check_for_updates=True
        """
        from pyhis import cache
        cache_dict = cache._cache

        # query timseries so that it gets set up in the cache
        cached_ts = cache.CacheTimeSeries(
            self.service.sites['Aransas95_D1'].timeseries['CON001'])

        cached_ts.values = cached_ts.values[:4]
        cache.db_session.commit()

        # test that the db has been altered
        del cache._cache['timeseries'][(TWDB_WSDL_URL, 'TWDBSondes',
                                        'Aransas95_D1', 'CON001')]

        # query timseries so that it gets set up in the cache
        cached_ts = cache.CacheTimeSeries(
            self.service.sites['Aransas95_D1'].timeseries['CON001'])
        assert Assert(cached_ts.values.count()) == 4

        # test that check_for_updates sucessfully updates the
        # timeseries values
        del cache._cache['timeseries'][(TWDB_WSDL_URL, 'TWDBSondes',
                                        'Aransas95_D1', 'CON001')]

        test_ts = self.service.sites['Aransas95_D1'].timeseries['CON001']
        series, quantity = cache.get_series_and_quantity_for_timeseries(
            test_ts, check_for_updates=True)
        assert Assert(len(series)) == 725

class TWDBNoCacheTests(TWDBTestBase):
    """Run tests without cache backend"""

    def __context__(self):
        self.service = pyhis.Service(
            TWDB_WSDL_URL,
            use_cache=False)
        yield
        del self.service

class HISCentralTests(TestBase):
    @test
    def all_services(self):
        assert len(pyhis.his_central.services()) > 70

class ShapeTest(TestBase):
    def __context__(self):
        if os.path.exists(TEST_CACHE_DATABASE_PATH):
            os.remove(TEST_CACHE_DATABASE_PATH)
        pyhis.cache.init_cache(TEST_CACHE_DATABASE_PATH)

        self.service = pyhis.Service(
            TWDB_WSDL_URL,
            use_cache=True)
        yield
        del self.service

    @test
    def get_sites_within_shapefile(self):
        sites = self.service.get_sites_within_shapefile(TEST_SHAPEFILE_PATH)
        assert len(sites) == 13

if __name__ == '__main__':
    # suite = Tests([TWDBFreshCacheTests(),
    #                TWDBCacheTests(),
    #                TWDBNoCacheTests()])
    suite = Tests([
        # TWDBFreshCacheTests(),
        # TWDBCacheTests(),
        # TWDBNoCacheTests(),
        # HISCentralTests(),
        ShapeTest()
        ])

    suite.run()
<commit_msg>enable all tests by default in test.py<commit_after>import os
import tempfile

from attest import Assert, Tests, TestBase, test

import pyhis

TWDB_WSDL_URL = 'http://his.crwr.utexas.edu/TWDB_Sondes/cuahsi_1_0.asmx?WSDL'

#TWDB_WSDL_URL = 'file://' + os.path.abspath('./twdb_wsdl.xml')
USGS_WSDL_URL = 'file://' + os.path.abspath('./usgs_wsdl.xml')

TEST_CACHE_DATABASE_PATH = os.path.join(tempfile.gettempdir(),
                                        'pyhis_test_cache.db')

TEST_SHAPEFILE_PATH = 'shapefiles/pyhis_test.shp'

class TWDBTestBase(TestBase):
    """Base class for using TWDB dataset as a service"""

    @test
    def get_site(self):
        site = self.service.get_site('TWDBSondes', 'Aransas95_D1')
        assert Assert(site.latitude) == 28.06666667
        assert Assert(site.longitude) == -97.20333333
        assert Assert(site.name) == 'Upper Copano Bay'
        assert Assert(site.service.url) == TWDB_WSDL_URL

    @test
    def check_sites(self):
        assert Assert(len(self.service.get_all_sites())) == 74
        assert Assert(len(self.service.sites)) == 74

    @test
    def check_variables(self):
        assert Assert(len(self.service.sites['Aransas95_D1'].timeseries)) == 5
        assert Assert(len(self.service.sites['Christmas92_D1'].timeseries)) == 5

    @test
    def check_dataframe(self):
        df = self.service.sites['Aransas95_D1'].dataframe
        assert Assert(len(df['SAL001'])) == 725
        df = self.service.sites['Christmas92_D1'].dataframe
        assert Assert(len(df['SAL001'])) == 1044

class TWDBFreshCacheTests(TWDBTestBase):
    """Run tests with cache backend"""

    def __context__(self):
        if os.path.exists(TEST_CACHE_DATABASE_PATH):
            os.remove(TEST_CACHE_DATABASE_PATH)
        pyhis.cache.init_cache(TEST_CACHE_DATABASE_PATH)

        self.service = pyhis.Service(
            TWDB_WSDL_URL,
            use_cache=True)
        yield
        del self.service

class TWDBCacheTests(TWDBTestBase):
    """Run tests with cache backend"""

    def __context__(self):
        pyhis.cache.init_cache(TEST_CACHE_DATABASE_PATH)
        self.service = pyhis.Service(
            TWDB_WSDL_URL,
            use_cache=True)
        yield
        del self.service

    @test
    def get_series_and_quantity_check_for_updates(self):
        """
        tests that cache.get_timeseries_dict_for_site()
        actually checks for updates when check_for_updates=True
        """
        from pyhis import cache
        cache_dict = cache._cache

        # query timseries so that it gets set up in the cache
        cached_ts = cache.CacheTimeSeries(
            self.service.sites['Aransas95_D1'].timeseries['CON001'])

        cached_ts.values = cached_ts.values[:4]
        cache.db_session.commit()

        # test that the db has been altered
        del cache._cache['timeseries'][(TWDB_WSDL_URL, 'TWDBSondes',
                                        'Aransas95_D1', 'CON001')]

        # query timseries so that it gets set up in the cache
        cached_ts = cache.CacheTimeSeries(
            self.service.sites['Aransas95_D1'].timeseries['CON001'])
        assert Assert(cached_ts.values.count()) == 4

        # test that check_for_updates sucessfully updates the
        # timeseries values
        del cache._cache['timeseries'][(TWDB_WSDL_URL, 'TWDBSondes',
                                        'Aransas95_D1', 'CON001')]

        test_ts = self.service.sites['Aransas95_D1'].timeseries['CON001']
        series, quantity = cache.get_series_and_quantity_for_timeseries(
            test_ts, check_for_updates=True)
        assert Assert(len(series)) == 725

class TWDBNoCacheTests(TWDBTestBase):
    """Run tests without cache backend"""

    def __context__(self):
        self.service = pyhis.Service(
            TWDB_WSDL_URL,
            use_cache=False)
        yield
        del self.service

class HISCentralTests(TestBase):
    @test
    def all_services(self):
        assert len(pyhis.his_central.services()) > 70

class ShapeTests(TestBase):
    def __context__(self):
        if os.path.exists(TEST_CACHE_DATABASE_PATH):
            os.remove(TEST_CACHE_DATABASE_PATH)
        pyhis.cache.init_cache(TEST_CACHE_DATABASE_PATH)

        self.service = pyhis.Service(
            TWDB_WSDL_URL,
            use_cache=True)
        yield
        del self.service

    @test
    def get_sites_within_shapefile(self):
        sites = self.service.get_sites_within_shapefile(TEST_SHAPEFILE_PATH)
        assert len(sites) == 13

if __name__ == '__main__':
    # suite = Tests([TWDBFreshCacheTests(),
    #                TWDBCacheTests(),
    #                TWDBNoCacheTests()])
    suite = Tests([
        TWDBFreshCacheTests(),
        TWDBCacheTests(),
        TWDBNoCacheTests(),
        HISCentralTests(),
        ShapeTests()
        ])

    suite.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
import matplotlib2tikz
import testfunctions

import os
import tempfile
from importlib import import_module
import hashlib
import subprocess
from PIL import Image
import imagehash
from matplotlib import pyplot as pp

def test_generator():
    for name in testfunctions.__all__:
        print(name)
        test = import_module('testfunctions.' + name)
        yield check_hash, test, name

def check_hash(test, name):
    # import the test
    test.plot()
    # convert to tikz file
    handle, tmp_base = tempfile.mkstemp(prefix=name)
    tikz_file = tmp_base + '_tikz.tex'
    matplotlib2tikz.save(
        tikz_file,
        figurewidth='7.5cm',
        show_info=False
        )

    # save reference figure
    mpl_reference = tmp_base + '_reference.pdf'
    pp.savefig(mpl_reference)

    # create a latex wrapper for the tikz
    wrapper = '''\\documentclass{standalone}
\\usepackage{pgfplots}
\\usepgfplotslibrary{groupplots}
\\pgfplotsset{compat=newest}
\\begin{document}
\\input{%s}
\\end{document}''' % tikz_file
    tex_file = tmp_base + '.tex'
    with open(tex_file, 'w') as f:
        f.write(wrapper)

    # change into the directory of the TeX file
    os.chdir(os.path.dirname(tex_file))

    # compile the output to pdf
    tex_out = subprocess.check_output(
        # use pdflatex for now until travis features a more modern lualatex
        ['pdflatex', '--interaction=nonstopmode', tex_file],
        stderr=subprocess.STDOUT
        )
    pdf_file = tmp_base + '.pdf'

    # Convert PDF to PNG.
    subprocess.check_call(
        ['pdftoppm', '-rx', '600', '-ry', '600', '-png', pdf_file, tmp_base],
        stdout=FNULL,
        stderr=subprocess.STDOUT
        )
    png_file = tmp_base + '-1.png'

    # compute the phash of the PNG
    phash = imagehash.phash(Image.open(png_file)).__str__()

    if test.phash != phash:
        # Compute the Hamming distance between the two 64-bit numbers
        hamming_dist = bin(int(phash, 16) ^ int(test.phash, 16)).count('1')
        print('Output file: %s' % png_file)
        print('computed pHash:  %s' % phash)
        print('reference pHash: %s' % test.phash)
        print(
            'Hamming distance: %s (out of %s)' %
            (hamming_dist, 4 * len(phash))
            )
        print('pdflatex output:')
        print(tex_output)
        if 'DISPLAY' not in os.environ:
            # upload to chunk.io if we're on a headless client
            out = subprocess.check_output(
                ['curl', '-sT', mpl_reference, 'chunk.io'],
                stderr=subprocess.STDOUT
                )
            print('Uploaded reference matplotlib PDF file to %s' % out)

            out = subprocess.check_output(
                ['curl', '-sT', tikz_file, 'chunk.io'],
                stderr=subprocess.STDOUT
                )
            print('Uploaded TikZ file to %s' % out)

            out = subprocess.check_output(
                ['curl', '-sT', pdf_file, 'chunk.io'],
                stderr=subprocess.STDOUT
                )
            print('Uploaded output PDF file to %s' % out)

            out = subprocess.check_output(
                ['curl', '-sT', png_file, 'chunk.io'],
                stderr=subprocess.STDOUT
                )
            print('Uploaded output PNG file to %s' % out)

    assert test.phash == phash
<commit_msg>test: fix oversight<commit_after># -*- coding: utf-8 -*-
#
import matplotlib2tikz
import testfunctions

import os
import tempfile
from importlib import import_module
import hashlib
import subprocess
from PIL import Image
import imagehash
from matplotlib import pyplot as pp

def test_generator():
    for name in testfunctions.__all__:
        print(name)
        test = import_module('testfunctions.' + name)
        yield check_hash, test, name

def check_hash(test, name):
    # import the test
    test.plot()
    # convert to tikz file
    handle, tmp_base = tempfile.mkstemp(prefix=name)
    tikz_file = tmp_base + '_tikz.tex'
    matplotlib2tikz.save(
        tikz_file,
        figurewidth='7.5cm',
        show_info=False
        )

    # save reference figure
    mpl_reference = tmp_base + '_reference.pdf'
    pp.savefig(mpl_reference)

    # create a latex wrapper for the tikz
    wrapper = '''\\documentclass{standalone}
\\usepackage{pgfplots}
\\usepgfplotslibrary{groupplots}
\\pgfplotsset{compat=newest}
\\begin{document}
\\input{%s}
\\end{document}''' % tikz_file
    tex_file = tmp_base + '.tex'
    with open(tex_file, 'w') as f:
        f.write(wrapper)

    # change into the directory of the TeX file
    os.chdir(os.path.dirname(tex_file))

    # compile the output to pdf
    tex_out = subprocess.check_output(
        # use pdflatex for now until travis features a more modern lualatex
        ['pdflatex', '--interaction=nonstopmode', tex_file],
        stderr=subprocess.STDOUT
        )
    pdf_file = tmp_base + '.pdf'

    # Convert PDF to PNG.
    ptp_out = subprocess.check_output(
        ['pdftoppm', '-rx', '600', '-ry', '600', '-png', pdf_file, tmp_base],
        stderr=subprocess.STDOUT
        )
    png_file = tmp_base + '-1.png'

    # compute the phash of the PNG
    phash = imagehash.phash(Image.open(png_file)).__str__()

    if test.phash != phash:
        # Compute the Hamming distance between the two 64-bit numbers
        hamming_dist = bin(int(phash, 16) ^ int(test.phash, 16)).count('1')
        print('Output file: %s' % png_file)
        print('computed pHash:  %s' % phash)
        print('reference pHash: %s' % test.phash)
        print(
            'Hamming distance: %s (out of %s)' %
            (hamming_dist, 4 * len(phash))
            )
        print('pdflatex output:')
        print(tex_out)

        print('pdftoppm output:')
        print(ptp_out)

        if 'DISPLAY' not in os.environ:
            # upload to chunk.io if we're on a headless client
            out = subprocess.check_output(
                ['curl', '-sT', mpl_reference, 'chunk.io'],
                stderr=subprocess.STDOUT
                )
            print('Uploaded reference matplotlib PDF file to %s' % out)

            out = subprocess.check_output(
                ['curl', '-sT', tikz_file, 'chunk.io'],
                stderr=subprocess.STDOUT
                )
            print('Uploaded TikZ file to %s' % out)

            out = subprocess.check_output(
                ['curl', '-sT', pdf_file, 'chunk.io'],
                stderr=subprocess.STDOUT
                )
            print('Uploaded output PDF file to %s' % out)

            out = subprocess.check_output(
                ['curl', '-sT', png_file, 'chunk.io'],
                stderr=subprocess.STDOUT
                )
            print('Uploaded output PNG file to %s' % out)

    assert test.phash == phash
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
 Copyright (C) Yadu Nand B <yadudoc1729@gmail.com> - All Rights Reserved
 Unauthorized copying of this file, via any medium is strictly prohibited
 Proprietary and confidential
 Written by Yadu Nand B <yadudoc1729@gmail.com>, September 2015
'''

import os
import logging
import bottle
import requests
import time
from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.table import Table
import boto.dynamodb2 as ddb
import boto.ec2
import boto.sqs
import boto.sns
import boto.ses
from bottle import app, template
from boto.s3.connection import S3Connection
from datetime import datetime
from datetime import date
from dateutil.relativedelta import relativedelta

log_levels = { "DEBUG"   : logging.DEBUG,
               "INFO"    : logging.INFO,
               "WARNING" : logging.WARNING,
               "ERROR"   : logging.ERROR,
               "CRITICAL": logging.CRITICAL
           }

# Returns true if the credentials were updated
def update_creds_from_metadata_server(app):
    #Todo error check for timeout errors from http access
    #TOdo error catch for json decode failure

    if "keys.expiry" in app.config and app.config["keys.expiry"] > (datetime.now() + relativedelta(hours=1)):
        logging.debug("Update creds from metadata cancelled {0} < {1}".format(
            app.config["keys.expiry"],
            datetime.now()))
        return False

    URL  = app.config["metadata.credurl"]
    role = requests.get(URL).content
    URL  = URL + role
    data = requests.get(URL).json()

    app.config["keys.expiry"]     = datetime.strptime(str(data['Expiration']), '%Y-%m-%dT%H:%M:%SZ')
    app.config["keys.key_id"]     = str(data['AccessKeyId'])
    app.config["keys.key_secret"] = str(data['SecretAccessKey'])
    app.config["keys.key_token"]  = str(data['Token'])

    URL  = app.config["metadata.metaserver"]
    data = requests.get(URL).text
    app.config["instance_id"] = str(data)

    URL  = app.config["metadata.metaserver"]
    data = requests.get(URL+"placement/availability-zone/").text
    app.config["region"] = str(data)

    URL  = app.config["metadata.metaidentity"]
    data = requests.get(URL).json()
    app.config["identity"] = data

    if "doReload" in app.config and app.config["doReload"] == True:
        init(app)

    return True

##################################################################
# Annoy human with email
##################################################################
def send_success_mail(data, app):
    sesconn     =  app.config['ses.conn']
    job_id      =  data.get('job_id')
    rec_email   =  data.get('user_email')
    rec_name    =  data.get('username')
    src_email   =  app.config['ses.email_sender']
    url         =  app.config['server.url']

    body = template('./templates/completion_email.tpl',
                    username=rec_name,
                    job_id=job_id,
                    url=url)

    st = sesconn.send_email(src_email,
                            "[JES] Your Job has completed",
                            body,
                            [rec_email])
    print st
    return st

##################################################################
# Send condolences for job failure
##################################################################
def send_failure_mail(data, app):
    sesconn     =  app.config['ses.conn']
    job_id      =  data.get('job_id')
    rec_email   =  data.get('user_email')
    rec_name    =  data.get('username')
    src_email   =  app.config['ses.email_sender']
    url         =  app.config['server.url']

    body = template('./templates/failure_email.tpl',
                    username=rec_name,
                    job_id=job_id,
                    url=url)

    st = sesconn.send_email(src_email,
                            "[JES] Your Job has failed",
                            body,
                            [rec_email])
    print st
    return st

def init(app):

    ec2  = boto.ec2.connect_to_region(app.config["identity"]['region'],
                                      aws_access_key_id=app.config['keys.key_id'],
                                      aws_secret_access_key=app.config['keys.key_secret'],
                                      security_token=app.config['keys.key_token'])

    # Get meta tags
    meta_tags = {}
    for tag in ec2.get_all_tags():
        meta_tags[str(tag.name)] = str(tag.value)

    # Log the metadata tags
    app.config["instance.tags"] = meta_tags
    for k in meta_tags:
        logging.debug("[TAGS] {0} : {1}".format(k, meta_tags[k]))

    sqs  = boto.sqs.connect_to_region(app.config["identity"]['region'],
                                      aws_access_key_id=app.config['keys.key_id'],
                                      aws_secret_access_key=app.config['keys.key_secret'],
                                      security_token=app.config['keys.key_token'])

    sns  = boto.sns.connect_to_region(app.config["identity"]['region'],
                                      aws_access_key_id=app.config['keys.key_id'],
                                      aws_secret_access_key=app.config['keys.key_secret'],
                                      security_token=app.config['keys.key_token'])

    ses  = boto.ses.connect_to_region(app.config["identity"]['region'],
                                      aws_access_key_id=app.config['keys.key_id'],
                                      aws_secret_access_key=app.config['keys.key_secret'],
                                      security_token=app.config['keys.key_token'])

    s3   = S3Connection(aws_access_key_id=app.config['keys.key_id'],
                        aws_secret_access_key=app.config['keys.key_secret'],
                        security_token=app.config['keys.key_token'])

    dyno = Table(app.config["instance.tags"]["DynamoDBTableName"],#app.config['dynamodb.table_name'],
                 schema=[HashKey("job_id")],
                 connection=ddb.connect_to_region(app.config['dynamodb.region'],
                                                  aws_access_key_id=app.config['keys.key_id'],
                                                  aws_secret_access_key=app.config['keys.key_secret'],
                                                  security_token=app.config['keys.key_token']))

    app.config["ec2.conn"]  = ec2
    app.config["sns.conn"]  = sns
    app.config["sqs.conn"]  = sqs
    app.config["ses.conn"]  = ses
    app.config["s3.conn"]   = s3
    app.config["dyno.conn"] = dyno
    app.config["doReload"]  = True

    return app

def connect_to_dynamodb(app):
    stat = update_creds_from_metadata_server(app)

    # If an entry exists for the table and the credentials have
    # not been updated then skip the connection
    if "dynamodb.table" in app.config and not stat:
        return app

    dbconn = ddb.connect_to_region(app.config['dynamodb.region'],
                                   aws_access_key_id=app.config['keys.key_id'],
                                   aws_secret_access_key=app.config['keys.key_secret'],
                                   security_token=app.config['keys.key_token'])

    dyno = Table(app.config["instance.tags"]["DynamoDBTableName"], #app.config['dynamodb.table_name'],
                 schema=[HashKey("CustomerUUID")],
                 connection=dbconn)

    app.config["dynamodb.table"] = dyno
    return app

def load_configs(filename):
    app = bottle.default_app()
    try:
        app.config.load_config(filename)
    except Exception, e:
        logging.error("Exception {0} in load_config".format(e))
        exit(-1)

    logging.debug("Config : \n {0}".format(app.config))

    for keys in app.config:
        if keys.startswith('keys.'):
            print keys
            keyfile = app.config[keys].replace('\"', '')

            logging.debug("Keyfile : {0}".format(keyfile.replace('\"', '')))
            if not os.path.isfile(keyfile):
                print "Key file {0} missing!".format(keyfile)
                logging.error("Key file {0} missing!".format(keyfile))
                exit(-1)
            with open(keyfile, 'r') as kf:
                ks = kf.readlines()
                sp = ks[1].split(',')
                app.config[keys] = kf.read()
                app.config["keys.key_id"] = sp[1]
                app.config["keys.key_secret"] = sp[2]
                app.config["keys.key_token"] = ''
                #print "keys : ", app.config[keys]
    if 'metadata.credurl' in app.config:
        update_creds_from_metadata_server(app)

    init(app)
    return app
<commit_msg>Earlier config_manager code pulled metadata and instance tags from the ec2 connection and not specifically the tags associated with an instance. This resulted in worker nodes not identifying their specific queues. Fixed.<commit_after>#!/usr/bin/env python
'''
 Copyright (C) Yadu Nand B <yadudoc1729@gmail.com> - All Rights Reserved
 Unauthorized copying of this file, via any medium is strictly prohibited
 Proprietary and confidential
 Written by Yadu Nand B <yadudoc1729@gmail.com>, September 2015
'''

import os
import logging
import bottle
import requests
import time
from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.table import Table
import boto.dynamodb2 as ddb
import boto.ec2
import boto.sqs
import boto.sns
import boto.ses
from bottle import app, template
from boto.s3.connection import S3Connection
from datetime import datetime
from datetime import date
from dateutil.relativedelta import relativedelta

log_levels = { "DEBUG"   : logging.DEBUG,
               "INFO"    : logging.INFO,
               "WARNING" : logging.WARNING,
               "ERROR"   : logging.ERROR,
               "CRITICAL": logging.CRITICAL
           }

# Returns true if the credentials were updated
def update_creds_from_metadata_server(app):
    #Todo error check for timeout errors from http access
    #TOdo error catch for json decode failure

    if "keys.expiry" in app.config and app.config["keys.expiry"] > (datetime.now() + relativedelta(hours=1)):
        logging.debug("Update creds from metadata cancelled {0} < {1}".format(
            app.config["keys.expiry"],
            datetime.now()))
        return False

    URL  = app.config["metadata.credurl"]
    role = requests.get(URL).content
    URL  = URL + role
    data = requests.get(URL).json()

    app.config["keys.expiry"]     = datetime.strptime(str(data['Expiration']), '%Y-%m-%dT%H:%M:%SZ')
    app.config["keys.key_id"]     = str(data['AccessKeyId'])
    app.config["keys.key_secret"] = str(data['SecretAccessKey'])
    app.config["keys.key_token"]  = str(data['Token'])

    URL  = app.config["metadata.metaserver"]
    data = requests.get(URL+"instance-id").text
    app.config["instance_id"] = str(data)

    URL  = app.config["metadata.metaserver"]
    data = requests.get(URL+"placement/availability-zone/").text
    app.config["region"] = str(data)

    URL  = app.config["metadata.metaidentity"]
    data = requests.get(URL).json()
    app.config["identity"] = data

    if "doReload" in app.config and app.config["doReload"] == True:
        init(app)

    return True

##################################################################
# Annoy human with email
##################################################################
def send_success_mail(data, app):
    sesconn     =  app.config['ses.conn']
    job_id      =  data.get('job_id')
    rec_email   =  data.get('user_email')
    rec_name    =  data.get('username')
    src_email   =  app.config['ses.email_sender']
    url         =  app.config['server.url']

    body = template('./templates/completion_email.tpl',
                    username=rec_name,
                    job_id=job_id,
                    url=url)

    st = sesconn.send_email(src_email,
                            "[JES] Your Job has completed",
                            body,
                            [rec_email])
    print st
    return st

##################################################################
# Send condolences for job failure
##################################################################
def send_failure_mail(data, app):
    sesconn     =  app.config['ses.conn']
    job_id      =  data.get('job_id')
    rec_email   =  data.get('user_email')
    rec_name    =  data.get('username')
    src_email   =  app.config['ses.email_sender']
    url         =  app.config['server.url']

    body = template('./templates/failure_email.tpl',
                    username=rec_name,
                    job_id=job_id,
                    url=url)

    st = sesconn.send_email(src_email,
                            "[JES] Your Job has failed",
                            body,
                            [rec_email])
    print st
    return st

def init(app):

    ec2  = boto.ec2.connect_to_region(app.config["identity"]['region'],
                                      aws_access_key_id=app.config['keys.key_id'],
                                      aws_secret_access_key=app.config['keys.key_secret'],
                                      security_token=app.config['keys.key_token'])
    #print "instance id : ", app.config["instance_id"]
    # Get meta tags
    reservation = ec2.get_all_instances(instance_ids=app.config["instance_id"])
    meta_tags = {}
    if reservation :
        for tag in reservation[0].instances[0].tags:
            meta_tags[str(tag)] = str(reservation[0].instances[0].tags[tag])
            #print str(tag), str(reservation[0].instances[0].tags[tag])

    #meta_tags = {}
    #for tag in ec2.get_all_tags():
    #    meta_tags[str(tag.name)] = str(tag.value)

    # Log the metadata tags
    app.config["instance.tags"] = meta_tags
    for k in meta_tags:
        logging.debug("[TAGS] {0} : {1}".format(k, meta_tags[k]))

    sqs  = boto.sqs.connect_to_region(app.config["identity"]['region'],
                                      aws_access_key_id=app.config['keys.key_id'],
                                      aws_secret_access_key=app.config['keys.key_secret'],
                                      security_token=app.config['keys.key_token'])

    sns  = boto.sns.connect_to_region(app.config["identity"]['region'],
                                      aws_access_key_id=app.config['keys.key_id'],
                                      aws_secret_access_key=app.config['keys.key_secret'],
                                      security_token=app.config['keys.key_token'])

    ses  = boto.ses.connect_to_region(app.config["identity"]['region'],
                                      aws_access_key_id=app.config['keys.key_id'],
                                      aws_secret_access_key=app.config['keys.key_secret'],
                                      security_token=app.config['keys.key_token'])

    s3   = S3Connection(aws_access_key_id=app.config['keys.key_id'],
                        aws_secret_access_key=app.config['keys.key_secret'],
                        security_token=app.config['keys.key_token'])

    dyno = Table(app.config["instance.tags"]["DynamoDBTableName"],#app.config['dynamodb.table_name'],
                 schema=[HashKey("job_id")],
                 connection=ddb.connect_to_region(app.config['dynamodb.region'],
                                                  aws_access_key_id=app.config['keys.key_id'],
                                                  aws_secret_access_key=app.config['keys.key_secret'],
                                                  security_token=app.config['keys.key_token']))

    app.config["ec2.conn"]  = ec2
    app.config["sns.conn"]  = sns
    app.config["sqs.conn"]  = sqs
    app.config["ses.conn"]  = ses
    app.config["s3.conn"]   = s3
    app.config["dyno.conn"] = dyno
    app.config["doReload"]  = True

    return app

def connect_to_dynamodb(app):
    stat = update_creds_from_metadata_server(app)

    # If an entry exists for the table and the credentials have
    # not been updated then skip the connection
    if "dynamodb.table" in app.config and not stat:
        return app

    dbconn = ddb.connect_to_region(app.config['dynamodb.region'],
                                   aws_access_key_id=app.config['keys.key_id'],
                                   aws_secret_access_key=app.config['keys.key_secret'],
                                   security_token=app.config['keys.key_token'])

    dyno = Table(app.config["instance.tags"]["DynamoDBTableName"], #app.config['dynamodb.table_name'],
                 schema=[HashKey("CustomerUUID")],
                 connection=dbconn)

    app.config["dynamodb.table"] = dyno
    return app

def load_configs(filename):
    app = bottle.default_app()
    try:
        app.config.load_config(filename)
    except Exception, e:
        logging.error("Exception {0} in load_config".format(e))
        exit(-1)

    logging.debug("Config : \n {0}".format(app.config))

    for keys in app.config:
        if keys.startswith('keys.'):
            print keys
            keyfile = app.config[keys].replace('\"', '')

            logging.debug("Keyfile : {0}".format(keyfile.replace('\"', '')))
            if not os.path.isfile(keyfile):
                print "Key file {0} missing!".format(keyfile)
                logging.error("Key file {0} missing!".format(keyfile))
                exit(-1)
            with open(keyfile, 'r') as kf:
                ks = kf.readlines()
                sp = ks[1].split(',')
                app.config[keys] = kf.read()
                app.config["keys.key_id"] = sp[1]
                app.config["keys.key_secret"] = sp[2]
                app.config["keys.key_token"] = ''
                #print "keys : ", app.config[keys]
    if 'metadata.credurl' in app.config:
        update_creds_from_metadata_server(app)

    init(app)
    return app
<|endoftext|>


<commit_before>#!/usr/bin/env python
__author__ = 'Tomasz Turowski'
__copyright__	= "Copyright 2015"
__version__		= "0.1"
__credits__		= ["Tomasz Turowski"]
__email__		= "twturowski@gmail.com"
__status__		= "Production"

"""script using ruffus pipeline to do for each novofile in the folder: mkdir, and run pypileup,
then run pileupsToConcat.py and run script with plotting parameters: analyse_tRNA_from_concat.py"""

from ruffus import *
import subprocess, re, os, argparse, time, yaml, math, shutil
import gwide.methods as gtm

# def novo2concat():
#seting up option parser
parser = argparse.ArgumentParser(description='Usage: ruffus scirpt designed to make concat file from *.novo files. Make new folder, cp or ln into all novofiles and run novo2concat. IMPORTANT: name of novo file should be name of experiment')
parser.add_argument("-g", "--gtf_file", dest="gtf_file", help="Provide the path to your gtf file.",
                     type=str, default=None)
parser.add_argument("-t", "--tab_file", dest="tab_file", help="Provide the path to your tab genome file.",
                     type=str, default=None)
parser.add_argument("-r", dest="ranges", help="Set up ranges for pyPileup. Default = 250", default=250)
parser.add_argument("--3end", dest="three_end", help="Use pyPileup option --3end to only report counts for the 3' end of the reads. Default = False",
                    action="store_true", default=False)
parser.add_argument("--5end", dest="five_end",
                    help="Use pyPileup option --5end to only report counts for the 5' end of the reads. Default = False",
                    action="store_true", default=False)
parser.add_argument("-l", dest="list_file", help="Provide the FULL path to your gene_names.list file.", type=str, default=None, required=True)
parser.add_argument("--tree", dest="tree", help="If you want to leave tree of catalogs including pilups within. Default = None.",
                     action="store_true", default=False)
parser.add_argument("-p", dest="prefix", help="Prefix for concat file name", type=str, default="")
args = parser.parse_args()

gtf, tab, ranges = gtm.getGTF(args.gtf_file), gtm.getTAB(args.tab_file), str(args.ranges)
print "Using GTF file: " + gtf
print "Using TAB genome file: " + tab

#listing novo files
files = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.novo')]  #gives list of files in current directory
directories = [re.sub(r'.novo$', '', d) for d in files]
links = []
root_dir = os.getcwd()

#making directories
for f, d in zip(files, directories):
    os.mkdir(d)
    os.chdir(d)
    subprocess.call('ln -s ../' + f + ' ' + f, shell=True)
    links.append(os.path.abspath('./'+f))
    os.chdir(root_dir)

#setting up concat and log files name
concat_name = args.prefix+"_r"+ranges
if args.three_end == True:
    concat_name = concat_name + "_3end"
elif args.five_end == True:
    concat_name = concat_name + "_5end"

#seting up no. of processes to use
number_of_processors_to_use = len(files)

#making pileups in separate directories
@subdivide(
    links,
    formatter(),
    os.path.join("{path[0]}", "*.pileup")
)
def create_pileup_files(input_file, output_files):
    os.chdir(os.path.dirname(input_file))
    if args.three_end == True:
        subprocess.call(r'pyPileup.py --gtf=' + gtf + ' --tab=' + tab + ' -g ' + args.list_file + ' -f ' + input_file + ' -r ' + ranges + ' --3end',
            shell=True)
    elif args.five_end == True:
        subprocess.call(r'pyPileup.py --gtf=' + gtf + ' --tab=' + tab + ' -g ' + args.list_file + ' -f ' + input_file + ' -r ' + ranges + ' --5end',
            shell=True)
    else:
        subprocess.call(r'pyPileup.py --gtf='+gtf+' --tab='+tab+' -g '+args.list_file+' -f ' + input_file + ' -r '+ranges, shell=True)
    # subprocess.call(r'rm anti*', shell=True)
    os.chdir(root_dir)

#creating concat file
@merge(create_pileup_files, os.path.join(root_dir, concat_name+".concat"))
def merge_files(infiles, concat):
    print "# Creating concat file..."
    #below pileupsToConcac.py script is implemented

    def extract_names(line):
        path = line.strip()
        path_elem = path.split('/')
        exp_name = path_elem[len(path_elem)-2]
        return str(exp_name)

    line_number = 0
    line_number_list = list()
    output_dict = dict()

    for file_path in infiles:
        exp_name = extract_names(file_path)
        if "sense-reads" in file_path:
            pileup_file = open(file_path)
            for line in pileup_file:
                if line.startswith('# total number of mapped reads:'):
                    line_elem_1 = line.strip().split('\t')
                    total_mapped_reads = float(line_elem_1[1])
                    normalizator = 1000000.0/total_mapped_reads
                if not line.startswith('#'):
                    line_elements = line.strip().split('\t')
                    if len(line_elements) > 1:
                        line_number = line_number + 1
                        line_number_list.append(line_number)
                        line_list = list()
                        for i in line_elements:
                            line_list.append(str(i))
                        line_list.append(exp_name)
                        line_list.append(str(int(math.ceil(float(line_elements[3])*normalizator))))
                        line_list.append(str(int(math.ceil(float(line_elements[4])*normalizator))))
                        line_list.append(str(int(math.ceil(float(line_elements[5])*normalizator))))
                        output_dict[str(line_number)] = line_list

    line_number_list.sort()
    output = open(concat, 'w')
    output.write("# concat file from pileup files created: "+time.ctime()+"\n")
    output.write("# gene\tposition\tnucleotide\thits\tsubstitutions\tdeletions\texperiment\thits_pM\tsubst_pM\tdel_pM\n")
    for i in line_number_list:
        output.write('\t'.join(output_dict[str(i)]) + '\n')
    output.close()

#version using dataframe but was extreamly slow!
    # columns = ['gene','position','nucleotide','hits','substitutions','deletions','exp_name','n_hits','n_substitutions','n_deletions']
    # data = pd.DataFrame(columns=columns)
    # for file_path in infiles:
    #     exp_name = extract_names(file_path) #extracts folder_name from file path; folder_name = name from name.novo
    #     pileup_file = open(file_path)
    #     print file_path
    #     for line in pileup_file:
    #         if line.startswith('# total number of mapped reads:'):
    #             line_elem_1 = line.strip().split('\t')
    #             total_mapped_reads = float(line_elem_1[1])
    #             normalizator = 1000000.0/total_mapped_reads
    #         if not line.startswith('#'):
    #             line_elements = line.strip().split('\t')
    #             if len(line_elements) > 1:
    #                 row_data = pd.DataFrame(columns=columns)
    #                 row_data['gene'], row_data['position'], row_data['nucleotide'], row_data['hits'], row_data['substitutions'], row_data['deletions'] = [line_elements[0]], [line_elements[1]], [line_elements[2]], [line_elements[3]], [line_elements[4]], [line_elements[5]]
    #                 row_data['exp_name'] = [exp_name]
    #                 row_data['n_hits'] = [str(int(math.ceil(float(line_elements[3])*normalizator)))]
    #                 row_data['n_substitutions'] = [str(int(math.ceil(float(line_elements[4])*normalizator)))]
    #                 row_data['n_deletions'] = [str(int(math.ceil(float(line_elements[5])*normalizator)))]
    #                 data.append(row_data)
    # print data
    # data.to_csv(concat, sep='\t')

pipeline_run(multiprocess=number_of_processors_to_use)

#deleting directories
if args.tree == False:
    for d in directories:
        shutil.rmtree(d)

# printing summary
log = open(concat_name+"_log.txt", 'w')
log.write("Ruffus script run on "+time.strftime("%d/%m/%Y")+' at '+time.strftime("%H:%M:%S")+'\n')
log.write("Using GTF file: " + gtf+'\n')
log.write("Using TAB genome file: " + tab+'\n')
log.write("Using gene_names.list file: " +args.list_file+'\n')
log.write("Ranges set up to: "+ranges+'\n')
log.close()

<commit_msg>--antisense reads<commit_after>#!/usr/bin/env python
__author__ = 'Tomasz Turowski'
__copyright__	= "Copyright 2015"
__version__		= "0.1"
__credits__		= ["Tomasz Turowski"]
__email__		= "twturowski@gmail.com"
__status__		= "Production"

"""script using ruffus pipeline to do for each novofile in the folder: mkdir, and run pypileup,
then run pileupsToConcat.py and run script with plotting parameters: analyse_tRNA_from_concat.py"""

from ruffus import *
import subprocess, re, os, argparse, time, yaml, math, shutil
import gwide.methods as gtm

# def novo2concat():
#seting up option parser
parser = argparse.ArgumentParser(description='Usage: ruffus scirpt designed to make concat file from *.novo files. Make new folder, cp or ln into all novofiles and run novo2concat. IMPORTANT: name of novo file should be name of experiment')
parser.add_argument("-g", "--gtf_file", dest="gtf_file", help="Provide the path to your gtf file.",
                     type=str, default=None)
parser.add_argument("-t", "--tab_file", dest="tab_file", help="Provide the path to your tab genome file.",
                     type=str, default=None)
parser.add_argument("-r", dest="ranges", help="Set up ranges for pyPileup. Default = 250", default=250)
parser.add_argument("--3end", dest="three_end", help="Use pyPileup option --3end to only report counts for the 3' end of the reads. Default = False",
                    action="store_true", default=False)
parser.add_argument("--5end", dest="five_end",
                    help="Use pyPileup option --5end to only report counts for the 5' end of the reads. Default = False",
                    action="store_true", default=False)
parser.add_argument("-l", dest="list_file", help="Provide the FULL path to your gene_names.list file.", type=str, default=None, required=True)
parser.add_argument("--tree", dest="tree", help="If you want to leave tree of catalogs including pilups within. Default = None.",
                     action="store_true", default=False)
parser.add_argument("--anti", dest="anti",  help="Create additional concat file with antisense reads Default = None.",
                    action="store_true", default=False)
parser.add_argument("-p", dest="prefix", help="Prefix for concat file name", type=str, default="")
args = parser.parse_args()

gtf, tab, ranges = gtm.getGTF(args.gtf_file), gtm.getTAB(args.tab_file), str(args.ranges)
print "Using GTF file: " + gtf
print "Using TAB genome file: " + tab

#listing novo files
files = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.novo')]  #gives list of files in current directory
directories = [re.sub(r'.novo$', '', d) for d in files]
links = []
root_dir = os.getcwd()

#making directories
for f, d in zip(files, directories):
    os.mkdir(d)
    os.chdir(d)
    subprocess.call('ln -s ../' + f + ' ' + f, shell=True)
    links.append(os.path.abspath('./'+f))
    os.chdir(root_dir)

#setting up concat and log files nameq
concat_name = args.prefix+"_r"+ranges
if args.three_end == True:
    concat_name = concat_name + "_3end"
elif args.five_end == True:
    concat_name = concat_name + "_5end"

#seting up no. of processes to use
number_of_processors_to_use = len(files)

#making pileups in separate directories
@subdivide(
    links,
    formatter(),
    os.path.join("{path[0]}", "*.pileup")
)
def create_pileup_files(input_file, output_files):
    os.chdir(os.path.dirname(input_file))
    command_part = ''
    if args.three_end: command_part = command_part+' --3end'
    if args.five_end: command_part = command_part+' --5end'
    if args.anti: command_part = command_part + ' --anti_sense'

    subprocess.call(r'pyPileup.py --gtf='+gtf+' --tab='+tab+' -g '+args.list_file+' -f ' + input_file + ' -r '+ranges + command_part, shell=True)
    os.chdir(root_dir)

#creating concat file
@merge(create_pileup_files, os.path.join(root_dir, concat_name+".concat"))
def merge_files(infiles, concat):
    print "# Creating concat file..."
    #below pileupsToConcac.py script is implemented

    def extract_names(line):
        path = line.strip()
        path_elem = path.split('/')
        exp_name = path_elem[len(path_elem)-2]
        return str(exp_name)

    line_number = 0
    line_number_list = list()
    output_dict = dict()

    for file_path in infiles:
        exp_name = extract_names(file_path)
        if "sense-reads" in file_path:
            pileup_file = open(file_path)
            for line in pileup_file:
                if line.startswith('# total number of mapped reads:'):
                    line_elem_1 = line.strip().split('\t')
                    total_mapped_reads = float(line_elem_1[1])
                    normalizator = 1000000.0/total_mapped_reads
                if not line.startswith('#'):
                    line_elements = line.strip().split('\t')
                    if len(line_elements) > 1:
                        line_number = line_number + 1
                        line_number_list.append(line_number)
                        line_list = list()
                        for i in line_elements:
                            line_list.append(str(i))
                        line_list.append(exp_name)
                        line_list.append(str(int(math.ceil(float(line_elements[3])*normalizator))))
                        line_list.append(str(int(math.ceil(float(line_elements[4])*normalizator))))
                        line_list.append(str(int(math.ceil(float(line_elements[5])*normalizator))))
                        output_dict[str(line_number)] = line_list

    line_number_list.sort()
    output = open(concat, 'w')
    output.write("# concat file from pileup files created: "+time.ctime()+"\n")
    output.write("# gene\tposition\tnucleotide\thits\tsubstitutions\tdeletions\texperiment\thits_pM\tsubst_pM\tdel_pM\n")
    for i in line_number_list:
        output.write('\t'.join(output_dict[str(i)]) + '\n')
    output.close()

    if args.anti == True:
        line_number = 0
        line_number_list = list()
        output_dict = dict()

        for file_path in infiles:
            exp_name = extract_names(file_path)
            if "anti_sense-reads" in file_path:
                pileup_file = open(file_path)
                for line in pileup_file:
                    if line.startswith('# total number of mapped reads:'):
                        line_elem_1 = line.strip().split('\t')
                        total_mapped_reads = float(line_elem_1[1])
                        normalizator = 1000000.0 / total_mapped_reads
                    if not line.startswith('#'):
                        line_elements = line.strip().split('\t')
                        if len(line_elements) > 1:
                            line_number = line_number + 1
                            line_number_list.append(line_number)
                            line_list = list()
                            for i in line_elements:
                                line_list.append(str(i))
                            line_list.append(exp_name)
                            line_list.append(str(int(math.ceil(float(line_elements[3]) * normalizator))))
                            line_list.append(str(int(math.ceil(float(line_elements[4]) * normalizator))))
                            line_list.append(str(int(math.ceil(float(line_elements[5]) * normalizator))))
                            output_dict[str(line_number)] = line_list

        line_number_list.sort()
        output = open('antisense_reads_'+concat, 'w')
        output.write("# concat file from pileup files created: " + time.ctime() + "\n")
        output.write(
            "# gene\tposition\tnucleotide\thits\tsubstitutions\tdeletions\texperiment\thits_pM\tsubst_pM\tdel_pM\n")
        for i in line_number_list:
            output.write('\t'.join(output_dict[str(i)]) + '\n')
        output.close()

#version using dataframe but was extreamly slow!
    # columns = ['gene','position','nucleotide','hits','substitutions','deletions','exp_name','n_hits','n_substitutions','n_deletions']
    # data = pd.DataFrame(columns=columns)
    # for file_path in infiles:
    #     exp_name = extract_names(file_path) #extracts folder_name from file path; folder_name = name from name.novo
    #     pileup_file = open(file_path)
    #     print file_path
    #     for line in pileup_file:
    #         if line.startswith('# total number of mapped reads:'):
    #             line_elem_1 = line.strip().split('\t')
    #             total_mapped_reads = float(line_elem_1[1])
    #             normalizator = 1000000.0/total_mapped_reads
    #         if not line.startswith('#'):
    #             line_elements = line.strip().split('\t')
    #             if len(line_elements) > 1:
    #                 row_data = pd.DataFrame(columns=columns)
    #                 row_data['gene'], row_data['position'], row_data['nucleotide'], row_data['hits'], row_data['substitutions'], row_data['deletions'] = [line_elements[0]], [line_elements[1]], [line_elements[2]], [line_elements[3]], [line_elements[4]], [line_elements[5]]
    #                 row_data['exp_name'] = [exp_name]
    #                 row_data['n_hits'] = [str(int(math.ceil(float(line_elements[3])*normalizator)))]
    #                 row_data['n_substitutions'] = [str(int(math.ceil(float(line_elements[4])*normalizator)))]
    #                 row_data['n_deletions'] = [str(int(math.ceil(float(line_elements[5])*normalizator)))]
    #                 data.append(row_data)
    # print data
    # data.to_csv(concat, sep='\t')

pipeline_run(multiprocess=number_of_processors_to_use)

#deleting directories
if args.tree == False:
    for d in directories:
        shutil.rmtree(d)

# printing summary
log = open(concat_name+"_log.txt", 'w')
log.write("Ruffus script run on "+time.strftime("%d/%m/%Y")+' at '+time.strftime("%H:%M:%S")+'\n')
log.write("Using GTF file: " + gtf+'\n')
log.write("Using TAB genome file: " + tab+'\n')
log.write("Using gene_names.list file: " +args.list_file+'\n')
log.write("Ranges set up to: "+ranges+'\n')
log.close()

<|endoftext|>


<commit_before>from datetime import timedelta
import logging
from collections import namedtuple
import os
from time import time

from django.core.management import BaseCommand

from corehq.apps.app_manager.models import Application
from corehq.util.couch import DocUpdate, iter_update

logger = logging.getLogger('app_migration')
logger.setLevel('DEBUG')

def get_all_app_ids(domain=None, include_builds=False):
    key = [domain]
    if not include_builds:
        key += [None]

    return {r['id'] for r in Application.get_db().view(
        'app_manager/applications',
        startkey=key,
        endkey=key + [{}],
        reduce=False,
    ).all()}

SaveError = namedtuple('SaveError', 'id error reason')

class AppMigrationCommandBase(BaseCommand):
    """
    Base class for commands that want to migrate apps.
    """
    chunk_size = 100
    include_builds = False
    include_linked_apps = False
    # Overwrite these to avoid conflicting naming with another managment command.
    DOMAIN_LIST_FILENAME = "app_migration_command_domain_list.txt"
    DOMAIN_PROGRESS_NUMBER_FILENAME = "app_migration_command_domain_progress.txt"

    options = {}

    def add_arguments(self, parser):
        parser.add_argument(
            '--failfast',
            action='store_true',
            dest='failfast',
            default=False,
            help='Stop processing if there is an error',
        )
        parser.add_argument(
            '--domain',
            help='Migrate only this domain',
        )
        parser.add_argument(
            '--dry-run',
            action='store_true',
            default=False,
            help="Perform the migration but don't save any changes",
        )
        parser.add_argument(
            '--start-from-scratch',
            action='store_true',
            default=False,
            help="If existing progress files for this command exist, turn this flag on to ignore them and start"
                 "from scratch.",
        )

    def handle(self, **options):
        start_time = time()
        self.options = options
        if self.options['domain']:
            domains = [self.options['domain']]
            domain_list_position = 0
        else:
            can_continue_progress, domain_list_position, domains = self.try_to_continue_progress()
            if not can_continue_progress:
                domains = self.get_domains() or [None]
                self.store_domain_list(domains)
        for domain in domains:
            app_ids = self.get_app_ids(domain)
            logger.info('migrating {} apps{}'.format(len(app_ids), f" in {domain}" if domain else ""))
            iter_update(Application.get_db(), self._migrate_app, app_ids, verbose=True, chunksize=self.chunk_size)
            domain_list_position = self.increment_progress(domain_list_position)
        self.remove_storage_files()
        end_time = time()
        execution_time_seconds = end_time - start_time
        logger.info(f"Completed in {timedelta(seconds=execution_time_seconds)}.")

    @property
    def is_dry_run(self):
        return self.options.get('dry_run', False)

    @property
    def log_info(self):
        return self.options.get("verbosity", 0) > 1

    @property
    def log_debug(self):
        return self.options.get("verbosity", 0) > 2

    @property
    def restartable(self):
        return self.options.get('restartable', False)

    def _doc_types(self):
        doc_types = ["Application", "Application-Deleted"]
        if self.include_linked_apps:
            doc_types.extend(["LinkedApplication", "LinkedApplication-Deleted"])
        return doc_types

    def _migrate_app(self, app_doc):
        try:
            if app_doc["doc_type"] in self._doc_types():
                migrated_app = self.migrate_app(app_doc)
                if migrated_app and not self.is_dry_run:
                    return DocUpdate(self.increment_app_version(migrated_app))
        except Exception as e:
            logger.exception("App {id} not properly migrated".format(id=app_doc['_id']))
            if self.options['failfast']:
                raise e

    @staticmethod
    def increment_app_version(app_doc):
        try:
            copy_of = app_doc['copy_of']
            version = app_doc['version']
        except KeyError:
            return
        if copy_of and version:
            app_doc['version'] = version + 1
        return app_doc

    def get_app_ids(self, domain=None):
        return get_all_app_ids(domain=domain, include_builds=self.include_builds)

    def get_domains(self):
        return None

    def migrate_app(self, app):
        """Return the app dict if the doc is to be saved else None"""
        raise NotImplementedError()

    def increment_progress(self, domain_list_position):
        logger.info(f"Migrated domain #{domain_list_position}")
        domain_list_position += 1
        with open(self.DOMAIN_PROGRESS_NUMBER_FILENAME, 'w') as f:
            f.write(str(domain_list_position))
        return domain_list_position

    def store_domain_list(self, domains):
        with open(self.DOMAIN_LIST_FILENAME, 'w') as f:
            f.writelines(f'{domain}\n' for domain in domains)

    def try_to_continue_progress(self):
        if not self.options['start_from_scratch']:
            try:
                with open(self.DOMAIN_PROGRESS_NUMBER_FILENAME, 'r') as f:
                    domain_list_position = int(f.readline())
                with open(self.DOMAIN_LIST_FILENAME, 'r') as f:
                    domains = []
                    for line in f:
                        domains.append(line[:-1])
                logger.info(f"Continuing migration progress at domain number {domain_list_position}...")
                return True, domain_list_position, domains
            except FileNotFoundError:
                logger.info("Domain progress file(s) not found. Starting from scratch...")
        return False, 0, None

    def remove_storage_files(self):
        try:
            os.remove(self.DOMAIN_LIST_FILENAME)
            os.remove(self.DOMAIN_PROGRESS_NUMBER_FILENAME)
        except FileNotFoundError:
            pass
<commit_msg>fix continuation bug<commit_after>from datetime import timedelta
import logging
from collections import namedtuple
import os
from time import time

from django.core.management import BaseCommand

from corehq.apps.app_manager.models import Application
from corehq.util.couch import DocUpdate, iter_update

logger = logging.getLogger('app_migration')
logger.setLevel('DEBUG')

def get_all_app_ids(domain=None, include_builds=False):
    key = [domain]
    if not include_builds:
        key += [None]

    return {r['id'] for r in Application.get_db().view(
        'app_manager/applications',
        startkey=key,
        endkey=key + [{}],
        reduce=False,
    ).all()}

SaveError = namedtuple('SaveError', 'id error reason')

class AppMigrationCommandBase(BaseCommand):
    """
    Base class for commands that want to migrate apps.
    """
    chunk_size = 100
    include_builds = False
    include_linked_apps = False
    # Overwrite these to avoid conflicting naming with another managment command.
    DOMAIN_LIST_FILENAME = "app_migration_command_domain_list.txt"
    DOMAIN_PROGRESS_NUMBER_FILENAME = "app_migration_command_domain_progress.txt"

    options = {}

    def add_arguments(self, parser):
        parser.add_argument(
            '--failfast',
            action='store_true',
            dest='failfast',
            default=False,
            help='Stop processing if there is an error',
        )
        parser.add_argument(
            '--domain',
            help='Migrate only this domain',
        )
        parser.add_argument(
            '--dry-run',
            action='store_true',
            default=False,
            help="Perform the migration but don't save any changes",
        )
        parser.add_argument(
            '--start-from-scratch',
            action='store_true',
            default=False,
            help="If existing progress files for this command exist, turn this flag on to ignore them and start"
                 "from scratch.",
        )

    def handle(self, **options):
        start_time = time()
        self.options = options
        if self.options['domain']:
            domains = [self.options['domain']]
            domain_list_position = 0
        else:
            can_continue_progress, domain_list_position, domains = self.try_to_continue_progress()
            if can_continue_progress:
                domains = domains[domain_list_position:]
            else:
                domains = self.get_domains() or [None]
                self.store_domain_list(domains)
        for domain in domains:
            app_ids = self.get_app_ids(domain)
            logger.info('migrating {} apps{}'.format(len(app_ids), f" in {domain}" if domain else ""))
            iter_update(Application.get_db(), self._migrate_app, app_ids, verbose=True, chunksize=self.chunk_size)
            domain_list_position = self.increment_progress(domain_list_position)
        self.remove_storage_files()
        end_time = time()
        execution_time_seconds = end_time - start_time
        logger.info(f"Completed in {timedelta(seconds=execution_time_seconds)}.")

    @property
    def is_dry_run(self):
        return self.options.get('dry_run', False)

    @property
    def log_info(self):
        return self.options.get("verbosity", 0) > 1

    @property
    def log_debug(self):
        return self.options.get("verbosity", 0) > 2

    @property
    def restartable(self):
        return self.options.get('restartable', False)

    def _doc_types(self):
        doc_types = ["Application", "Application-Deleted"]
        if self.include_linked_apps:
            doc_types.extend(["LinkedApplication", "LinkedApplication-Deleted"])
        return doc_types

    def _migrate_app(self, app_doc):
        try:
            if app_doc["doc_type"] in self._doc_types():
                migrated_app = self.migrate_app(app_doc)
                if migrated_app and not self.is_dry_run:
                    return DocUpdate(self.increment_app_version(migrated_app))
        except Exception as e:
            logger.exception("App {id} not properly migrated".format(id=app_doc['_id']))
            if self.options['failfast']:
                raise e

    @staticmethod
    def increment_app_version(app_doc):
        try:
            copy_of = app_doc['copy_of']
            version = app_doc['version']
        except KeyError:
            return
        if copy_of and version:
            app_doc['version'] = version + 1
        return app_doc

    def get_app_ids(self, domain=None):
        return get_all_app_ids(domain=domain, include_builds=self.include_builds)

    def get_domains(self):
        return None

    def migrate_app(self, app):
        """Return the app dict if the doc is to be saved else None"""
        raise NotImplementedError()

    def increment_progress(self, domain_list_position):
        logger.info(f"Migrated domain #{domain_list_position}")
        domain_list_position += 1
        with open(self.DOMAIN_PROGRESS_NUMBER_FILENAME, 'w') as f:
            f.write(str(domain_list_position))
        return domain_list_position

    def store_domain_list(self, domains):
        with open(self.DOMAIN_LIST_FILENAME, 'w') as f:
            f.writelines(f'{domain}\n' for domain in domains)

    def try_to_continue_progress(self):
        if not self.options['start_from_scratch']:
            try:
                with open(self.DOMAIN_PROGRESS_NUMBER_FILENAME, 'r') as f:
                    domain_list_position = int(f.readline())
                with open(self.DOMAIN_LIST_FILENAME, 'r') as f:
                    domains = []
                    for line in f:
                        domains.append(line[:-1])
                logger.info(f"Continuing migration progress at domain number {domain_list_position}...")
                return True, domain_list_position, domains
            except FileNotFoundError:
                logger.info("Domain progress file(s) not found. Starting from scratch...")
        return False, 0, None

    def remove_storage_files(self):
        try:
            os.remove(self.DOMAIN_LIST_FILENAME)
            os.remove(self.DOMAIN_PROGRESS_NUMBER_FILENAME)
        except FileNotFoundError:
            pass
<|endoftext|>


<commit_before>import mimetypes

from corehq.form_processor.abstract_models import IsImageMixin
from dimagi.ext.couchdbkit import StringProperty, IntegerProperty, DictProperty
from dimagi.utils.couch import LooselyEqualDocumentSchema

"""
Shared models live here to avoid cyclical import issues
"""

class CommCareCaseIndex(LooselyEqualDocumentSchema):
    """
    In CaseXML v2 we support indices, which link a case to other cases.
    """
    identifier = StringProperty()
    referenced_type = StringProperty()
    referenced_id = StringProperty()
    # relationship = "child" for index to a parent case (default)
    # relationship = "extension" for index to a host case
    relationship = StringProperty('child', choices=['child', 'extension'])

    @property
    def referenced_case(self):
        """
        For a 'forward' index this is the case that the the index points to.
        For a 'reverse' index this is the case that owns the index.
        See ``corehq/couchapps/case_indices/views/related/map.js``

        :return: referenced case
        """
        if not hasattr(self, "_case"):
            from casexml.apps.case.models import CommCareCase
            self._case = CommCareCase.get(self.referenced_id)
        return self._case

    @classmethod
    def from_case_index_update(cls, index):
        return cls(identifier=index.identifier,
                   referenced_type=index.referenced_type,
                   referenced_id=index.referenced_id,
                   relationship=index.relationship,)

    def __str__(self):
        return (
            "CommCareCaseIndex("
            "identifier='{index.identifier}', "
            "referenced_type='{index.referenced_type}', "
            "referenced_id='{index.referenced_id}', "
            "relationship='{index.relationship}'"
            ")"
        ).format(index=self)

    def __cmp__(self, other):
        a = str(self)
        b = str(other)
        return (a > b) - (a < b)

    def __repr__(self):
        return str(self)

class CommCareCaseAttachment(LooselyEqualDocumentSchema, IsImageMixin):
    identifier = StringProperty()
    attachment_src = StringProperty()
    attachment_from = StringProperty()
    attachment_name = StringProperty()
    server_mime = StringProperty()  # Server detected MIME
    server_md5 = StringProperty()  # Couch detected hash

    attachment_size = IntegerProperty()  # file size
    attachment_properties = DictProperty()  # width, height, other relevant metadata

    @property
    def content_type(self):
        return self.server_mime

    @property
    def is_present(self):
        """
        Helper method to see if this is a delete vs. update

        NOTE this is related to but reversed logic from
        `casexml.apps.case.xml.parser.CaseAttachment.is_delete`.
        """
        return self.attachment_src or self.attachment_from

    @classmethod
    def from_case_index_update(cls, attachment):
        if attachment.attachment_src or attachment.attachment_from:
            guessed = mimetypes.guess_type(attachment.attachment_src)
            if len(guessed) > 0 and guessed[0] is not None:
                mime_type = guessed[0]
            else:
                mime_type = None

            ret = cls(identifier=attachment.identifier,
                      attachment_src=attachment.attachment_src,
                      attachment_from=attachment.attachment_from,
                      attachment_name=attachment.attachment_name,
                      server_mime=mime_type)
        else:
            ret = cls(identifier=attachment.identifier)
        return ret

class IndexHoldingMixIn(object):
    """
    Since multiple objects need this functionality, implement it as a mixin
    """

    def has_index(self, id):
        return id in (i.identifier for i in self.indices)

    def get_index(self, id):
        found = [i for i in self.indices if i.identifier == id]
        if found:
            assert(len(found) == 1)
            return found[0]
        return None

    def get_index_by_ref_id(self, doc_id):
        found = [i for i in self.indices if i.referenced_id == doc_id]
        if found:
            assert(len(found) == 1)
            return found[0]
        return None

    def update_indices(self, index_update_list):
        for index_update in index_update_list:
            if index_update.referenced_id:
                # NOTE: used to check the existence of the referenced
                # case here but is moved into the pre save processing
                pass
            if self.has_index(index_update.identifier):
                if not index_update.referenced_id:
                    # empty ID = delete
                    self.indices.remove(self.get_index(index_update.identifier))
                else:
                    # update
                    index = self.get_index(index_update.identifier)
                    index.referenced_type = index_update.referenced_type
                    index.referenced_id = index_update.referenced_id
                    index.relationship = index_update.relationship
            else:
                # no id, no index
                if index_update.referenced_id:
                    self.indices.append(CommCareCaseIndex(identifier=index_update.identifier,
                                                          referenced_type=index_update.referenced_type,
                                                          referenced_id=index_update.referenced_id,
                                                          relationship=index_update.relationship,))

    def remove_index_by_ref_id(self, doc_id):
        index = self.get_index_by_ref_id(doc_id)
        if not index:
            raise ValueError('index with id %s not found in doc %s' % (id, self._id))
        self.indices.remove(index)
<commit_msg>Replaced __cmp__ with __lt__<commit_after>import mimetypes

from corehq.form_processor.abstract_models import IsImageMixin
from dimagi.ext.couchdbkit import StringProperty, IntegerProperty, DictProperty
from dimagi.utils.couch import LooselyEqualDocumentSchema

"""
Shared models live here to avoid cyclical import issues
"""

class CommCareCaseIndex(LooselyEqualDocumentSchema):
    """
    In CaseXML v2 we support indices, which link a case to other cases.
    """
    identifier = StringProperty()
    referenced_type = StringProperty()
    referenced_id = StringProperty()
    # relationship = "child" for index to a parent case (default)
    # relationship = "extension" for index to a host case
    relationship = StringProperty('child', choices=['child', 'extension'])

    @property
    def referenced_case(self):
        """
        For a 'forward' index this is the case that the the index points to.
        For a 'reverse' index this is the case that owns the index.
        See ``corehq/couchapps/case_indices/views/related/map.js``

        :return: referenced case
        """
        if not hasattr(self, "_case"):
            from casexml.apps.case.models import CommCareCase
            self._case = CommCareCase.get(self.referenced_id)
        return self._case

    @classmethod
    def from_case_index_update(cls, index):
        return cls(identifier=index.identifier,
                   referenced_type=index.referenced_type,
                   referenced_id=index.referenced_id,
                   relationship=index.relationship,)

    def __str__(self):
        return (
            "CommCareCaseIndex("
            "identifier='{index.identifier}', "
            "referenced_type='{index.referenced_type}', "
            "referenced_id='{index.referenced_id}', "
            "relationship='{index.relationship}'"
            ")"
        ).format(index=self)

    def __lt__(self, other):
        return str(self) < str(other)

    def __repr__(self):
        return str(self)

class CommCareCaseAttachment(LooselyEqualDocumentSchema, IsImageMixin):
    identifier = StringProperty()
    attachment_src = StringProperty()
    attachment_from = StringProperty()
    attachment_name = StringProperty()
    server_mime = StringProperty()  # Server detected MIME
    server_md5 = StringProperty()  # Couch detected hash

    attachment_size = IntegerProperty()  # file size
    attachment_properties = DictProperty()  # width, height, other relevant metadata

    @property
    def content_type(self):
        return self.server_mime

    @property
    def is_present(self):
        """
        Helper method to see if this is a delete vs. update

        NOTE this is related to but reversed logic from
        `casexml.apps.case.xml.parser.CaseAttachment.is_delete`.
        """
        return self.attachment_src or self.attachment_from

    @classmethod
    def from_case_index_update(cls, attachment):
        if attachment.attachment_src or attachment.attachment_from:
            guessed = mimetypes.guess_type(attachment.attachment_src)
            if len(guessed) > 0 and guessed[0] is not None:
                mime_type = guessed[0]
            else:
                mime_type = None

            ret = cls(identifier=attachment.identifier,
                      attachment_src=attachment.attachment_src,
                      attachment_from=attachment.attachment_from,
                      attachment_name=attachment.attachment_name,
                      server_mime=mime_type)
        else:
            ret = cls(identifier=attachment.identifier)
        return ret

class IndexHoldingMixIn(object):
    """
    Since multiple objects need this functionality, implement it as a mixin
    """

    def has_index(self, id):
        return id in (i.identifier for i in self.indices)

    def get_index(self, id):
        found = [i for i in self.indices if i.identifier == id]
        if found:
            assert(len(found) == 1)
            return found[0]
        return None

    def get_index_by_ref_id(self, doc_id):
        found = [i for i in self.indices if i.referenced_id == doc_id]
        if found:
            assert(len(found) == 1)
            return found[0]
        return None

    def update_indices(self, index_update_list):
        for index_update in index_update_list:
            if index_update.referenced_id:
                # NOTE: used to check the existence of the referenced
                # case here but is moved into the pre save processing
                pass
            if self.has_index(index_update.identifier):
                if not index_update.referenced_id:
                    # empty ID = delete
                    self.indices.remove(self.get_index(index_update.identifier))
                else:
                    # update
                    index = self.get_index(index_update.identifier)
                    index.referenced_type = index_update.referenced_type
                    index.referenced_id = index_update.referenced_id
                    index.relationship = index_update.relationship
            else:
                # no id, no index
                if index_update.referenced_id:
                    self.indices.append(CommCareCaseIndex(identifier=index_update.identifier,
                                                          referenced_type=index_update.referenced_type,
                                                          referenced_id=index_update.referenced_id,
                                                          relationship=index_update.relationship,))

    def remove_index_by_ref_id(self, doc_id):
        index = self.get_index_by_ref_id(doc_id)
        if not index:
            raise ValueError('index with id %s not found in doc %s' % (id, self._id))
        self.indices.remove(index)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# import the necessary packages
from cu_coursecatalog_spider.items import Course, Major, Minor
import scrapy
import json

# Spider used to crawl through the webpage and get info
class SchmidCatalogSpider(scrapy.Spider):

	# Name of the spider
	name = "schmid-catalog-spider"

	# URLs of each school's catalog
	start_urls = [
		'https://www.chapman.edu/catalog/oc/current/ug/content/8174.htm', # Schmid College
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/3610.htm', # Argyros School
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/3695.htm', # College of Education
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/3807.htm', # Dodge College
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/9075.htm', # Crean College
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/3910.htm', # Wilkinson College
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/7580.htm', # College of Performing Arts
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/12155.htm', # School of Pharmacy
	]

	# In charge of processing the response and returning the scraped data as objects
	def parse(self, response):
		empty = []
		emptyStr = ''
		ignore = u'\r\n'
		nbsp = u'\xa0'

		# Used to get major names
		descs = []
		reqs = []
		for selector in response.xpath('//p/span/a'):
			majorTitle = selector.xpath('@title').extract()[0]
			print majorTitle
			subHeadingStr = ''
			for sel in response.xpath('//*[(name()=\'p\' or name()=\'table\') and (preceding-sibling::h2[1][.=\''+majorTitle+'\'])]'):
				links = sel.xpath('a/text()').extract()
				subHeading = sel.xpath('span/text()').extract()
				description = sel.xpath('text()').extract()
				tableTxt = sel.xpath('tr/td')
				# If there are links then they must be put into the description
				if links != empty:
					# Fix for links that are not adding if it's the first index
					offset = 1
					if description[0].encode('utf-8').startswith('–'):
						offset = 0
					for i in range(len(links)):
						description.insert((2*i)+offset, links[i])
					description = ''.join(description)
					descs.insert(len(descs), description.encode('utf-8'))
				elif subHeading != empty and description != empty:
					for i in range(0, len(subHeading)):
						subHeadingStr = subHeadingStr + subHeading[i] + description[i]
				elif description != empty:
					if ignore not in description:
						if subHeadingStr != emptyStr:
							subHeadingStr = subHeadingStr + ' ' + description[0]
						else:
							descs.insert(len(descs), description[0].encode('utf-8'))
				elif subHeading != empty:
					descs.insert(len(descs), subHeading[0].encode('utf-8'))
				if tableTxt != empty:
					# Fix for table description that doesn't have any links
					tableSel = tableTxt.xpath('p/a/text()')
					if tableSel == empty or tableSel == None:
						tableSel = tableTxt.xpath('p/text()').extract()
						tableDesc = ''
						for uTxt in tableSel:
							if nbsp not in uTxt:
								tableDesc += ' ' + uTxt.encode('utf-8')
						print 'tableDescNoLink: ', tableDesc
						stripChars  = ' '
						if subHeadingStr != emptyStr:
							reqs.insert(len(reqs), subHeadingStr)
							stripChars += subHeadingStr[-11].encode('utf-8') + subHeadingStr[-10].encode('utf-8')
							subHeadingStr = ''
						reqs.insert(len(reqs), tableDesc.strip(stripChars))

					# If the table consists of a list of subjects or is a description with links
					else:
						tableDesc = tableTxt.xpath('p/text()').extract()
						tableSel = tableSel.extract()
						if subHeadingStr != emptyStr:
							reqs.insert(len(reqs), subHeadingStr.encode('utf-8'))
							subHeadingStr = ''
						for i in range(len(tableSel)):
							if nbsp in tableDesc[i]:
								# special condition where its a description with links
								tableDesc = tableTxt.xpath('p/text()').extract()
								del tableDesc[0]
								del tableDesc[-1]
								print tableDesc
								print tableSel
								for j in range(len(tableSel)):
									tableDesc.insert((2*j)+1, tableSel[j].encode('utf-8'))
								tableDesc = ''.join(tableDesc)
								tableSel = tableDesc
								break
							tableSel[i] = tableSel[i].encode('utf-8')
						reqs.insert(len(reqs), tableSel)
						print 'subjList or descWithLink: ', tableSel
				print 'subHeadingStr: ', subHeadingStr
				print 'links: ', links
				print 'subHeading: ', subHeading
				print 'desc: ', description
				print
			print descs
			print reqs
			yield Major(title=majorTitle, department='Schmid College of Science and Technology', description=descs, requirements=reqs)
			descs = []
			reqs = []

		# Used to get minor names
		# descs = []
		# reqs = []
		# for selector in response.xpath('//h3[re:test(., \'Minor in\', \'i\')]'):
		# 	minorTitle = selector.xpath('text()').extract()[0]
		# 	subHeadingStr = ''
		# 	for sel in selector.xpath('//*[(name()=\'p\' or name()=\'table\') and (preceding-sibling::h3[1][.=\''+minorTitle+'\'])]'):
		# 		links = sel.xpath('a/text()').extract()
		# 		subHeading = sel.xpath('span/text()').extract()
		# 		description = sel.xpath('text()').extract()
		# 		tableTxt = sel.xpath('tr/td')
		# 		# If there are links then they must be put into the description
		# 		if links != empty:
		# 			# Fix for links that are not adding if it's the first index
		# 			offset = 1
		# 			if description[0].encode('utf-8').startswith('–'):
		# 				offset = 0
		# 			for i in range(len(links)):
		# 				description.insert((2*i)+offset, links[i])
		# 			description = ''.join(description)
		# 			descs.insert(len(descs), description.encode('utf-8'))
		# 		elif subHeading != empty and description != empty:
		# 			for i in range(0, len(subHeading)):
		# 				subHeadingStr = subHeadingStr + subHeading[i] + description[i]
		# 		elif description != empty:
		# 			if ignore not in description:
		# 				if subHeadingStr != emptyStr:
		# 					subHeadingStr = subHeadingStr + ' ' + description[0]
		# 				else:
		# 					descs.insert(len(descs), description[0].encode('utf-8'))
		# 		elif subHeading != empty:
		# 			descs.insert(len(descs), subHeading[0].encode('utf-8'))
		# 		if tableTxt != empty:
		# 			# Fix for table description that doesn't have any links
		# 			tableSel = tableTxt.xpath('p/a/text()')
		# 			if tableSel == empty or tableSel == None:
		# 				tableSel = tableTxt.xpath('p/text()').extract()
		# 				tableDesc = ''
		# 				for uTxt in tableSel:
		# 					if nbsp not in uTxt:
		# 						tableDesc += ' ' + uTxt.encode('utf-8')
		# 				print tableDesc
		# 				reqs.insert(len(reqs), tableDesc.strip(' '))

		# 			# If the table consists of a list of subjects or is a description with links
		# 			else:
		# 				tableDesc = tableTxt.xpath('p/text()').extract()
		# 				tableSel = tableSel.extract()
		# 				reqs.insert(len(reqs), subHeadingStr.encode('utf-8'))
		# 				subHeadingStr = ''
		# 				for i in range(len(tableSel)):
		# 					if nbsp in tableDesc[i]:
		# 						# special condition where its a description with links
		# 						tableDesc = tableTxt.xpath('p/text()').extract()
		# 						del tableDesc[0]
		# 						del tableDesc[-1]
		# 						print tableDesc
		# 						print tableSel
		# 						for j in range(len(tableSel)):
		# 							tableDesc.insert((2*j)+1, tableSel[j].encode('utf-8'))
		# 						tableDesc = ''.join(tableDesc)
		# 						tableSel = tableDesc
		# 						break
		# 					tableSel[i] = tableSel[i].encode('utf-8')
		# 				reqs.insert(len(reqs), tableSel)
		# 				print tableSel
		# 		print subHeadingStr
		# 		print links
		# 		print subHeading
		# 		print description
		# 		print
		# 	print descs
		# 	print reqs
		# 	yield Minor(title=minorTitle, department='Schmid College of Science and Technology', description=descs, requirements=reqs)
		# 	descs = []
		# 	reqs = []

		# # Used to get courses
		# subject = ''
		# number = -1
		# name = ' '
		# heading = True
		# for sel in response.xpath('//*[(name()=\'h3\' and re:test(., \'^((?!Minor in).)*$\', \'i\')) or (name()=\'p\' and @class=\'coursedescription\')]'):
		# 	if (heading == True):
		# 		courseStr = sel.xpath('text()').extract()[0]
		# 		courseStr = courseStr.split()
		# 		name = ' '.join(courseStr[2:])
		# 		subject = courseStr[0]
		# 		number = courseStr[1]
		# 		heading = False
		# 	else:
		# 		prerequisites = sel.xpath('a/text()').extract()
		# 		description = sel.xpath('text()').extract()
		# 		if (prerequisites != empty):
		# 			for i in range(len(prerequisites)):
		# 				description.insert((2*i)+1, prerequisites[i])
		# 		description = ''.join(description)
		# 		heading = True
		# 		yield Course(subject=subject, number=number, name=name, description=description)
<commit_msg>URL changed to '15-16 catalog<commit_after># -*- coding: utf-8 -*-
# import the necessary packages
from cu_coursecatalog_spider.items import Course, Major, Minor
import scrapy
import json

# Spider used to crawl through the webpage and get info
class SchmidCatalogSpider(scrapy.Spider):

	# Name of the spider
	name = "schmid-catalog-spider"

	# URLs of each school's catalog
	start_urls = [
		'https://www.chapman.edu/catalog/oc/current/ug/content/8174.htm', # Schmid College
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/3610.htm', # Argyros School
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/3695.htm', # College of Education
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/3807.htm', # Dodge College
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/9075.htm', # Crean College
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/3910.htm', # Wilkinson College
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/7580.htm', # College of Performing Arts
		# 'https://www.chapman.edu/catalog/oc/current/ug/content/12155.htm', # School of Pharmacy
	]

	# In charge of processing the response and returning the scraped data as objects
	def parse(self, response):
		empty = []
		emptyStr = ''
		ignore = u'\r\n'
		nbsp = u'\xa0'

		department = response.xpath('//h1[1]/text()').extract()[0]

		# Used to get major names
		descs = []
		reqs = []
		for selector in response.xpath('//h2[contains(text(), \'Bachelor of Science in \')]'):
			majorTitle = selector.xpath('text()').extract()[0]
			print majorTitle
			subHeadingStr = ''
			for sel in response.xpath('//*[(name()=\'p\' or name()=\'table\') and (preceding-sibling::h2[1][.=\''+majorTitle+'\'])]'):
				links = sel.xpath('a/text()').extract()
				subHeading = sel.xpath('span/text()').extract()
				description = sel.xpath('text()').extract()
				tableTxt = sel.xpath('tr/td')
				# If there are links then they must be put into the description
				if links != empty:
					# Fix for links that are not adding if it's the first index
					offset = 1
					if description[0].encode('utf-8').startswith('–'):
						offset = 0
					for i in range(len(links)):
						description.insert((2*i)+offset, links[i])
					description = ''.join(description)
					descs.insert(len(descs), description.encode('utf-8'))
				elif subHeading != empty and description != empty:
					for i in range(0, len(subHeading)):
						subHeadingStr = subHeadingStr + subHeading[i] + description[i]
				elif description != empty:
					if ignore not in description:
						if subHeadingStr != emptyStr:
							subHeadingStr = subHeadingStr + ' ' + description[0]
						else:
							descs.insert(len(descs), description[0].encode('utf-8'))
				elif subHeading != empty:
					descs.insert(len(descs), subHeading[0].encode('utf-8'))
				if tableTxt != empty:
					# Fix for table description that doesn't have any links
					tableSel = tableTxt.xpath('p/a/text()')
					if tableSel == empty or tableSel == None:
						tableSel = tableTxt.xpath('p/text()').extract()
						tableDesc = ''
						for uTxt in tableSel:
							if nbsp not in uTxt:
								tableDesc += ' ' + uTxt.encode('utf-8')
						print 'tableDescNoLink: ', tableDesc
						stripChars  = ' '
						if subHeadingStr != emptyStr:
							reqs.insert(len(reqs), subHeadingStr)
							stripChars += subHeadingStr[-11].encode('utf-8') + subHeadingStr[-10].encode('utf-8')
							subHeadingStr = ''
						reqs.insert(len(reqs), tableDesc.strip(stripChars))

					# If the table consists of a list of subjects or is a description with links
					else:
						tableDesc = tableTxt.xpath('p/text()').extract()
						tableSel = tableSel.extract()
						if subHeadingStr != emptyStr:
							reqs.insert(len(reqs), subHeadingStr.encode('utf-8'))
							subHeadingStr = ''
						for i in range(len(tableSel)):
							if nbsp in tableDesc[i]:
								# special condition where its a description with links
								tableDesc = tableTxt.xpath('p/text()').extract()
								del tableDesc[0]
								del tableDesc[-1]
								print tableDesc
								print tableSel
								for j in range(len(tableSel)):
									tableDesc.insert((2*j)+1, tableSel[j].encode('utf-8'))
								tableDesc = ''.join(tableDesc)
								tableSel = tableDesc
								break
							tableSel[i] = tableSel[i].encode('utf-8')
						reqs.insert(len(reqs), tableSel)
						print 'subjList or descWithLink: ', tableSel
				print 'subHeadingStr: ', subHeadingStr
				print 'links: ', links
				print 'subHeading: ', subHeading
				print 'desc: ', description
				print
			print descs
			print reqs
			yield Major(title=majorTitle, department=department, description=descs, requirements=reqs)
			descs = []
			reqs = []

		# Used to get minor names
		# descs = []
		# reqs = []
		# for selector in response.xpath('//h3[re:test(., \'Minor in\', \'i\')]'):
		# 	minorTitle = selector.xpath('text()').extract()[0]
		# 	subHeadingStr = ''
		# 	for sel in selector.xpath('//*[(name()=\'p\' or name()=\'table\') and (preceding-sibling::h3[1][.=\''+minorTitle+'\'])]'):
		# 		links = sel.xpath('a/text()').extract()
		# 		subHeading = sel.xpath('span/text()').extract()
		# 		description = sel.xpath('text()').extract()
		# 		tableTxt = sel.xpath('tr/td')
		# 		# If there are links then they must be put into the description
		# 		if links != empty:
		# 			# Fix for links that are not adding if it's the first index
		# 			offset = 1
		# 			if description[0].encode('utf-8').startswith('–'):
		# 				offset = 0
		# 			for i in range(len(links)):
		# 				description.insert((2*i)+offset, links[i])
		# 			description = ''.join(description)
		# 			descs.insert(len(descs), description.encode('utf-8'))
		# 		elif subHeading != empty and description != empty:
		# 			for i in range(0, len(subHeading)):
		# 				subHeadingStr = subHeadingStr + subHeading[i] + description[i]
		# 		elif description != empty:
		# 			if ignore not in description:
		# 				if subHeadingStr != emptyStr:
		# 					subHeadingStr = subHeadingStr + ' ' + description[0]
		# 				else:
		# 					descs.insert(len(descs), description[0].encode('utf-8'))
		# 		elif subHeading != empty:
		# 			descs.insert(len(descs), subHeading[0].encode('utf-8'))
		# 		if tableTxt != empty:
		# 			# Fix for table description that doesn't have any links
		# 			tableSel = tableTxt.xpath('p/a/text()')
		# 			if tableSel == empty or tableSel == None:
		# 				tableSel = tableTxt.xpath('p/text()').extract()
		# 				tableDesc = ''
		# 				for uTxt in tableSel:
		# 					if nbsp not in uTxt:
		# 						tableDesc += ' ' + uTxt.encode('utf-8')
		# 				print tableDesc
		# 				reqs.insert(len(reqs), tableDesc.strip(' '))

		# 			# If the table consists of a list of subjects or is a description with links
		# 			else:
		# 				tableDesc = tableTxt.xpath('p/text()').extract()
		# 				tableSel = tableSel.extract()
		# 				reqs.insert(len(reqs), subHeadingStr.encode('utf-8'))
		# 				subHeadingStr = ''
		# 				for i in range(len(tableSel)):
		# 					if nbsp in tableDesc[i]:
		# 						# special condition where its a description with links
		# 						tableDesc = tableTxt.xpath('p/text()').extract()
		# 						del tableDesc[0]
		# 						del tableDesc[-1]
		# 						print tableDesc
		# 						print tableSel
		# 						for j in range(len(tableSel)):
		# 							tableDesc.insert((2*j)+1, tableSel[j].encode('utf-8'))
		# 						tableDesc = ''.join(tableDesc)
		# 						tableSel = tableDesc
		# 						break
		# 					tableSel[i] = tableSel[i].encode('utf-8')
		# 				reqs.insert(len(reqs), tableSel)
		# 				print tableSel
		# 		print subHeadingStr
		# 		print links
		# 		print subHeading
		# 		print description
		# 		print
		# 	print descs
		# 	print reqs
		# 	yield Minor(title=minorTitle, department='Schmid College of Science and Technology', description=descs, requirements=reqs)
		# 	descs = []
		# 	reqs = []

		# # Used to get courses
		# subject = ''
		# number = -1
		# name = ' '
		# heading = True
		# for sel in response.xpath('//*[(name()=\'h3\' and re:test(., \'^((?!Minor in).)*$\', \'i\')) or (name()=\'p\' and @class=\'coursedescription\')]'):
		# 	if (heading == True):
		# 		courseStr = sel.xpath('text()').extract()[0]
		# 		courseStr = courseStr.split()
		# 		name = ' '.join(courseStr[2:])
		# 		subject = courseStr[0]
		# 		number = courseStr[1]
		# 		heading = False
		# 	else:
		# 		prerequisites = sel.xpath('a/text()').extract()
		# 		description = sel.xpath('text()').extract()
		# 		if (prerequisites != empty):
		# 			for i in range(len(prerequisites)):
		# 				description.insert((2*i)+1, prerequisites[i])
		# 		description = ''.join(description)
		# 		heading = True
		# 		yield Course(subject=subject, number=number, name=name, description=description)
<|endoftext|>


<commit_before>import os
import h5py
import numpy as np
import bisect
import sys
import matplotlib.pyplot as plt

# import our MS libraries
sys.path.append('/Users/palmer/Documents/python_codebase/')
from pyMS.mass_spectrum import mass_spectrum
from pyIMS.ion_datacube import ion_datacube
class inMemoryIMS_hdf5():
    def __init__(self,filename,min_mz=0,max_mz=np.inf,index_range=[]):
        file_size = os.path.getsize(filename)
        self.load_file(filename,min_mz,max_mz)

    def load_file(self,filename,min_mz=0,max_mz=np.inf,min_int=0.):
        # parse file to get required parameters
        # can use thin hdf5 wrapper for getting data from file
        self.file_dir, self.filename = file_type=os.path.splitext(filename)
        self.file_type = file_type
        self.hdf = h5py.File(filename,'r')   #Readonly, file must exist
	if index_range == []:
        	self.index_list = map(int,self.hdf['/spectral_data'].keys())
	else:
		self.index_list = index_range
        self.coords = self.get_coords()
        # load data into memory
        self.mz_list = []
        self.count_list = []
        self.idx_list = []
        for ii in self.index_list:
            # load spectrum, keep values gt0 (shouldn't be here anyway)
            this_spectrum = self.get_spectrum(ii)
            mzs,counts = this_spectrum.get_spectrum(source='centroids')
	    counts=counts[mzs>min_mz]
	    counts=counts[mzs<max_mz]
	    mzs=mzs[mzs>min_mz]
	    mzs=mzs[mzs<max_mz]
            ct_gt0 = counts>min_int
            mzs = mzs[ct_gt0]
            counts=counts[ct_gt0]
	    idx = ii*np.ones((len(mzs),),dtype =int)
            # append ever-growing lists (should probably be preallocated or piped to disk and re-loaded)
	    mz_merged=[]
	    counts_merged=[]
	    idx_merged=[]
            while ~isempty(mzs) & ~isempty(mz_list):
		# clean up if one list is empty
		if isempty(mzs):
			while ~isempty(mz_list):
				mz_merged.append(mz_list.pop())
		if isempty(mz_list):
			while ~isempty(mzs):
				mz_merged.append(mzs.pop())
		if mzs[-1]>mz_list[-1]:
			mz_merged.append(mzs.pop())
			counts_merged.append(counts.pop())
			idx_merged.append(idx.pop())
		else:
			mz_merged.append(mz_list.pop())
			counts_merged.append(count_list.pop())
			idx_merged.append(idx_list.pop())
		mz_merged.reverse
		count_merged.reverse()
		idx_merged.reverse()
		mz_list = mz_merged
		count_list=count_merged
		idx_list=idx_merged
            for c in list(counts):
                self.count_list.append(c)
            for ix in ii*np.ones((len(mzs),),dtype =int):
                self.idx_list.append(ix)
            #if ii > 200:
            #    break
        # to np arrays
	print 'loaded spectra'
        self.mz_list = np.asarray(self.mz_list)
        self.count_list = np.asarray(self.count_list)
        self.idx_list = np.asarray(self.idx_list)
        # sort by mz for fast image formation
        mz_order = np.argsort(self.mz_list)
        self.mz_list = self.mz_list[mz_order]
        self.count_list = self.count_list[mz_order]
        self.idx_list = self.idx_list[mz_order]
        print 'file loaded'
    def get_coords(self):
        coords = np.zeros((len(self.index_list),3))
        for k in self.index_list:
            coords[k,:] = self.hdf['/spectral_data/'+str(k)+'/coordinates/']
        return coords
    def get_spectrum(self,index):
        this_spectrum = mass_spectrum()
        tmp_str='/spectral_data/%d/' %(index)
        this_spectrum.add_centroids(self.hdf[tmp_str+'/centroid_mzs/'],self.hdf[tmp_str+'/centroid_intensities/'])
        return this_spectrum
    def get_ion_image(self,mzs,tols,tol_type='ppm'):
        tols = tols*mzs/1e6 # to m/z
        data_out = ion_datacube()
        data_out.add_coords(self.coords)
        for mz,tol in zip(mzs,tols):
            mz_upper = mz + tol
            mz_lower = mz - tol
            print mz_lower
            print mz_upper
            idx_left = bisect.bisect_left(self.mz_list,mz_lower)
            idx_right = bisect.bisect_right(self.mz_list,mz_upper)
            # slice list for code clarity
            count_vect = np.concatenate((np.asarray([0]),self.count_list[idx_left:idx_right],np.asarray([0])))
            idx_vect = np.concatenate((np.asarray([0]),self.idx_list[idx_left:idx_right],np.asarray([max(self.index_list)])))
            # bin vectors
            ion_vect=np.bincount(idx_vect,count_vect)
            data_out.add_xic(ion_vect,[mz],[tol])
        return data_out
<commit_msg>_dev should be branch, not file<commit_after><|endoftext|>