<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class EcpDataVisSdk(BundlePackage):
    """ECP Data & Vis SDK"""

    homepage = "https://github.com/chuckatkins/ecp-data-viz-sdk"

    tags = ['ecp']
    maintainers = ['chuckatkins']

    version('1.0')

    ############################################################
    # Variants
    ############################################################

    # I/O
    variant('adios2', default=True, description="Enable ADIOS2")
    variant('darshan', default=True, description="Enable Darshan")
    variant('faodel', default=False, description="Enable FAODEL")
    variant('hdf5', default=True, description="Enable HDF5")
    variant('pnetcdf', default=True, description="Enable PNetCDF")
    variant('unifyfs', default=True, description="Enable UnifyFS")
    variant('veloc', default=True, description="Enable VeloC")

    # Vis
    variant('ascent', default=False, description="Enable Ascent")
    variant('paraview', default=False, description="Enable ParaView")
    variant('sz', default=True, description="Enable SZ")
    variant('vtkm', default=False, description="Enable VTK-m")
    variant('zfp', default=True, description="Enable ZFP")

    # Outstanding concretization issues
    # variant('cinema', default=True, description="Enable Cinema")

    # Outstanding build issues
    # variant('catalyst', default=False, description="Enable Catalyst")
    # variant('visit', default=False, description="Enable VisIt")

    ############################################################
    # This is a messy workaround until the clingo concretizer can be required.
    # The intent is to map package variants to dependency variants:
    #   Package variants a, and b, mapping to dependency variants A and B
    #   produce the following set of dependencies:
    #     depends_on('foo+A+B', when='+a+b')
    #     depends_on('foo+A~B', when='+a~b')
    #     depends_on('foo~A+B', when='~a+b')
    #     depends_on('foo~A~B', when='~a~b')
    #   The clingo concretizer will allow that to be expressed much simpler by
    #   only considering defaults once everything else is resolved:
    #     depends_on('foo')
    #     depends_on('foo+A', when='+a')
    #     depends_on('foo+B', when='+b')
    ############################################################

    # Helper function to generate dependencies on the Cartesian product of
    # variants.  If a dictionary is passed then it provides a mapping of
    # package variant name to dependency variant name.  Otherwise assume they
    # are the same variant name in both the package and dependency
    def variants2deps(dep_spec, pkg_spec, variants):
        if not type(variants) is dict:
            variants = dict([(v, v) for v in variants])
        n = len(variants)
        for i in range(0, pow(2, n)):
            state = ['+' if d == '1' else '~' for d in format(i, '0' + str(n) + 'b')]
            [pkg_vars, dep_vars] = [''.join(v) for v in zip(
                *[(s + pv, s + dv) for s, (pv, dv) in zip(state, variants.items())])]
            dependency = ' '.join((dep_spec, dep_vars))
            predicate = ' '.join((pkg_spec, pkg_vars))
            depends_on(dependency, when=predicate)

    ############################################################
    # Dependencies
    ############################################################
    variants2deps('adios2+shared+mpi+fortran+python+blosc+sst+ssc+dataman',
                  '+adios2', ['hdf5', 'sz', 'zfp'])

    depends_on('darshan-runtime+mpi', when='+darshan')
    depends_on('darshan-util', when='+darshan')

    variants2deps('faodel+shared+mpi network=libfabric', '+faodel', ['hdf5'])

    depends_on('hdf5 +shared+mpi', when='+hdf5')
    # +fortran breaks the concretizer... Needs new concretizer
    # depends_on('hdf5 +shared+mpi+fortran', when='+hdf5')

    depends_on('parallel-netcdf+shared+fortran', when='+pnetcdf')

    variants2deps('unifyfs+fortran', '+unifyfs', ['hdf5'])

    depends_on('veloc', when='+veloc')

    depends_on('ascent+shared+mpi+fortran+openmp+python+vtkh+dray', when='+ascent')
    depends_on('catalyst', when='+catalyst')

    depends_on('py-cinema-lib', when='+cinema')
    depends_on('py-cinemasci', when='+cinema')

    variants2deps('paraview+shared+mpi+python3+kits', '+paraview', ['hdf5'])
    # +adios2 is not yet enabled in the paraview package
    # depends_on('paraview+adios2', when='+paraview +adios2')

    depends_on('visit', when='+visit')

    depends_on('vtk-m+shared+mpi+openmp+rendering', when='+vtkm')

    # +python is currently broken in sz
    # variants2deps('sz+shared+fortran+python+random_access', '+sz', ['hdf5'])
    variants2deps('sz+shared+fortran+random_access', '+sz', ['hdf5'])

    depends_on('zfp', when='+zfp')
<commit_msg>ecp-data-vis-sdk: Disable +fortran for unifyfs (#24096)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class EcpDataVisSdk(BundlePackage):
    """ECP Data & Vis SDK"""

    homepage = "https://github.com/chuckatkins/ecp-data-viz-sdk"

    tags = ['ecp']
    maintainers = ['chuckatkins']

    version('1.0')

    ############################################################
    # Variants
    ############################################################

    # I/O
    variant('adios2', default=True, description="Enable ADIOS2")
    variant('darshan', default=True, description="Enable Darshan")
    variant('faodel', default=False, description="Enable FAODEL")
    variant('hdf5', default=True, description="Enable HDF5")
    variant('pnetcdf', default=True, description="Enable PNetCDF")
    variant('unifyfs', default=True, description="Enable UnifyFS")
    variant('veloc', default=True, description="Enable VeloC")

    # Vis
    variant('ascent', default=False, description="Enable Ascent")
    variant('paraview', default=False, description="Enable ParaView")
    variant('sz', default=True, description="Enable SZ")
    variant('vtkm', default=False, description="Enable VTK-m")
    variant('zfp', default=True, description="Enable ZFP")

    # Outstanding concretization issues
    # variant('cinema', default=True, description="Enable Cinema")

    # Outstanding build issues
    # variant('catalyst', default=False, description="Enable Catalyst")
    # variant('visit', default=False, description="Enable VisIt")

    ############################################################
    # This is a messy workaround until the clingo concretizer can be required.
    # The intent is to map package variants to dependency variants:
    #   Package variants a, and b, mapping to dependency variants A and B
    #   produce the following set of dependencies:
    #     depends_on('foo+A+B', when='+a+b')
    #     depends_on('foo+A~B', when='+a~b')
    #     depends_on('foo~A+B', when='~a+b')
    #     depends_on('foo~A~B', when='~a~b')
    #   The clingo concretizer will allow that to be expressed much simpler by
    #   only considering defaults once everything else is resolved:
    #     depends_on('foo')
    #     depends_on('foo+A', when='+a')
    #     depends_on('foo+B', when='+b')
    ############################################################

    # Helper function to generate dependencies on the Cartesian product of
    # variants.  If a dictionary is passed then it provides a mapping of
    # package variant name to dependency variant name.  Otherwise assume they
    # are the same variant name in both the package and dependency
    def variants2deps(dep_spec, pkg_spec, variants):
        if not type(variants) is dict:
            variants = dict([(v, v) for v in variants])
        n = len(variants)
        for i in range(0, pow(2, n)):
            state = ['+' if d == '1' else '~' for d in format(i, '0' + str(n) + 'b')]
            [pkg_vars, dep_vars] = [''.join(v) for v in zip(
                *[(s + pv, s + dv) for s, (pv, dv) in zip(state, variants.items())])]
            dependency = ' '.join((dep_spec, dep_vars))
            predicate = ' '.join((pkg_spec, pkg_vars))
            depends_on(dependency, when=predicate)

    ############################################################
    # Dependencies
    ############################################################
    variants2deps('adios2+shared+mpi+fortran+python+blosc+sst+ssc+dataman',
                  '+adios2', ['hdf5', 'sz', 'zfp'])

    depends_on('darshan-runtime+mpi', when='+darshan')
    depends_on('darshan-util', when='+darshan')

    variants2deps('faodel+shared+mpi network=libfabric', '+faodel', ['hdf5'])

    depends_on('hdf5 +shared+mpi', when='+hdf5')
    # +fortran breaks the concretizer... Needs new concretizer
    # depends_on('hdf5 +shared+mpi+fortran', when='+hdf5')

    depends_on('parallel-netcdf+shared+fortran', when='+pnetcdf')

    variants2deps('unifyfs', '+unifyfs ', ['hdf5'])

    depends_on('veloc', when='+veloc')

    depends_on('ascent+shared+mpi+fortran+openmp+python+vtkh+dray', when='+ascent')
    depends_on('catalyst', when='+catalyst')

    depends_on('py-cinema-lib', when='+cinema')
    depends_on('py-cinemasci', when='+cinema')

    variants2deps('paraview+shared+mpi+python3+kits', '+paraview', ['hdf5'])
    # +adios2 is not yet enabled in the paraview package
    # depends_on('paraview+adios2', when='+paraview +adios2')

    depends_on('visit', when='+visit')

    depends_on('vtk-m+shared+mpi+openmp+rendering', when='+vtkm')

    # +python is currently broken in sz
    # variants2deps('sz+shared+fortran+python+random_access', '+sz', ['hdf5'])
    variants2deps('sz+shared+fortran+random_access', '+sz', ['hdf5'])

    depends_on('zfp', when='+zfp')
<|endoftext|>


<commit_before># License: none (public domain)

import AlamoDrafthouseAPI as api
import dateutil.parser
import errno
import gzip
import hashlib
import logging
import os
import pickle
import smtplib
import tinycss

from bs4 import BeautifulSoup
from datetime import datetime, timedelta
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from itertools import groupby, count
from InlineCSS import styled
from GoogleCalendar import GoogleCalendar

RESOURCES_DIRECTORY = 'resources'
CREDENTIALS_FILE = os.path.join(RESOURCES_DIRECTORY, 'config', 'credentials.dat')
PICKLE_FILE = os.path.join(RESOURCES_DIRECTORY, 'cache', 'pancake.pickle')
RECIPIENTS_FILE = os.path.join(RESOURCES_DIRECTORY, 'config', 'pancake.list')
STYLE_FILE = os.path.join(RESOURCES_DIRECTORY, 'css', 'pancake.css')
TEMPLATE_FILE = os.path.join(RESOURCES_DIRECTORY, 'template', 'pancake.html')

DATE_FORMAT = '%A, %B %d, %Y'
TIME_FORMAT = '%I:%M%p'

log = logging.getLogger(__name__)

def date_string(dt):
    """Returns a date string representation of the given datetime object."""
    return datetime.strftime(dt, DATE_FORMAT)

def time_string(dt):
    """Returns a time string representation of the given datetime object."""
    time = datetime.strftime(dt, TIME_FORMAT).lstrip('0')
    return time[:-2] + 'p' if time.endswith('PM') else time[:-2]

def datetime_string(dt):
    """Returns a date and time string representation of the given datetime object."""
    return date_string(dt) + ' - ' + time_string(dt)

def pancake_sort_key(pancake):
    """Key to sort pancakes by film title, cinema location, and datetime of the show."""
    return pancake['film'], pancake['cinema'], pancake['datetime']

def html_showtimes(pancakes):
    """Returns a list of pancake showtimes, as pancake HTML."""
    showtimes = []
    for pancake in pancakes:
        soup = BeautifulSoup('')
        soup.span['class'] = pancake['status']

        if pancake['status'] == 'onsale':
            anchor = soup.new_tag('a', href=pancake['url'])
            anchor.append(time_string(pancake['datetime']))
            soup.span.append(anchor)
        else:  # pancake['status'] == 'soldout' or pancake['status'] == 'notonsale'
            soup.span.append(time_string(pancake['datetime']))

        showtimes.append(str(soup))
    return showtimes

def html_digest(pancakes):
    """Returns pancake styled HTML digest of the given pancakes."""
    pancakes = sorted(pancakes, key=pancake_sort_key)

    # things to group by
    by_film_and_cinema = lambda p: (p['film_uid'], p['film'], p['cinema_url'], p['cinema'])
    by_day = lambda p: p['datetime'].date()

    soup = BeautifulSoup('')
    for key, pancakes in groupby(pancakes, key=by_film_and_cinema):
        film_uid, film, cinema_url, cinema = key

        film_heading = BeautifulSoup('

')
        film_heading.h1['class'] = 'film_heading'
        film_heading.a['href'] = 'https://drafthouse.com/uid/' + film_uid
        film_heading.a.append(film)

        if cinema_url:
            cinema_heading = BeautifulSoup('

')
            cinema_heading.h2['class'] = 'cinema_heading'
            cinema_heading.a['href'] = cinema_url
            cinema_heading.a.append(cinema)
        else:
            cinema_heading = BeautifulSoup('

')
            cinema_heading.h2['class'] = 'cinema_heading'
            cinema_heading.h2.append(cinema)

        item_data = []
        for day, pancakes in groupby(pancakes, key=by_day):
            item_data.append((date_string(day), ', '.join(html_showtimes(pancakes))))

        item_list = BeautifulSoup('

')
        item_list.ul['class'] = 'pancake_items'
        for data, n in zip(item_data, count(1)):
            day, showtimes = data

            item = item_list.new_tag('li')
            item['class'] = 'pancake_item'

            item_content = '{day} - {showtimes}'.format(day=day, showtimes=showtimes)
            item.append(BeautifulSoup(item_content))
            item_list.ul.append(item)

        soup.append(film_heading)
        soup.append(cinema_heading)
        soup.append(item_list)

    content = str(soup)

    # load CSS stylesheet
    try:
        parser = tinycss.make_parser('page3')
        stylesheet = parser.parse_stylesheet_file(STYLE_FILE)
        style = {
            r.selector.as_css(): {
                d.name: d.value.as_css() for d in r.declarations
            } for r in stylesheet.rules
        }
    except Exception as e:
        log.warn('could not load CSS style file: {}'.format(e))
        style = None

    # load HTML template
    try:
        template = open(TEMPLATE_FILE, 'r').read()
        return styled(template.format(content=content), style)
    except Exception as e:
        log.warn('could not load HTML template file: {}'.format(e))
        return styled(content, style)

def text_digest(pancakes):
    """Returns a plain text digest of the given pancakes."""
    text = ''
    for pancake in sorted(pancakes, key=pancake_sort_key):
        if pancake['status'] == 'onsale':
            status = 'On sale now!'
        elif pancake['status'] == 'soldout':
            status = 'Sold out.'
        else:  # pancake['status'] == 'notonsale'
            status = 'Not on sale yet.'
        params = (
            pancake['film'].encode('utf-8'),
            pancake['cinema'],
            date_string(pancake['datetime']),
            time_string(pancake['datetime']),
            status,
        )
        text += '{}\n{}\n{}\n{}\n{}'.format(*params)
        if pancake['status'] == 'onsale':
            text += '\n{}'.format(pancake['url'])
        text += '\n\n'
    return text

def notify(pancakes, recipients):
    """Sends digest email(s) to recipients given pancakes (no email sent if pancakes is empty)."""
    if not pancakes:
        return

    plain = text_digest(pancakes)
    log.info('digest:\n{}'.format(plain))

    if not recipients:
        return

    msg = MIMEMultipart('alternative')
    msg['Subject'] = 'Pancake Master: {}'.format(datetime_string(datetime.now()))
    msg['To'] = ', '.join(recipients)
    msg['From'] = recipients[0]
    msg.attach(MIMEText(plain, 'plain'))
    msg.attach(MIMEText(html_digest(pancakes), 'html'))

    try:
        s = smtplib.SMTP('localhost')
        s.sendmail(msg['From'], recipients, msg.as_string())
        s.quit()
        log.info('sent email(s) to {}'.format(', '.join(recipients)))
    except Exception as e:
        log.error('email fail: {}'.format(e))
        raise

def pancake_key(pancake):
    """Creates a unique id for a given pancake."""
    m = hashlib.md5()
    m.update(pancake['film'])
    m.update(pancake['cinema'])
    m.update(datetime_string(pancake['datetime']))
    return m.hexdigest()

def save_database(db):
    """Saves pancake database to disk."""
    filename = PICKLE_FILE
    log.info('saving {}'.format(filename))
    try:
        with gzip.GzipFile(filename, 'wb') as f:
            f.write(pickle.dumps(db, 1))
    except Exception as e:
        log.error('load failure: {}'.format(e))
        raise

def load_database():
    """Unpickles and decompresses the given file and returns the created object."""
    filename = PICKLE_FILE
    log.info('loading {}'.format(filename))

    try:
        with gzip.GzipFile(filename, 'rb') as f:
            buf = ''
            while True:
                data = f.read()
                if data == '':
                    break
                buf += data
            return pickle.loads(buf)
    except Exception as e:
        log.error('load failure: {}'.format(e))
        log.warn('creating new pancake database...')
    return {}

def update_pancakes(db, pancakes):
    """Updates database given the list of all pancakes, returns list of updated pancakes."""
    updated = []
    for pancake in pancakes:
        key = pancake_key(pancake)

        if key in db:
            if db[key]['status'] == 'notonsale' and pancake['status'] == 'onsale':
                updated.append(pancake)
        else:
            updated.append(pancake)

        db[key] = pancake
    return updated

def prune_database(db):
    """Removes old pancakes from the database."""
    for key, pancake in db.items():
        if pancake['datetime'].date() < datetime.now().date():
            del db[key]

def update_calendar(pancakes, market_timezone):
    """Updates our Google Calendar with Master Pancake events."""
    if not pancakes:
        return

    gcal = GoogleCalendar(CREDENTIALS_FILE)
    calendar = next((c for c in gcal.calendar_list() if c['summary'] == 'Master Pancakes'), None)

    if not calendar:
        raise Exception("Can't find 'Master Pancakes' calendar")

    events = gcal.events(calendar['id'])

    for pancake in pancakes:
        start_time = pancake['datetime']
        end_time = start_time + timedelta(hours=2)  # Master Pancakes typically run 2 hours

        if pancake['status'] == 'notonsale':
            event_status = 'Not yet on sale.'
        elif pancake['status'] == 'onsale':
            event_status = 'On sale now!'
        else:  # pancake['status'] == 'soldout':
            event_status = 'Sold out.'

        for event in events:
            event_datetime = dateutil.parser.parse(event['start']['dateTime'])
            if (
                event['summary'] == pancake['film']
                and event['location'] == pancake['cinema']
                and event_datetime == start_time
            ):
                if event['description'].starts_with(event_status):
                    log.info('{} already in calendar'.format(pancake['film']))
                    break
                else:
                    log.warn('TODO: {} should be updated'.format(pancake['film']))
        else:
            newevent = {
                'summary': pancake['film'],
                'location': pancake['cinema'],
                'description': event_status,
                'start': {
                    'dateTime': start_time.isoformat(),
                    'timeZone': market_timezone.zone
                },
                'end': {
                    'dateTime': end_time.isoformat(),
                    'timeZone': market_timezone.zone
                }
            }
            if pancake['url']:
                newevent['description'] += '\n' + pancake['url']
            gcal.insert_event(calendar['id'], newevent)

def load_rerecipients():
    """Returns list of email addresses to notify."""
    try:
        return [line.strip() for line in open(RECIPIENTS_FILE)]
    except:
        log.warn('no email recipients found, not sending email notifications...')
    return []

def mkdir_p(path):
    """Make directory without error if it already exists."""
    try:
        os.makedirs(path, exist_ok=True)  # python 3.2+
    except TypeError as e:
        try:
            os.makedirs(path)
        except OSError as e:  # python <2.5
            if e.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                raise

def clear_cache():
    """Deletes existing pancake database."""
    try:
        os.remove(PICKLE_FILE)
    except:
        log.exception('clearing cache:')

def main(market, market_timezone, disable_notify=False, disable_calendar=False):
    """Fetches pancake data, send notifications, and reports updates."""
    mkdir_p(os.path.join(RESOURCES_DIRECTORY, 'config'))
    mkdir_p(os.path.join(RESOURCES_DIRECTORY, 'cache'))

    db = load_database()
    recipients = load_rerecipients()

    try:
        pancakes = api.query_pancakes(market, market_timezone)
    except:
        pancakes = []
        log.exception('api error:')

    updated = update_pancakes(db, pancakes)

    if not disable_calendar:
        try:
            update_calendar(updated, market_timezone)
        except:
            log.exception('calendar error:')

    if not disable_notify:
        try:
            notify(updated, recipients)
        except:
            log.exception('notification error:')

    prune_database(db)
    save_database(db)
<commit_msg>Fixed typo: unicode.starts_with should be unicode.startswith.<commit_after># License: none (public domain)

import AlamoDrafthouseAPI as api
import dateutil.parser
import errno
import gzip
import hashlib
import logging
import os
import pickle
import smtplib
import tinycss

from bs4 import BeautifulSoup
from datetime import datetime, timedelta
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from itertools import groupby, count
from InlineCSS import styled
from GoogleCalendar import GoogleCalendar

RESOURCES_DIRECTORY = 'resources'
CREDENTIALS_FILE = os.path.join(RESOURCES_DIRECTORY, 'config', 'credentials.dat')
PICKLE_FILE = os.path.join(RESOURCES_DIRECTORY, 'cache', 'pancake.pickle')
RECIPIENTS_FILE = os.path.join(RESOURCES_DIRECTORY, 'config', 'pancake.list')
STYLE_FILE = os.path.join(RESOURCES_DIRECTORY, 'css', 'pancake.css')
TEMPLATE_FILE = os.path.join(RESOURCES_DIRECTORY, 'template', 'pancake.html')

DATE_FORMAT = '%A, %B %d, %Y'
TIME_FORMAT = '%I:%M%p'

log = logging.getLogger(__name__)

def date_string(dt):
    """Returns a date string representation of the given datetime object."""
    return datetime.strftime(dt, DATE_FORMAT)

def time_string(dt):
    """Returns a time string representation of the given datetime object."""
    time = datetime.strftime(dt, TIME_FORMAT).lstrip('0')
    return time[:-2] + 'p' if time.endswith('PM') else time[:-2]

def datetime_string(dt):
    """Returns a date and time string representation of the given datetime object."""
    return date_string(dt) + ' - ' + time_string(dt)

def pancake_sort_key(pancake):
    """Key to sort pancakes by film title, cinema location, and datetime of the show."""
    return pancake['film'], pancake['cinema'], pancake['datetime']

def html_showtimes(pancakes):
    """Returns a list of pancake showtimes, as pancake HTML."""
    showtimes = []
    for pancake in pancakes:
        soup = BeautifulSoup('')
        soup.span['class'] = pancake['status']

        if pancake['status'] == 'onsale':
            anchor = soup.new_tag('a', href=pancake['url'])
            anchor.append(time_string(pancake['datetime']))
            soup.span.append(anchor)
        else:  # pancake['status'] == 'soldout' or pancake['status'] == 'notonsale'
            soup.span.append(time_string(pancake['datetime']))

        showtimes.append(str(soup))
    return showtimes

def html_digest(pancakes):
    """Returns pancake styled HTML digest of the given pancakes."""
    pancakes = sorted(pancakes, key=pancake_sort_key)

    # things to group by
    by_film_and_cinema = lambda p: (p['film_uid'], p['film'], p['cinema_url'], p['cinema'])
    by_day = lambda p: p['datetime'].date()

    soup = BeautifulSoup('')
    for key, pancakes in groupby(pancakes, key=by_film_and_cinema):
        film_uid, film, cinema_url, cinema = key

        film_heading = BeautifulSoup('

')
        film_heading.h1['class'] = 'film_heading'
        film_heading.a['href'] = 'https://drafthouse.com/uid/' + film_uid
        film_heading.a.append(film)

        if cinema_url:
            cinema_heading = BeautifulSoup('

')
            cinema_heading.h2['class'] = 'cinema_heading'
            cinema_heading.a['href'] = cinema_url
            cinema_heading.a.append(cinema)
        else:
            cinema_heading = BeautifulSoup('

')
            cinema_heading.h2['class'] = 'cinema_heading'
            cinema_heading.h2.append(cinema)

        item_data = []
        for day, pancakes in groupby(pancakes, key=by_day):
            item_data.append((date_string(day), ', '.join(html_showtimes(pancakes))))

        item_list = BeautifulSoup('

')
        item_list.ul['class'] = 'pancake_items'
        for data, n in zip(item_data, count(1)):
            day, showtimes = data

            item = item_list.new_tag('li')
            item['class'] = 'pancake_item'

            item_content = '{day} - {showtimes}'.format(day=day, showtimes=showtimes)
            item.append(BeautifulSoup(item_content))
            item_list.ul.append(item)

        soup.append(film_heading)
        soup.append(cinema_heading)
        soup.append(item_list)

    content = str(soup)

    # load CSS stylesheet
    try:
        parser = tinycss.make_parser('page3')
        stylesheet = parser.parse_stylesheet_file(STYLE_FILE)
        style = {
            r.selector.as_css(): {
                d.name: d.value.as_css() for d in r.declarations
            } for r in stylesheet.rules
        }
    except Exception as e:
        log.warn('could not load CSS style file: {}'.format(e))
        style = None

    # load HTML template
    try:
        template = open(TEMPLATE_FILE, 'r').read()
        return styled(template.format(content=content), style)
    except Exception as e:
        log.warn('could not load HTML template file: {}'.format(e))
        return styled(content, style)

def text_digest(pancakes):
    """Returns a plain text digest of the given pancakes."""
    text = ''
    for pancake in sorted(pancakes, key=pancake_sort_key):
        if pancake['status'] == 'onsale':
            status = 'On sale now!'
        elif pancake['status'] == 'soldout':
            status = 'Sold out.'
        else:  # pancake['status'] == 'notonsale'
            status = 'Not on sale yet.'
        params = (
            pancake['film'].encode('utf-8'),
            pancake['cinema'],
            date_string(pancake['datetime']),
            time_string(pancake['datetime']),
            status,
        )
        text += '{}\n{}\n{}\n{}\n{}'.format(*params)
        if pancake['status'] == 'onsale':
            text += '\n{}'.format(pancake['url'])
        text += '\n\n'
    return text

def notify(pancakes, recipients):
    """Sends digest email(s) to recipients given pancakes (no email sent if pancakes is empty)."""
    if not pancakes:
        return

    plain = text_digest(pancakes)
    log.info('digest:\n{}'.format(plain))

    if not recipients:
        return

    msg = MIMEMultipart('alternative')
    msg['Subject'] = 'Pancake Master: {}'.format(datetime_string(datetime.now()))
    msg['To'] = ', '.join(recipients)
    msg['From'] = recipients[0]
    msg.attach(MIMEText(plain, 'plain'))
    msg.attach(MIMEText(html_digest(pancakes), 'html'))

    try:
        s = smtplib.SMTP('localhost')
        s.sendmail(msg['From'], recipients, msg.as_string())
        s.quit()
        log.info('sent email(s) to {}'.format(', '.join(recipients)))
    except Exception as e:
        log.error('email fail: {}'.format(e))
        raise

def pancake_key(pancake):
    """Creates a unique id for a given pancake."""
    m = hashlib.md5()
    m.update(pancake['film'])
    m.update(pancake['cinema'])
    m.update(datetime_string(pancake['datetime']))
    return m.hexdigest()

def save_database(db):
    """Saves pancake database to disk."""
    filename = PICKLE_FILE
    log.info('saving {}'.format(filename))
    try:
        with gzip.GzipFile(filename, 'wb') as f:
            f.write(pickle.dumps(db, 1))
    except Exception as e:
        log.error('load failure: {}'.format(e))
        raise

def load_database():
    """Unpickles and decompresses the given file and returns the created object."""
    filename = PICKLE_FILE
    log.info('loading {}'.format(filename))

    try:
        with gzip.GzipFile(filename, 'rb') as f:
            buf = ''
            while True:
                data = f.read()
                if data == '':
                    break
                buf += data
            return pickle.loads(buf)
    except Exception as e:
        log.error('load failure: {}'.format(e))
        log.warn('creating new pancake database...')
    return {}

def update_pancakes(db, pancakes):
    """Updates database given the list of all pancakes, returns list of updated pancakes."""
    updated = []
    for pancake in pancakes:
        key = pancake_key(pancake)

        if key in db:
            if db[key]['status'] == 'notonsale' and pancake['status'] == 'onsale':
                updated.append(pancake)
        else:
            updated.append(pancake)

        db[key] = pancake
    return updated

def prune_database(db):
    """Removes old pancakes from the database."""
    for key, pancake in db.items():
        if pancake['datetime'].date() < datetime.now().date():
            del db[key]

def update_calendar(pancakes, market_timezone):
    """Updates our Google Calendar with Master Pancake events."""
    if not pancakes:
        return

    gcal = GoogleCalendar(CREDENTIALS_FILE)
    calendar = next((c for c in gcal.calendar_list() if c['summary'] == 'Master Pancakes'), None)

    if not calendar:
        raise Exception("Can't find 'Master Pancakes' calendar")

    events = gcal.events(calendar['id'])

    for pancake in pancakes:
        start_time = pancake['datetime']
        end_time = start_time + timedelta(hours=2)  # Master Pancakes typically run 2 hours

        if pancake['status'] == 'notonsale':
            event_status = 'Not yet on sale.'
        elif pancake['status'] == 'onsale':
            event_status = 'On sale now!'
        else:  # pancake['status'] == 'soldout':
            event_status = 'Sold out.'

        for event in events:
            event_datetime = dateutil.parser.parse(event['start']['dateTime'])
            if (
                event['summary'] == pancake['film']
                and event['location'] == pancake['cinema']
                and event_datetime == start_time
            ):
                if event['description'].startswith(event_status):
                    log.info('{} already in calendar'.format(pancake['film']))
                    break
                else:
                    log.warn('TODO: {} should be updated'.format(pancake['film']))
        else:
            newevent = {
                'summary': pancake['film'],
                'location': pancake['cinema'],
                'description': event_status,
                'start': {
                    'dateTime': start_time.isoformat(),
                    'timeZone': market_timezone.zone
                },
                'end': {
                    'dateTime': end_time.isoformat(),
                    'timeZone': market_timezone.zone
                }
            }
            if pancake['url']:
                newevent['description'] += '\n' + pancake['url']
            gcal.insert_event(calendar['id'], newevent)

def load_rerecipients():
    """Returns list of email addresses to notify."""
    try:
        return [line.strip() for line in open(RECIPIENTS_FILE)]
    except:
        log.warn('no email recipients found, not sending email notifications...')
    return []

def mkdir_p(path):
    """Make directory without error if it already exists."""
    try:
        os.makedirs(path, exist_ok=True)  # python 3.2+
    except TypeError as e:
        try:
            os.makedirs(path)
        except OSError as e:  # python <2.5
            if e.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                raise

def clear_cache():
    """Deletes existing pancake database."""
    try:
        os.remove(PICKLE_FILE)
    except:
        log.exception('clearing cache:')

def main(market, market_timezone, disable_notify=False, disable_calendar=False):
    """Fetches pancake data, send notifications, and reports updates."""
    mkdir_p(os.path.join(RESOURCES_DIRECTORY, 'config'))
    mkdir_p(os.path.join(RESOURCES_DIRECTORY, 'cache'))

    db = load_database()
    recipients = load_rerecipients()

    try:
        pancakes = api.query_pancakes(market, market_timezone)
    except:
        pancakes = []
        log.exception('api error:')

    updated = update_pancakes(db, pancakes)

    if not disable_calendar:
        try:
            update_calendar(updated, market_timezone)
        except:
            log.exception('calendar error:')

    if not disable_notify:
        try:
            notify(updated, recipients)
        except:
            log.exception('notification error:')

    prune_database(db)
    save_database(db)
<|endoftext|>


<commit_before>from boadata.core import DataObject
from boadata.core.data_conversion import MethodConversion, DataConversion
from .mixins import GetItemMixin, StatisticsMixin, NumericalMixin, AsArrayMixin
import pandas as pd
import numpy as np
import types
from .. import wrap

class _PandasBase(DataObject, GetItemMixin, StatisticsMixin, NumericalMixin):
    def __to_csv__(self, uri, **kwargs):
        self.inner_data.to_csv(uri, **kwargs)
        klass = DataObject.registered_types["csv"]
        return klass.from_uri(uri=uri, source=self)

@DataObject.proxy_methods("head")
class PandasDataFrameBase(_PandasBase):
    real_type = pd.DataFrame

    def histogram(self, bins=None, columns=None, weights=None, **kwargs):
        """Histogram data on all numeric columns.

        :param bins: number of bins or edges (numpy-like)
        :param weights: as in numpy.histogram, but can be a column name as well

        kwargs
        ------
        - dropna: don't include nan values (these make zero values)
        - range: as in numpy.histogram

        In contrast to pandas hist method, this does not show the histogram.
        """
        if not columns:
            # All numeric
            columns = [col for col in self.columns if self.inner_data[col].dtype.kind in "iuf"]
        if isinstance(columns, str):
            columns = [columns]
        if isinstance(weights, str):
            columns = [col for col in columns if col != weights]
        weights = self[weights]
        return {col: self[col].histogram(bins=bins, weights=weights, **kwargs) for col in columns}

    def sql(self, sql, table_name=None):
        """Run SQL query on columns of the table.

        :param table_name: name of the temporary table (default is the name of the dataframe)

        Uses SQLite in-memory storage to create temporary table.
        """
        from sqlalchemy import create_engine
        from boadata import wrap

        if not table_name:
            table_name = self.name
        if not table_name:
            raise RuntimeError("Cannot run SQL queries on unnamed dataframe. Specify table_name argument...")
        engine = create_engine('sqlite:///:memory:')
        self.inner_data.to_sql(table_name, engine)
        # TODO: some clean up???
        return wrap(pd.read_sql_query(sql, engine), source=self)

    def __to_pandas_data_frame__(self):
        return PandasDataFrame(inner_data=self.inner_data, source=self)

    def __to_xy_dataseries__(self, x=None, y=None, **kwargs):
        constructor = DataObject.registered_types["xy_dataseries"]
        if not x and not y:
            # Auto from 2 dim
            if len(self.columns) == 1:
                return self[self.columns[0]].convert("xy_dataseries")
            elif len(self.columns) == 2:
                xdata = self[self.columns[0]]
                ydata = self[self.columns[1]]
                xname = xdata.name or "x"
                yname = ydata.name or "y"
            else:
                raise RuntimeError("Cannot convert dataframes with more than 2 columns")
        elif x:
            if not y:
                try:
                    ydata = self.evaluate(x)
                except:
                    ydata = self[x]
                xdata = range(ydata.shape[0])       # TODO: proper index???
                xname = "x"
                yname = x
            else:
                try:
                    xdata = self.evaluate(x)
                except:
                    xdata = self[x]
                try:
                    ydata = self.evaluate(y)
                except:
                    ydata = self[y]
                xname = x
                yname = y
        else:
            raise RuntimeError("Cannot specify col2 and not col1.")
        return constructor(xdata, ydata, xname=kwargs.get("xname", xname), yname=kwargs.get("yname", yname))

    def __to_excel_sheet__(self, uri):
        if "::" in uri:
            file, sheet = uri.split("::")
        else:
            file = uri
            sheet = self.name or "Unknown"
        self.inner_data.to_excel(file, sheet)
        uri = "{0}::{1}".format(file, sheet)
        klass = DataObject.registered_types.get("excel_sheet")
        return klass.from_uri(uri=uri, source=self)

    def drop_columns(self, columns, allow_nonexistent=False):
        if isinstance(columns, str):
            columns = [columns]
        if allow_nonexistent:
            columns = [column for column in columns if column in self.columns]
        self.inner_data.drop(columns, axis=1, inplace=True)

    def rename_columns(self, col_dict):
        """Change columns names.

        :param col_dict: New names
        :type col_dict: list | dict

        If col_dict is a dict, it is used as a mapping (non-matching ignored)
        If col_dict is a list, all columns are renamed to this (size checked)
        """
        if isinstance(col_dict, list):
            if len(col_dict) != len(self.columns):
                raise RuntimeError("Invalid number of columns to rename")
            new_names = col_dict
        elif isinstance(col_dict, dict):
            new_names = [col_dict.get(col, col) for col in self.columns]
        elif isinstance(col_dict, types.FunctionType):
            new_names = [col_dict(col) for col in self.columns]
        else:
            raise RuntimeError("Column names not understood.")
        self.inner_data.columns = new_names

    def add_column(self, expression, name=None):
        if name in self.columns:
            raise RuntimeError("Column already exists: {0}".format(name))
        self._create_column(expression, name)

    def _create_column(self, expression, name=None):
        if isinstance(expression, str):
            new_column = self.evaluate(expression, wrap=False)
            if not name:
                name = expression
        elif isinstance(expression, PandasSeriesBase):
            new_column = expression.inner_data
            if not name:
                name = new_column.name
        elif isinstance(expression, pd.Series):
            new_column = expression
            if not name:
                name = new_column.name
        self.inner_data[name] = new_column

    def replace_column(self, name, expression):
        if name not in self.columns:
            raise IndexError("The column {0} does not exist".format(name))
        self._create_column(expression, name)

    def dropna(self, **kwargs):
        kwargs["inplace"] = True
        self.inner_data.dropna(**kwargs)

    def append(self, other, **kwargs):
        other = wrap(other)
        if not isinstance(other, PandasDataFrameBase):
            try:
                other = other.convert("pandas_data_frame")
            except:
                raise TypeError("Only dataframes may be appended to dataframes.")
        if self.columns:
            if other.columns:
                if not other.columns == self.columns:
                    raise RuntimeError("Both dataframes must have same column names")
            if not np.all(other.inner_data.dtypes == self.inner_data.dtypes):
                raise RuntimeError("Both dataframes must have same column dtypes.")
        self.inner_data = self.inner_data.append(other.inner_data, **kwargs)

@DataObject.proxy_methods("dropna", "head")
@DataObject.proxy_methods("histogram", through="numpy_array")
@DataObject.proxy_methods("abs")
class PandasSeriesBase(_PandasBase, AsArrayMixin):
    real_type = pd.Series

    @property
    def ndim(self):
        return 1

    def __repr__(self):
        return "{0} (name={1}, shape={2}, dtype={3})".format(self.__class__.__name__,
                                                             self.inner_data.name, self.shape, self.dtype)

    def __to_xy_dataseries__(self, **kwargs):
        constructor = DataObject.registered_types["xy_dataseries"]
        x = range(self.shape[0])     # TODO: Change to proper index
        y = self
        if self.inner_data.name:
            name = self.inner_data.name
        else:
            name = "data"
        return constructor(x, y, xname="#", yname=name)

    def __to_pandas_data_frame__(self, name=None):
        if not name:
            name = self.name or "Data"
        df = pd.DataFrame()
        df[name] = self.inner_data
        return DataObject.from_native(df, source=self)

    def mode(self):
        """Mode interpreted as in scipy.mode"""
        result = self.inner_data.mode()
        return result.min()

@DataObject.register_type(default=True)
@MethodConversion.enable_to("numpy_array", method_name="as_matrix")
class PandasDataFrame(PandasDataFrameBase):
    type_name = "pandas_data_frame"

    def __init__(self, inner_data=None, *args, **kwargs):
        if inner_data is None:
            inner_data = pd.DataFrame()
        super(PandasDataFrame, self).__init__(inner_data, *args, **kwargs)

    def __repr__(self):
        return "{0} (name={1}, shape={2})".format(self.__class__.__name__, self.name, self.shape)

@DataObject.register_type(default=True)
@MethodConversion.enable_to("numpy_array", method_name="as_matrix")
class PandasSeries(PandasSeriesBase):
    type_name = "pandas_series"

    real_type = pd.Series

    @property
    def ndim(self):
        return 1
<commit_msg>...doc...<commit_after>from boadata.core import DataObject
from boadata.core.data_conversion import MethodConversion, DataConversion
from .mixins import GetItemMixin, StatisticsMixin, NumericalMixin, AsArrayMixin
import pandas as pd
import numpy as np
import types
from .. import wrap

class _PandasBase(DataObject, GetItemMixin, StatisticsMixin, NumericalMixin):
    """Shared behaviour for all pandas-based types.

    These include Series and DataFrame based types.
    """
    def __to_csv__(self, uri, **kwargs):
        self.inner_data.to_csv(uri, **kwargs)
        klass = DataObject.registered_types["csv"]
        return klass.from_uri(uri=uri, source=self)

@DataObject.proxy_methods("head")
class PandasDataFrameBase(_PandasBase):
    real_type = pd.DataFrame

    def histogram(self, bins=None, columns=None, weights=None, **kwargs):
        """Histogram data on all numeric columns.

        :param bins: number of bins or edges (numpy-like)
        :param weights: as in numpy.histogram, but can be a column name as well

        kwargs
        ------
        - dropna: don't include nan values (these make zero values)
        - range: as in numpy.histogram

        In contrast to pandas hist method, this does not show the histogram.
        """
        if not columns:
            # All numeric
            columns = [col for col in self.columns if self.inner_data[col].dtype.kind in "iuf"]
        if isinstance(columns, str):
            columns = [columns]
        if isinstance(weights, str):
            columns = [col for col in columns if col != weights]
        weights = self[weights]
        return {col: self[col].histogram(bins=bins, weights=weights, **kwargs) for col in columns}

    def sql(self, sql, table_name=None):
        """Run SQL query on columns of the table.

        :param table_name: name of the temporary table (default is the name of the dataframe)

        Uses SQLite in-memory storage to create temporary table.
        """
        from sqlalchemy import create_engine
        from boadata import wrap

        if not table_name:
            table_name = self.name
        if not table_name:
            raise RuntimeError("Cannot run SQL queries on unnamed dataframe. Specify table_name argument...")
        engine = create_engine('sqlite:///:memory:')
        self.inner_data.to_sql(table_name, engine)
        # TODO: some clean up???
        return wrap(pd.read_sql_query(sql, engine), source=self)

    def __to_pandas_data_frame__(self):
        return PandasDataFrame(inner_data=self.inner_data, source=self)

    def __to_xy_dataseries__(self, x=None, y=None, **kwargs):
        constructor = DataObject.registered_types["xy_dataseries"]
        if not x and not y:
            # Auto from 2 dim
            if len(self.columns) == 1:
                return self[self.columns[0]].convert("xy_dataseries")
            elif len(self.columns) == 2:
                xdata = self[self.columns[0]]
                ydata = self[self.columns[1]]
                xname = xdata.name or "x"
                yname = ydata.name or "y"
            else:
                raise RuntimeError("Cannot convert dataframes with more than 2 columns")
        elif x:
            if not y:
                try:
                    ydata = self.evaluate(x)
                except:
                    ydata = self[x]
                xdata = range(ydata.shape[0])       # TODO: proper index???
                xname = "x"
                yname = x
            else:
                try:
                    xdata = self.evaluate(x)
                except:
                    xdata = self[x]
                try:
                    ydata = self.evaluate(y)
                except:
                    ydata = self[y]
                xname = x
                yname = y
        else:
            raise RuntimeError("Cannot specify col2 and not col1.")
        return constructor(xdata, ydata, xname=kwargs.get("xname", xname), yname=kwargs.get("yname", yname))

    def __to_excel_sheet__(self, uri):
        if "::" in uri:
            file, sheet = uri.split("::")
        else:
            file = uri
            sheet = self.name or "Unknown"
        self.inner_data.to_excel(file, sheet)
        uri = "{0}::{1}".format(file, sheet)
        klass = DataObject.registered_types.get("excel_sheet")
        return klass.from_uri(uri=uri, source=self)

    def drop_columns(self, columns, allow_nonexistent=False):
        if isinstance(columns, str):
            columns = [columns]
        if allow_nonexistent:
            columns = [column for column in columns if column in self.columns]
        self.inner_data.drop(columns, axis=1, inplace=True)

    def rename_columns(self, col_dict):
        """Change columns names.

        :param col_dict: New names
        :type col_dict: list | dict

        If col_dict is a dict, it is used as a mapping (non-matching ignored)
        If col_dict is a list, all columns are renamed to this (size checked)
        """
        if isinstance(col_dict, list):
            if len(col_dict) != len(self.columns):
                raise RuntimeError("Invalid number of columns to rename")
            new_names = col_dict
        elif isinstance(col_dict, dict):
            new_names = [col_dict.get(col, col) for col in self.columns]
        elif isinstance(col_dict, types.FunctionType):
            new_names = [col_dict(col) for col in self.columns]
        else:
            raise RuntimeError("Column names not understood.")
        self.inner_data.columns = new_names

    def add_column(self, expression, name=None):
        if name in self.columns:
            raise RuntimeError("Column already exists: {0}".format(name))
        self._create_column(expression, name)

    def _create_column(self, expression, name=None):
        if isinstance(expression, str):
            new_column = self.evaluate(expression, wrap=False)
            if not name:
                name = expression
        elif isinstance(expression, PandasSeriesBase):
            new_column = expression.inner_data
            if not name:
                name = new_column.name
        elif isinstance(expression, pd.Series):
            new_column = expression
            if not name:
                name = new_column.name
        self.inner_data[name] = new_column

    def replace_column(self, name, expression):
        if name not in self.columns:
            raise IndexError("The column {0} does not exist".format(name))
        self._create_column(expression, name)

    def dropna(self, **kwargs):
        kwargs["inplace"] = True
        self.inner_data.dropna(**kwargs)

    def append(self, other, **kwargs):
        other = wrap(other)
        if not isinstance(other, PandasDataFrameBase):
            try:
                other = other.convert("pandas_data_frame")
            except:
                raise TypeError("Only dataframes may be appended to dataframes.")
        if self.columns:
            if other.columns:
                if not other.columns == self.columns:
                    raise RuntimeError("Both dataframes must have same column names")
            if not np.all(other.inner_data.dtypes == self.inner_data.dtypes):
                raise RuntimeError("Both dataframes must have same column dtypes.")
        self.inner_data = self.inner_data.append(other.inner_data, **kwargs)

@DataObject.proxy_methods("dropna", "head")
@DataObject.proxy_methods("histogram", through="numpy_array")
@DataObject.proxy_methods("abs")
class PandasSeriesBase(_PandasBase, AsArrayMixin):
    """Abstract class for all types based on pandas Series"""

    real_type = pd.Series

    @property
    def ndim(self):
        return 1

    def __repr__(self):
        return "{0} (name={1}, shape={2}, dtype={3})".format(self.__class__.__name__,
                                                             self.inner_data.name, self.shape, self.dtype)

    def __to_xy_dataseries__(self, **kwargs):
        constructor = DataObject.registered_types["xy_dataseries"]
        x = range(self.shape[0])     # TODO: Change to proper index
        y = self
        if self.inner_data.name:
            name = self.inner_data.name
        else:
            name = "data"
        return constructor(x, y, xname="#", yname=name)

    def __to_pandas_data_frame__(self, name=None):
        if not name:
            name = self.name or "Data"
        df = pd.DataFrame()
        df[name] = self.inner_data
        return DataObject.from_native(df, source=self)

    def mode(self):
        """Mode interpreted as in scipy.mode"""
        result = self.inner_data.mode()
        return result.min()

@DataObject.register_type(default=True)
@MethodConversion.enable_to("numpy_array", method_name="as_matrix")
class PandasDataFrame(PandasDataFrameBase):
    type_name = "pandas_data_frame"

    def __init__(self, inner_data=None, *args, **kwargs):
        if inner_data is None:
            inner_data = pd.DataFrame()
        super(PandasDataFrame, self).__init__(inner_data, *args, **kwargs)

    def __repr__(self):
        return "{0} (name={1}, shape={2})".format(self.__class__.__name__, self.name, self.shape)

@DataObject.register_type(default=True)
@MethodConversion.enable_to("numpy_array", method_name="as_matrix")
class PandasSeries(PandasSeriesBase):
    type_name = "pandas_series"

    real_type = pd.Series

    @property
    def ndim(self):
        return 1
<|endoftext|>


<commit_before>#
# CDR-Stats License
# http://www.cdr-stats.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2013 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#
from django.contrib.auth.decorators import login_required,\
    permission_required
from django.http import HttpResponseRedirect, Http404
from django.shortcuts import render_to_response, get_object_or_404
from django.template.context import RequestContext
from django.utils.translation import gettext as _
from django.conf import settings
from django.db.models import Count
from cdr_alert.models import Alarm, Blacklist, Whitelist, AlarmReport
from cdr_alert.constants import ALARM_COLUMN_NAME, ALARM_REPORT_COLUMN_NAME
from cdr_alert.forms import AlarmForm, BWCountryForm, BWPrefixForm,\
    AlarmReportForm
from cdr_alert.tasks import run_alarm
from cdr_alert.constants import ALERT_CONDITION, PERIOD
from common.common_functions import current_view, get_pagination_vars,\
    validate_days
from country_dialcode.models import Prefix
from datetime import datetime
from dateutil.relativedelta import relativedelta
import time
import logging

@permission_required('cdr_alert.alert_settings', login_url='/')
@login_required
def alarm_list(request):
    """Alarm list for the logged in user

    **Attributes**:

        * ``template`` - frontend/cdr_alert/alert_list.html

    **Logic Description**:

        * List all alarms which belong to the logged in user.
    """
    sort_col_field_list = ['id', 'name', 'period', 'type', 'alert_condition',
                           'alert_value', 'status', 'updated_date']
    default_sort_field = 'id'
    pagination_data =\
        get_pagination_vars(request, sort_col_field_list, default_sort_field)

    PAGE_SIZE = pagination_data['PAGE_SIZE']
    sort_order = pagination_data['sort_order']

    alarm_list = Alarm.objects\
        .filter(user=request.user).order_by(sort_order)

    template_name = 'frontend/cdr_alert/alarm/list.html'

    PAGE_SIZE = settings.PAGE_SIZE
    template_data = {
        'module': current_view(request),
        'msg': request.session.get('msg'),
        'error_msg': request.session.get('error_msg'),
        'rows': alarm_list,
        'total_count': alarm_list.count(),
        'PAGE_SIZE': PAGE_SIZE,
        'ALARM_COLUMN_NAME': ALARM_COLUMN_NAME,
        'col_name_with_order': pagination_data['col_name_with_order'],
    }
    request.session['msg'] = ''
    request.session['error_msg'] = ''
    return render_to_response(template_name, template_data,
        context_instance=RequestContext(request))

@permission_required('cdr_alert.add_alarm', login_url='/')
@login_required
def alarm_add(request):
    """Add new Alarm for the logged in user

    **Attributes**:

        * ``form`` - AlarmForm
        * ``template`` - frontend/cdr_alert/alarm/change.html

    **Logic Description**:

        * Add a new Alarm which will belong to the logged in user
          via the AlarmForm & get redirected to the Alarm list
    """
    form = AlarmForm()
    if request.method == 'POST':
        form = AlarmForm(request.POST)
        if form.is_valid():
            obj = form.save(commit=False)
            obj.user = request.user
            obj.save()
            request.session["msg"] = _('"%(name)s" added.') %\
                {'name': request.POST['name']}
            return HttpResponseRedirect('/alert/')
    template = 'frontend/cdr_alert/alarm/change.html'
    data = {
        'module': current_view(request),
        'form': form,
        'action': 'add',
    }
    return render_to_response(template, data,
        context_instance=RequestContext(request))

@permission_required('cdr_alert.alert_test', login_url='/')
@login_required
def alarm_test(request, object_id):
    """Test a alarm for a logged in user

    **Attributes**:

        * ``object_id`` - Selected alarm object

    **Logic Description**:

        * Test selected the alarm from the alarm list
    """
    alarm_data = {}
    alarm_obj = get_object_or_404(Alarm, pk=object_id, user=request.user)

    alarm_data = run_alarm(alarm_obj, logging)
    if alarm_obj.alert_condition != ALERT_CONDITION.IS_LESS_THAN or \
        alarm_obj.alert_condition != ALERT_CONDITION.IS_GREATER_THAN:
        alarm_data['diff'] = round(abs(alarm_data['current_value'] - alarm_data['previous_value']), 2)

    if alarm_obj.alert_condition == ALERT_CONDITION.PERCENTAGE_DECREASE_BY_MORE_THAN or \
        alarm_obj.alert_condition == ALERT_CONDITION.PERCENTAGE_INCREASE_BY_MORE_THAN:
        avg = (alarm_data['current_value'] + alarm_data['previous_value']) / 2
        alarm_data['percentage'] = round(alarm_data['diff'] / avg * 100, 2)

    template = 'frontend/cdr_alert/alarm/alarm_testing.html'
    data = {
        'alarm_obj': alarm_obj,
        'alarm_data': alarm_data,
        'ALERT_CONDITION': ALERT_CONDITION,
        'PERIOD': PERIOD,
    }
    return render_to_response(template, data,
        context_instance=RequestContext(request))

@permission_required('cdr_alert.delete_alarm', login_url='/')
@login_required
def alarm_del(request, object_id):
    """Delete a alarm for a logged in user

    **Attributes**:

        * ``object_id`` - Selected alarm object
        * ``object_list`` - Selected alarm objects

    **Logic Description**:

        * Delete selected the alarm from the alarm list
    """
    if int(object_id) != 0:
        # When object_id is not 0
        alarm = get_object_or_404(
            Alarm, pk=object_id, user=request.user)

        # 1) delete alarm
        request.session["msg"] = _('"%(name)s" is deleted.')\
            % {'name': alarm.name}
        alarm.delete()
    else:
        # When object_id is 0 (Multiple records delete)
        values = request.POST.getlist('select')
        values = ", ".join(["%s" % el for el in values])
        try:
            # 1) delete alarm
            alarm_list = Alarm.objects.filter(user=request.user)\
                .extra(where=['id IN (%s)' % values])
            if alarm_list:
                request.session["msg"] =\
                    _('%(count)s alarm(s) are deleted.')\
                    % {'count': alarm_list.count()}
                alarm_list.delete()
        except:
            raise Http404

    return HttpResponseRedirect('/alert/')

@permission_required('cdr_alert.change_alarm', login_url='/')
@login_required
def alarm_change(request, object_id):
    """Update/Delete Alarm for the logged in user

    **Attributes**:

        * ``object_id`` - Selected alarm object
        * ``form`` - AlarmForm
        * ``template`` - frontend/cdr_alert/alarm/change.html

    **Logic Description**:

        * Update/delete selected alarm from the alarm list
          via alarmForm & get redirected to alarm list
    """
    alarm = get_object_or_404(Alarm, pk=object_id, user=request.user)
    form = AlarmForm(instance=alarm)
    if request.method == 'POST':
        if request.POST.get('delete'):
            alarm_del(request, object_id)
            return HttpResponseRedirect('/alert/')
        else:
            form = AlarmForm(request.POST, instance=alarm)
            if form.is_valid():
                form.save()
                request.session["msg"] = _('"%(name)s" is updated.')\
                    % {'name': request.POST['name']}
                return HttpResponseRedirect('/alert/')

    template = 'frontend/cdr_alert/alarm/change.html'
    data = {
        'module': current_view(request),
        'form': form,
        'action': 'update',
    }
    return render_to_response(template, data,
        context_instance=RequestContext(request))

def last_seven_days_report(request, kwargs):
    comp_days = 7
    from_date = datetime.today()
    from_day = validate_days(from_date.year,
        from_date.month,
        from_date.day)
    from_year = from_date.year
    from_month = from_date.month
    end_date = datetime(from_year, from_month, from_day)
    start_date = end_date + relativedelta(days=-comp_days)
    start_date = datetime(start_date.year, start_date.month,
                          start_date.day, 0, 0, 0, 0)
    end_date = datetime(end_date.year, end_date.month,
                        end_date.day, 23, 59, 59, 999999)

    if start_date and end_date:
        kwargs['daterun__range'] = (start_date, end_date)

    select_data = {"daterun": "SUBSTR(CAST(daterun as CHAR(30)),1,10)"}

    alarm_data = AlarmReport.objects.extra(select=select_data)\
        .values('daterun')\
        .filter(**kwargs)\
        .annotate(Count('daterun'))\
        .order_by('-daterun')

    total_data = {}
    total_alert = 0
    charttype = "lineWithFocusChart"
    chartdata = {"x": []}
    for doc in alarm_data:
        daterun = str(doc['daterun'])

        graph_day = datetime(int(daterun[0:4]),
                             int(daterun[5:7]),
                             int(daterun[8:10]),
                             0, 0, 0, 0)
        dt = int(1000 * time.mktime(graph_day.timetuple()))

        if dt in total_data:
            total_data[dt]['alert_count'] += int(doc['daterun__count'])
        else:
            total_data[dt] = {
                'alert_count': int(doc['daterun__count'])
            }

        total_alert += int(doc['daterun__count'])

    # sorting on date col
    total_data = total_data.items()
    total_data = sorted(total_data, key=lambda k: k[0])

    xdata = []
    ydata = []
    for i in total_data:
        xdata.append(i[0])
        ydata.append(i[1]['alert_count'])

    tooltip_date = "%d %b %Y %H:%M %p"
    extra_serie = {"tooltip": {"y_start": "", "y_end": ""},
                   "date_format": tooltip_date}
    chartdata = {
        'x': xdata,
        'name1': 'Alert', 'y1': ydata, 'extra1': extra_serie,
    }

    data = {
        'start_date': start_date,
        'end_date': end_date,
        'total_data': total_data,
        'total_alert': total_alert,
        'charttype': charttype,
        'chartdata': chartdata,
    }
    return data

@permission_required('cdr_alert.alarm_report', login_url='/')
@login_required
def alert_report(request):
    """
    To get alarm report for logged in user

    **Attributes**:

        * ``form`` - AlarmReportForm
        * ``template`` - frontend/cdr_alert/alarm_report.html
    """
    form = AlarmReportForm(request.user, initial={'alarm': 0})
    sort_col_field_list = ['id', 'alarm', 'calculatedvalue', 'status', 'daterun']
    default_sort_field = 'id'
    pagination_data =\
        get_pagination_vars(request, sort_col_field_list, default_sort_field)

    PAGE_SIZE = pagination_data['PAGE_SIZE']
    sort_order = pagination_data['sort_order']
    alert_id = ''
    action = 'tabs-1'

    if request.method == 'POST':
        form = AlarmReportForm(request.user, request.POST)
        if form.is_valid():
            request.session['session_alarm'] = ''

            if request.POST.get('alarm'):
                alert_id = request.POST.get('alarm')
                request.session['session_alarm'] = alert_id

    post_var_with_page = 0
    try:
        if request.GET.get('page') or request.GET.get('sort_by'):
            post_var_with_page = 1
            alert_id = request.session.get('session_alarm')
            form = AlarmReportForm(request.user, initial={'alarm': alert_id})
        else:
            post_var_with_page = 1
            if request.method == 'GET':
                post_var_with_page = 0
    except:
        pass

    if post_var_with_page == 0:
        # default
        # unset session var
        request.session['session_alarm'] = ''

    kwargs = {}
    if alert_id and alert_id != 0:
        kwargs['alarm_id'] = alert_id
    kwargs['alarm__user'] = request.user

    alarm_report_list = AlarmReport.objects.filter(**kwargs).order_by(sort_order)
    days_report = last_seven_days_report(request, kwargs)

    total_data = days_report['total_data']
    start_date = days_report['start_date']
    end_date = days_report['end_date']

    template = 'frontend/cdr_alert/alarm_report.html'
    data = {
        'module': current_view(request),
        'form': form,
        'action': action,
        'total_data': total_data,
        'start_date': start_date,
        'end_date': end_date,
        'rows': alarm_report_list,
        'total_count': alarm_report_list.count(),
        'PAGE_SIZE': PAGE_SIZE,
        'ALARM_REPORT_COLUMN_NAME': ALARM_REPORT_COLUMN_NAME,
        'col_name_with_order': pagination_data['col_name_with_order'],
        'charttype': days_report['charttype'],
        'chartdata': days_report['chartdata'],
    }
    return render_to_response(template, data,
        context_instance=RequestContext(request))

@permission_required('cdr_alert.view_whitelist', login_url='/')
@permission_required('cdr_alert.view_blacklist', login_url='/')
@login_required
def trust_control(request):
    #Blacklist, Whitelist
    prefix_list = \
        map(str, Prefix.objects.values_list("prefix", flat=True).all().order_by('prefix'))
    prefix_list = (','.join('"' + item + '"' for item in prefix_list))
    prefix_list = "[" + str(prefix_list) + "]"

    blacklist = Blacklist.objects.filter(user=request.user).order_by('id')
    whitelist = Whitelist.objects.filter(user=request.user).order_by('id')

    # blacklist form
    bl_country_form = BWCountryForm()
    bl_prefix_form = BWPrefixForm()

    # whitelist form
    wl_country_form = BWCountryForm()
    wl_prefix_form = BWPrefixForm()

    template = 'frontend/cdr_alert/common_black_white_list.html'
    data = {
        'module': current_view(request),
        'prefix_list': prefix_list,
        'bl_country_form': bl_country_form,
        'bl_prefix_form': bl_prefix_form,
        'wl_country_form': wl_country_form,
        'wl_prefix_form': wl_prefix_form,
        'blacklist': blacklist,
        'whitelist': whitelist,
    }
    return render_to_response(template, data,
        context_instance=RequestContext(request))
<commit_msg>clean<commit_after>#
# CDR-Stats License
# http://www.cdr-stats.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2013 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#
from django.contrib.auth.decorators import login_required,\
    permission_required
from django.http import HttpResponseRedirect, Http404
from django.shortcuts import render_to_response, get_object_or_404
from django.template.context import RequestContext
from django.utils.translation import gettext as _
from django.conf import settings
from django.db.models import Count
from cdr_alert.models import Alarm, Blacklist, Whitelist, AlarmReport
from cdr_alert.constants import ALARM_COLUMN_NAME, ALARM_REPORT_COLUMN_NAME
from cdr_alert.forms import AlarmForm, BWCountryForm, BWPrefixForm,\
    AlarmReportForm
from cdr_alert.tasks import run_alarm
from cdr_alert.constants import ALERT_CONDITION, PERIOD
from common.common_functions import current_view, get_pagination_vars,\
    validate_days
from country_dialcode.models import Prefix
from datetime import datetime
from dateutil.relativedelta import relativedelta
import time
import logging

@permission_required('cdr_alert.alert_settings', login_url='/')
@login_required
def alarm_list(request):
    """Alarm list for the logged in user

    **Attributes**:

        * ``template`` - frontend/cdr_alert/alert_list.html

    **Logic Description**:

        * List all alarms which belong to the logged in user.
    """
    sort_col_field_list = ['id', 'name', 'period', 'type', 'alert_condition',
                           'alert_value', 'status', 'updated_date']
    default_sort_field = 'id'
    pagination_data =\
        get_pagination_vars(request, sort_col_field_list, default_sort_field)

    PAGE_SIZE = pagination_data['PAGE_SIZE']
    sort_order = pagination_data['sort_order']

    alarm_list = Alarm.objects\
        .filter(user=request.user).order_by(sort_order)

    template_name = 'frontend/cdr_alert/alarm/list.html'

    PAGE_SIZE = settings.PAGE_SIZE
    template_data = {
        'module': current_view(request),
        'msg': request.session.get('msg'),
        'error_msg': request.session.get('error_msg'),
        'rows': alarm_list,
        'total_count': alarm_list.count(),
        'PAGE_SIZE': PAGE_SIZE,
        'ALARM_COLUMN_NAME': ALARM_COLUMN_NAME,
        'col_name_with_order': pagination_data['col_name_with_order'],
    }
    request.session['msg'] = ''
    request.session['error_msg'] = ''
    return render_to_response(template_name, template_data,
        context_instance=RequestContext(request))

@permission_required('cdr_alert.add_alarm', login_url='/')
@login_required
def alarm_add(request):
    """Add new Alarm for the logged in user

    **Attributes**:

        * ``form`` - AlarmForm
        * ``template`` - frontend/cdr_alert/alarm/change.html

    **Logic Description**:

        * Add a new Alarm which will belong to the logged in user
          via the AlarmForm & get redirected to the Alarm list
    """
    form = AlarmForm()
    if request.method == 'POST':
        form = AlarmForm(request.POST)
        if form.is_valid():
            obj = form.save(commit=False)
            obj.user = request.user
            obj.save()
            request.session["msg"] = _('"%(name)s" added.') %\
                {'name': request.POST['name']}
            return HttpResponseRedirect('/alert/')
    template = 'frontend/cdr_alert/alarm/change.html'
    data = {
        'module': current_view(request),
        'form': form,
        'action': 'add',
    }
    return render_to_response(template, data,
        context_instance=RequestContext(request))

@permission_required('cdr_alert.alert_test', login_url='/')
@login_required
def alarm_test(request, object_id):
    """Test a alarm for a logged in user

    **Attributes**:

        * ``object_id`` - Selected alarm object

    **Logic Description**:

        * Test selected the alarm from the alarm list
    """
    alarm_data = {}
    alarm_obj = get_object_or_404(Alarm, pk=object_id, user=request.user)

    alarm_data = run_alarm(alarm_obj, logging)
    if (alarm_obj.alert_condition != ALERT_CONDITION.IS_LESS_THAN or
       alarm_obj.alert_condition != ALERT_CONDITION.IS_GREATER_THAN):
        alarm_data['diff'] = round(abs(alarm_data['current_value'] - alarm_data['previous_value']), 2)

    if (alarm_obj.alert_condition == ALERT_CONDITION.PERCENTAGE_DECREASE_BY_MORE_THAN or
       alarm_obj.alert_condition == ALERT_CONDITION.PERCENTAGE_INCREASE_BY_MORE_THAN):
        avg = (alarm_data['current_value'] + alarm_data['previous_value']) / 2
        alarm_data['percentage'] = round(alarm_data['diff'] / avg * 100, 2)

    template = 'frontend/cdr_alert/alarm/alarm_testing.html'
    data = {
        'alarm_obj': alarm_obj,
        'alarm_data': alarm_data,
        'ALERT_CONDITION': ALERT_CONDITION,
        'PERIOD': PERIOD,
    }
    return render_to_response(template, data,
        context_instance=RequestContext(request))

@permission_required('cdr_alert.delete_alarm', login_url='/')
@login_required
def alarm_del(request, object_id):
    """Delete a alarm for a logged in user

    **Attributes**:

        * ``object_id`` - Selected alarm object
        * ``object_list`` - Selected alarm objects

    **Logic Description**:

        * Delete selected the alarm from the alarm list
    """
    if int(object_id) != 0:
        # When object_id is not 0
        alarm = get_object_or_404(
            Alarm, pk=object_id, user=request.user)

        # 1) delete alarm
        request.session["msg"] = _('"%(name)s" is deleted.')\
            % {'name': alarm.name}
        alarm.delete()
    else:
        # When object_id is 0 (Multiple records delete)
        values = request.POST.getlist('select')
        values = ", ".join(["%s" % el for el in values])
        try:
            # 1) delete alarm
            alarm_list = Alarm.objects.filter(user=request.user)\
                .extra(where=['id IN (%s)' % values])
            if alarm_list:
                request.session["msg"] =\
                    _('%(count)s alarm(s) are deleted.')\
                    % {'count': alarm_list.count()}
                alarm_list.delete()
        except:
            raise Http404

    return HttpResponseRedirect('/alert/')

@permission_required('cdr_alert.change_alarm', login_url='/')
@login_required
def alarm_change(request, object_id):
    """Update/Delete Alarm for the logged in user

    **Attributes**:

        * ``object_id`` - Selected alarm object
        * ``form`` - AlarmForm
        * ``template`` - frontend/cdr_alert/alarm/change.html

    **Logic Description**:

        * Update/delete selected alarm from the alarm list
          via alarmForm & get redirected to alarm list
    """
    alarm = get_object_or_404(Alarm, pk=object_id, user=request.user)
    form = AlarmForm(instance=alarm)
    if request.method == 'POST':
        if request.POST.get('delete'):
            alarm_del(request, object_id)
            return HttpResponseRedirect('/alert/')
        else:
            form = AlarmForm(request.POST, instance=alarm)
            if form.is_valid():
                form.save()
                request.session["msg"] = _('"%(name)s" is updated.')\
                    % {'name': request.POST['name']}
                return HttpResponseRedirect('/alert/')

    template = 'frontend/cdr_alert/alarm/change.html'
    data = {
        'module': current_view(request),
        'form': form,
        'action': 'update',
    }
    return render_to_response(template, data,
        context_instance=RequestContext(request))

def last_seven_days_report(request, kwargs):
    comp_days = 7
    from_date = datetime.today()
    from_day = validate_days(from_date.year,
        from_date.month,
        from_date.day)
    from_year = from_date.year
    from_month = from_date.month
    end_date = datetime(from_year, from_month, from_day)
    start_date = end_date + relativedelta(days=-comp_days)
    start_date = datetime(start_date.year, start_date.month,
                          start_date.day, 0, 0, 0, 0)
    end_date = datetime(end_date.year, end_date.month,
                        end_date.day, 23, 59, 59, 999999)

    if start_date and end_date:
        kwargs['daterun__range'] = (start_date, end_date)

    select_data = {"daterun": "SUBSTR(CAST(daterun as CHAR(30)),1,10)"}

    alarm_data = AlarmReport.objects.extra(select=select_data)\
        .values('daterun')\
        .filter(**kwargs)\
        .annotate(Count('daterun'))\
        .order_by('-daterun')

    total_data = {}
    total_alert = 0
    charttype = "lineWithFocusChart"
    chartdata = {"x": []}
    for doc in alarm_data:
        daterun = str(doc['daterun'])

        graph_day = datetime(int(daterun[0:4]),
                             int(daterun[5:7]),
                             int(daterun[8:10]),
                             0, 0, 0, 0)
        dt = int(1000 * time.mktime(graph_day.timetuple()))

        if dt in total_data:
            total_data[dt]['alert_count'] += int(doc['daterun__count'])
        else:
            total_data[dt] = {
                'alert_count': int(doc['daterun__count'])
            }

        total_alert += int(doc['daterun__count'])

    # sorting on date col
    total_data = total_data.items()
    total_data = sorted(total_data, key=lambda k: k[0])

    xdata = []
    ydata = []
    for i in total_data:
        xdata.append(i[0])
        ydata.append(i[1]['alert_count'])

    tooltip_date = "%d %b %Y %H:%M %p"
    extra_serie = {"tooltip": {"y_start": "", "y_end": ""},
                   "date_format": tooltip_date}
    chartdata = {
        'x': xdata,
        'name1': 'Alert', 'y1': ydata, 'extra1': extra_serie,
    }

    data = {
        'start_date': start_date,
        'end_date': end_date,
        'total_data': total_data,
        'total_alert': total_alert,
        'charttype': charttype,
        'chartdata': chartdata,
    }
    return data

@permission_required('cdr_alert.alarm_report', login_url='/')
@login_required
def alert_report(request):
    """
    To get alarm report for logged in user

    **Attributes**:

        * ``form`` - AlarmReportForm
        * ``template`` - frontend/cdr_alert/alarm_report.html
    """
    form = AlarmReportForm(request.user, initial={'alarm': 0})
    sort_col_field_list = ['id', 'alarm', 'calculatedvalue', 'status', 'daterun']
    default_sort_field = 'id'
    pagination_data =\
        get_pagination_vars(request, sort_col_field_list, default_sort_field)

    PAGE_SIZE = pagination_data['PAGE_SIZE']
    sort_order = pagination_data['sort_order']
    alert_id = ''
    action = 'tabs-1'

    if request.method == 'POST':
        form = AlarmReportForm(request.user, request.POST)
        if form.is_valid():
            request.session['session_alarm'] = ''

            if request.POST.get('alarm'):
                alert_id = request.POST.get('alarm')
                request.session['session_alarm'] = alert_id

    post_var_with_page = 0
    try:
        if request.GET.get('page') or request.GET.get('sort_by'):
            post_var_with_page = 1
            alert_id = request.session.get('session_alarm')
            form = AlarmReportForm(request.user, initial={'alarm': alert_id})
        else:
            post_var_with_page = 1
            if request.method == 'GET':
                post_var_with_page = 0
    except:
        pass

    if post_var_with_page == 0:
        # default
        # unset session var
        request.session['session_alarm'] = ''

    kwargs = {}
    if alert_id and alert_id != 0:
        kwargs['alarm_id'] = alert_id
    kwargs['alarm__user'] = request.user

    alarm_report_list = AlarmReport.objects.filter(**kwargs).order_by(sort_order)
    days_report = last_seven_days_report(request, kwargs)

    total_data = days_report['total_data']
    start_date = days_report['start_date']
    end_date = days_report['end_date']

    template = 'frontend/cdr_alert/alarm_report.html'
    data = {
        'module': current_view(request),
        'form': form,
        'action': action,
        'total_data': total_data,
        'start_date': start_date,
        'end_date': end_date,
        'rows': alarm_report_list,
        'total_count': alarm_report_list.count(),
        'PAGE_SIZE': PAGE_SIZE,
        'ALARM_REPORT_COLUMN_NAME': ALARM_REPORT_COLUMN_NAME,
        'col_name_with_order': pagination_data['col_name_with_order'],
        'charttype': days_report['charttype'],
        'chartdata': days_report['chartdata'],
    }
    return render_to_response(template, data,
        context_instance=RequestContext(request))

@permission_required('cdr_alert.view_whitelist', login_url='/')
@permission_required('cdr_alert.view_blacklist', login_url='/')
@login_required
def trust_control(request):
    #Blacklist, Whitelist
    prefix_list = \
        map(str, Prefix.objects.values_list("prefix", flat=True).all().order_by('prefix'))
    prefix_list = (','.join('"' + item + '"' for item in prefix_list))
    prefix_list = "[" + str(prefix_list) + "]"

    blacklist = Blacklist.objects.filter(user=request.user).order_by('id')
    whitelist = Whitelist.objects.filter(user=request.user).order_by('id')

    # blacklist form
    bl_country_form = BWCountryForm()
    bl_prefix_form = BWPrefixForm()

    # whitelist form
    wl_country_form = BWCountryForm()
    wl_prefix_form = BWPrefixForm()

    template = 'frontend/cdr_alert/common_black_white_list.html'
    data = {
        'module': current_view(request),
        'prefix_list': prefix_list,
        'bl_country_form': bl_country_form,
        'bl_prefix_form': bl_prefix_form,
        'wl_country_form': wl_country_form,
        'wl_prefix_form': wl_prefix_form,
        'blacklist': blacklist,
        'whitelist': whitelist,
    }
    return render_to_response(template, data,
        context_instance=RequestContext(request))
<|endoftext|>


<commit_before>from cffi import FFI
from six.moves import range as srange

ffi = FFI()

CONSTANTS = [
    "X_PROTOCOL",
    "X_PROTOCOL_REVISION",
    "X_TCP_PORT",

    "XCB_NONE",
    "XCB_COPY_FROM_PARENT",
    "XCB_CURRENT_TIME",
    "XCB_NO_SYMBOL",

    "XCB_CONN_ERROR",
    "XCB_CONN_CLOSED_EXT_NOTSUPPORTED",
    "XCB_CONN_CLOSED_MEM_INSUFFICIENT",
    "XCB_CONN_CLOSED_REQ_LEN_EXCEED",
    "XCB_CONN_CLOSED_PARSE_ERR",
#    "XCB_CONN_CLOSED_INVALID_SCREEN",
#    "XCB_CONN_CLOSED_FDPASSING_FAILED",

    "XCB_REQUEST_CHECKED",
]

# constants
ffi.cdef('\n'.join("#define %s ..." % c for c in CONSTANTS))

# types
ffi.cdef("""
    // xcb.h
    typedef struct {
        uint8_t   response_type;  /**< Type of the response */
        uint8_t  pad0;           /**< Padding */
        uint16_t sequence;       /**< Sequence number */
        uint32_t length;         /**< Length of the response */
    } xcb_generic_reply_t;

    typedef struct {
        uint8_t   response_type;  /**< Type of the response */
        uint8_t  pad0;           /**< Padding */
        uint16_t sequence;       /**< Sequence number */
        uint32_t pad[7];         /**< Padding */
        uint32_t full_sequence;  /**< full sequence */
    } xcb_generic_event_t;

    typedef struct {
        uint8_t   response_type;  /**< Type of the response */
        uint8_t   error_code;     /**< Error code */
        uint16_t sequence;       /**< Sequence number */
        uint32_t resource_id;     /** < Resource ID for requests with side effects only */
        uint16_t minor_code;      /** < Minor opcode of the failed request */
        uint8_t major_code;       /** < Major opcode of the failed request */
        uint8_t pad0;
        uint32_t pad[5];         /**< Padding */
        uint32_t full_sequence;  /**< full sequence */
    } xcb_generic_error_t;

    typedef struct {
        unsigned int sequence;  /**< Sequence number */
    } xcb_void_cookie_t;

    typedef struct xcb_auth_info_t {
        int   namelen;
        char *name;
        int   datalen;
        char *data;
    } xcb_auth_info_t;

    typedef ... xcb_connection_t;

    // xproto.h
    typedef struct xcb_query_extension_reply_t {
        uint8_t  response_type;
        uint8_t  pad0;
        uint16_t sequence;
        uint32_t length;
        uint8_t  present;
        uint8_t  major_opcode;
        uint8_t  first_event;
        uint8_t  first_error;
    } xcb_query_extension_reply_t;

    typedef struct xcb_setup_t {
        uint8_t       status; /**<  */
        uint8_t       pad0; /**<  */
        uint16_t      protocol_major_version; /**<  */
        uint16_t      protocol_minor_version; /**<  */
        uint16_t      length; /**<  */
        uint32_t      release_number; /**<  */
        uint32_t      resource_id_base; /**<  */
        uint32_t      resource_id_mask; /**<  */
        uint32_t      motion_buffer_size; /**<  */
        uint16_t      vendor_len; /**<  */
        uint16_t      maximum_request_length; /**<  */
        uint8_t       roots_len; /**<  */
        uint8_t       pixmap_formats_len; /**<  */
        uint8_t       image_byte_order; /**<  */
        uint8_t       bitmap_format_bit_order; /**<  */
        uint8_t       bitmap_format_scanline_unit; /**<  */
        uint8_t       bitmap_format_scanline_pad; /**<  */
        uint8_t       min_keycode; /**<  */
        uint8_t       max_keycode; /**<  */
        uint8_t       pad1[4]; /**<  */
    } xcb_setup_t;

    typedef uint32_t xcb_drawable_t;
    typedef uint32_t xcb_visualid_t;

    typedef struct xcb_visualtype_t {
        xcb_visualid_t visual_id; /**<  */
        uint8_t        _class; /**<  */
        uint8_t        bits_per_rgb_value; /**<  */
        uint16_t       colormap_entries; /**<  */
        uint32_t       red_mask; /**<  */
        uint32_t       green_mask; /**<  */
        uint32_t       blue_mask; /**<  */
        uint8_t        pad0[4]; /**<  */
    } xcb_visualtype_t;

    // xcbext.h
    typedef struct xcb_extension_t {
        const char *name;
        int global_id;
    } xcb_extension_t;

    typedef struct {
        size_t count;
        xcb_extension_t *ext;
        uint8_t opcode;
        uint8_t isvoid;
    } xcb_protocol_request_t;

    // sys/uio.h
    struct iovec
    {
      void *iov_base; /* BSD uses caddr_t (1003.1g requires void *) */
      size_t iov_len; /* Must be size_t (1003.1g) */
    };

    // need to manually free some things that XCB allocates
    void free(void *ptr);
""")

# connection manipulation, mostly generated with:
# grep -v '^[ \/\}#]' xcb.h | grep -v '^typedef' | grep -v '^extern'
ffi.cdef("""
    int xcb_flush(xcb_connection_t *c);
    uint32_t xcb_get_maximum_request_length(xcb_connection_t *c);
    void xcb_prefetch_maximum_request_length(xcb_connection_t *c);
    xcb_generic_event_t *xcb_wait_for_event(xcb_connection_t *c);
    xcb_generic_event_t *xcb_poll_for_event(xcb_connection_t *c);
    const xcb_query_extension_reply_t *xcb_get_extension_data(xcb_connection_t *c, xcb_extension_t *ext);
    const xcb_setup_t *xcb_get_setup(xcb_connection_t *c);
    int xcb_get_file_descriptor(xcb_connection_t *c);
    int xcb_connection_has_error(xcb_connection_t *c);
    xcb_connection_t *xcb_connect_to_fd(int fd, xcb_auth_info_t *auth_info);
    void xcb_disconnect(xcb_connection_t *c);
    int xcb_parse_display(const char *name, char **host, int *display, int *screen);
    xcb_connection_t *xcb_connect(const char *displayname, int *screenp);
    xcb_connection_t *xcb_connect_to_display_with_auth_info(const char *display, xcb_auth_info_t *auth, int *screen);
    uint32_t xcb_generate_id(xcb_connection_t *c);
    xcb_generic_error_t *xcb_request_check(xcb_connection_t *c, xcb_void_cookie_t cookie);
""")

ffi.cdef("""
    unsigned int xcb_send_request(xcb_connection_t *c, int flags, struct iovec *vector, const xcb_protocol_request_t *request);
    void *xcb_wait_for_reply(xcb_connection_t *c, unsigned int request, xcb_generic_error_t **e);
    int xcb_poll_for_reply(xcb_connection_t *c, unsigned int request, void **reply, xcb_generic_error_t **error);
""")

C = ffi.verify("""
    #include <xcb/xcb.h>
    #include <xcb/xcbext.h>
""", libraries=['xcb'])

def bytes_to_cdata(bs):
    buf = ffi.new('char[]', bs)
    return buf

# cfficairo needs an xcb_visualtype_t
def visualtype_to_c_struct(vt):
    s = ffi.new("xcb_visualtype_t *")

    s.visual_id = vt.visual_id
    s._class = vt._class
    s.bits_per_rgb_value = vt.bits_per_rgb_value
    s.colormap_entries = vt.colormap_entries
    s.red_mask = vt.red_mask
    s.green_mask = vt.green_mask
    s.blue_mask = vt.blue_mask

    return s
<commit_msg>Include free()<commit_after>from cffi import FFI
from six.moves import range as srange

ffi = FFI()

CONSTANTS = [
    "X_PROTOCOL",
    "X_PROTOCOL_REVISION",
    "X_TCP_PORT",

    "XCB_NONE",
    "XCB_COPY_FROM_PARENT",
    "XCB_CURRENT_TIME",
    "XCB_NO_SYMBOL",

    "XCB_CONN_ERROR",
    "XCB_CONN_CLOSED_EXT_NOTSUPPORTED",
    "XCB_CONN_CLOSED_MEM_INSUFFICIENT",
    "XCB_CONN_CLOSED_REQ_LEN_EXCEED",
    "XCB_CONN_CLOSED_PARSE_ERR",
#    "XCB_CONN_CLOSED_INVALID_SCREEN",
#    "XCB_CONN_CLOSED_FDPASSING_FAILED",

    "XCB_REQUEST_CHECKED",
]

# constants
ffi.cdef('\n'.join("#define %s ..." % c for c in CONSTANTS))

# types
ffi.cdef("""
    // xcb.h
    typedef struct {
        uint8_t   response_type;  /**< Type of the response */
        uint8_t  pad0;           /**< Padding */
        uint16_t sequence;       /**< Sequence number */
        uint32_t length;         /**< Length of the response */
    } xcb_generic_reply_t;

    typedef struct {
        uint8_t   response_type;  /**< Type of the response */
        uint8_t  pad0;           /**< Padding */
        uint16_t sequence;       /**< Sequence number */
        uint32_t pad[7];         /**< Padding */
        uint32_t full_sequence;  /**< full sequence */
    } xcb_generic_event_t;

    typedef struct {
        uint8_t   response_type;  /**< Type of the response */
        uint8_t   error_code;     /**< Error code */
        uint16_t sequence;       /**< Sequence number */
        uint32_t resource_id;     /** < Resource ID for requests with side effects only */
        uint16_t minor_code;      /** < Minor opcode of the failed request */
        uint8_t major_code;       /** < Major opcode of the failed request */
        uint8_t pad0;
        uint32_t pad[5];         /**< Padding */
        uint32_t full_sequence;  /**< full sequence */
    } xcb_generic_error_t;

    typedef struct {
        unsigned int sequence;  /**< Sequence number */
    } xcb_void_cookie_t;

    typedef struct xcb_auth_info_t {
        int   namelen;
        char *name;
        int   datalen;
        char *data;
    } xcb_auth_info_t;

    typedef ... xcb_connection_t;

    // xproto.h
    typedef struct xcb_query_extension_reply_t {
        uint8_t  response_type;
        uint8_t  pad0;
        uint16_t sequence;
        uint32_t length;
        uint8_t  present;
        uint8_t  major_opcode;
        uint8_t  first_event;
        uint8_t  first_error;
    } xcb_query_extension_reply_t;

    typedef struct xcb_setup_t {
        uint8_t       status; /**<  */
        uint8_t       pad0; /**<  */
        uint16_t      protocol_major_version; /**<  */
        uint16_t      protocol_minor_version; /**<  */
        uint16_t      length; /**<  */
        uint32_t      release_number; /**<  */
        uint32_t      resource_id_base; /**<  */
        uint32_t      resource_id_mask; /**<  */
        uint32_t      motion_buffer_size; /**<  */
        uint16_t      vendor_len; /**<  */
        uint16_t      maximum_request_length; /**<  */
        uint8_t       roots_len; /**<  */
        uint8_t       pixmap_formats_len; /**<  */
        uint8_t       image_byte_order; /**<  */
        uint8_t       bitmap_format_bit_order; /**<  */
        uint8_t       bitmap_format_scanline_unit; /**<  */
        uint8_t       bitmap_format_scanline_pad; /**<  */
        uint8_t       min_keycode; /**<  */
        uint8_t       max_keycode; /**<  */
        uint8_t       pad1[4]; /**<  */
    } xcb_setup_t;

    typedef uint32_t xcb_drawable_t;
    typedef uint32_t xcb_visualid_t;

    typedef struct xcb_visualtype_t {
        xcb_visualid_t visual_id; /**<  */
        uint8_t        _class; /**<  */
        uint8_t        bits_per_rgb_value; /**<  */
        uint16_t       colormap_entries; /**<  */
        uint32_t       red_mask; /**<  */
        uint32_t       green_mask; /**<  */
        uint32_t       blue_mask; /**<  */
        uint8_t        pad0[4]; /**<  */
    } xcb_visualtype_t;

    // xcbext.h
    typedef struct xcb_extension_t {
        const char *name;
        int global_id;
    } xcb_extension_t;

    typedef struct {
        size_t count;
        xcb_extension_t *ext;
        uint8_t opcode;
        uint8_t isvoid;
    } xcb_protocol_request_t;

    // sys/uio.h
    struct iovec
    {
      void *iov_base; /* BSD uses caddr_t (1003.1g requires void *) */
      size_t iov_len; /* Must be size_t (1003.1g) */
    };

    // need to manually free some things that XCB allocates
    void free(void *ptr);
""")

# connection manipulation, mostly generated with:
# grep -v '^[ \/\}#]' xcb.h | grep -v '^typedef' | grep -v '^extern'
ffi.cdef("""
    int xcb_flush(xcb_connection_t *c);
    uint32_t xcb_get_maximum_request_length(xcb_connection_t *c);
    void xcb_prefetch_maximum_request_length(xcb_connection_t *c);
    xcb_generic_event_t *xcb_wait_for_event(xcb_connection_t *c);
    xcb_generic_event_t *xcb_poll_for_event(xcb_connection_t *c);
    const xcb_query_extension_reply_t *xcb_get_extension_data(xcb_connection_t *c, xcb_extension_t *ext);
    const xcb_setup_t *xcb_get_setup(xcb_connection_t *c);
    int xcb_get_file_descriptor(xcb_connection_t *c);
    int xcb_connection_has_error(xcb_connection_t *c);
    xcb_connection_t *xcb_connect_to_fd(int fd, xcb_auth_info_t *auth_info);
    void xcb_disconnect(xcb_connection_t *c);
    int xcb_parse_display(const char *name, char **host, int *display, int *screen);
    xcb_connection_t *xcb_connect(const char *displayname, int *screenp);
    xcb_connection_t *xcb_connect_to_display_with_auth_info(const char *display, xcb_auth_info_t *auth, int *screen);
    uint32_t xcb_generate_id(xcb_connection_t *c);
    xcb_generic_error_t *xcb_request_check(xcb_connection_t *c, xcb_void_cookie_t cookie);
""")

ffi.cdef("""
    unsigned int xcb_send_request(xcb_connection_t *c, int flags, struct iovec *vector, const xcb_protocol_request_t *request);
    void *xcb_wait_for_reply(xcb_connection_t *c, unsigned int request, xcb_generic_error_t **e);
    int xcb_poll_for_reply(xcb_connection_t *c, unsigned int request, void **reply, xcb_generic_error_t **error);
""")

C = ffi.verify("""
    #include <stdlib.h>
    #include <xcb/xcb.h>
    #include <xcb/xcbext.h>
""", libraries=['xcb'])

def bytes_to_cdata(bs):
    buf = ffi.new('char[]', bs)
    return buf

# cfficairo needs an xcb_visualtype_t
def visualtype_to_c_struct(vt):
    s = ffi.new("xcb_visualtype_t *")

    s.visual_id = vt.visual_id
    s._class = vt._class
    s.bits_per_rgb_value = vt.bits_per_rgb_value
    s.colormap_entries = vt.colormap_entries
    s.red_mask = vt.red_mask
    s.green_mask = vt.green_mask
    s.blue_mask = vt.blue_mask

    return s
<|endoftext|>