<commit_before>#!/usr/bin/env python

# Python 2/3 compatibility
from __future__ import (print_function, unicode_literals, division)
from future.standard_library import install_aliases
install_aliases()
from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request
from urllib.error import HTTPError

__metaclass__ = type

import httplib2
from apiclient.discovery import build
from oauth2client.client import flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import run

from IPython.display import display, HTML

import json
import pandas as pd
#pd.options.display.mpl_style = 'default'

from pprint import pprint

def ga():
    service = initialize_service()
    return service.data().ga()

def get_df(heading=None, **kwargs):
    service = initialize_service()
    result = service.data().ga().get(**kwargs).execute()
    df = results_wrapper(result, heading=heading)
    return df

def get_one(**kwargs):
    service = initialize_service()
    result = service.data().ga().get(**kwargs).execute()
    df = results_wrapper(result)
    if len(df.index) == 1 and len(df.columns) == 1:
        return df.iloc[0,0]
    else:
        raise ValueError(("result should contain just one value\n\n",
                         str(result)))

def scalar_type_sniff(scalar):
    try:
        scalar = float(scalar)
        if scalar == int(scalar):
            scalar = int(scalar)
    except:
        pass
    return scalar

def series_type_sniff(series):
    try:
        series = series.astype(float)
        if all(series == series.astype(int)):
            series = series.astype(int)
    except:
        pass
    return series

def results_wrapper(data,
                    print_notes=True,
                    set_index=None,
                    heading=None):

    def ga_clean(varname):
        return varname.replace('ga:', '')

    def add_df_heading(df, heading):
        return pd.concat([df], axis=1, keys=[heading])

    if print_notes:
        query = data['id']
        print(query)

    col_heads = [ga_clean(c.get('name')) for c in data['columnHeaders']]
    data_rows = data.get('rows')

    if not data_rows:
        print("No results")
        return None

    assert len(col_heads) == len(data_rows[0])

    data_row_dicts = [{k:v for (k, v) in zip(col_heads, row)}
                      for row in data_rows]

    df = pd.DataFrame(data_row_dicts)

    for c in df.columns:
        df[c] = series_type_sniff(df[c])

    if print_notes:
        warning = u"Result {} contain sampled data.".format(
            {True: u"DOES", False: u"does not"}
            [data['containsSampledData']])
        print(warning)

    if set_index:
        df.set_index(set_index, inplace=True)

    if heading:
        return add_df_heading(df, heading)
    else:
        return df

class ga_context:
    '''
    initialize a GA service instance with a view_id and
    a date range, to provide a shorthand for multiple related queries.

    usage:
        c = ga_context(view_id='ga:11111111', ('2015-01-01','2015-03-31'), label='Q1 2015')

        df1 = c.get() # default values: returns a dataframe with one number, total pageviews

        df2 = c.get(dimensions='ga:pageTitle') # titles by pageview count

        df3 = c.get(metrics='ga:pageviews', # default
                    sort='-ga:pageviews',   # default
                    filters='ga:pagePath=@site.com/folder,' #default=None
                            'ga:pagePath=@site.com/etc',
                    dimensions='ga:pageTitle',              #default=None
                    samplingLevel='HIGHER_PRECISION',
                    )
    '''
    # Default kwargs in a context object are set as each query is made, which
    # can produce unexpected results - that functionality should be removed.
    # (but for safety the full query URL is printed, so default values can be read off.)

    def __init__(self, view_id, date_range=None, start_date=None, end_date=None, label=None):
        if date_range:
            start_date = date_range[0]
            end_date = date_range[1]

        if not start_date and end_date:
            raise ValueError('date range or start/end dates must be specified')

        self.view_id = view_id
        self.start_date = start_date
        self.end_date = end_date
        self.service = initialize_service()
        self.label = label
        self.default_query = dict(metrics='ga:pageviews',
                                  filters=None,
                                  dimensions=None,
                                  sort='-ga:pageviews',
                                  max_results=1000,
                                  samplingLevel='HIGHER_PRECISION')

    def update_default_query(self, **update_items):
        self.default_query.update(update_items)

    def description(self):
        return (u'{l} ({s} to {e})'.format(l=self.label,
                                           s=self.start_date,
                                           e=self.end_date))

    def get(self, raw=False, show_heading=False, **kwargs):
        kw_default = self.default_query
        kw_default.update(kwargs)
        get_obj = self.service.data().ga().get(
            ids=self.view_id,
            start_date=self.start_date,
            end_date=self.end_date,
            #metrics='ga:pageviews',
            #sort='-ga:pageviews',
            #filters=filters,
            #dimensions=dimensions,
            #samplingLevel='HIGHER_PRECISION',
            **kw_default
            )
        if self.label:
            print(self.description())
        result = get_obj.execute()
        if raw:
            return result
        if show_heading:
            df_out = results_wrapper(result, heading=self.description())
        else:
            df_out = results_wrapper(result)
        return df_out

    def get_one(self, **kwargs):
        '''
        returns a scalar instead of a dataframe, and ensures that only one
        value was returned by the API.
        '''
        df = self.get(**kwargs)
        if df is None:
            print('(no results)')
            return None
        if len(df.index) == 1 and len(df.columns) == 1:
            return df.iloc[0,0]
        else:
            raise ValueError(("result should contain just one value\n\n",
                              str(df)))

# Declare constants and set configuration values

# The file with the OAuth 2.0 Client details for authentication and authorization.
from os import path
CLIENT_SECRETS = path.join(path.dirname(__file__), 'csec.json')
print('OAuth 2.0 Client auth secrets file: ', CLIENT_SECRETS)

# A file to store the access token
TOKEN_FILE_NAME = path.join(path.dirname(__file__), 'analytics.dat')

# The Flow object to be used if we need to authenticate.
FLOW = flow_from_clientsecrets(CLIENT_SECRETS,
    scope='https://www.googleapis.com/auth/analytics.readonly',
    message='client secrets JSON is missing: %s' % CLIENT_SECRETS)

def prepare_credentials():
    # Retrieve existing credendials
    storage = Storage(TOKEN_FILE_NAME)
    credentials = storage.get()

    # If existing credentials are invalid and Run Auth flow
    # the run method will store any new credentials
    if credentials is None or credentials.invalid:
        credentials = run(FLOW, storage)
        #run Auth Flow and store credentials
    return credentials

def initialize_service():
    # 1. Create an http object
    http = httplib2.Http()

    # 2. Authorize the http object
    # In this tutorial we first try to retrieve stored credentials. If
    # none are found then run the Auth Flow. This is handled by the
    # prepare_credentials() function defined earlier in the tutorial
    credentials = prepare_credentials()
    http = credentials.authorize(http)  # authorize the http object

    # 3. Build the Analytics Service Object with the authorized http object
    return build('analytics', 'v3', http=http)

#-----------------------------------------------------------
# additional convenience functions

def context_comparison(context_list, query_dict,
                       set_index=None, collapse_column_labels=False):

    dfs = {c.label: c.get(**query_dict)
          for c in context_list}
    if set_index:
        for k in dfs.keys():
            dfs[k] = dfs[k].set_index(set_index)

    outdf = pd.concat(dfs, axis=1)
    if collapse_column_labels:
        outdf.columns = ['{}: {}'.format(a, b)
                         for (a, b) in outdf.columns]
    return outdf

def query_comparison(query_dict_dict, context,
                     set_index=None, collapse_column_labels=False):

    dfs = {qkey: context.get(**query)
          for qkey, query in query_dict_dict.items()}
    if set_index:
        for k in dfs.keys():
            dfs[k] = dfs[k].set_index(set_index)

    outdf = pd.concat(dfs, axis=1)
    print(outdf.columns)
    if collapse_column_labels:
        outdf.columns = ['{}: {}'.format(a, b)
                         for (a, b) in outdf.columns]
    return outdf
<commit_msg>notify about sampling level and default to high precision; remove default ga:pageviews sorting if ga:pageviews not in metrics<commit_after>#!/usr/bin/env python

# Python 2/3 compatibility
from __future__ import (print_function, unicode_literals, division)
from future.standard_library import install_aliases
install_aliases()
from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request
from urllib.error import HTTPError

__metaclass__ = type

import httplib2
from apiclient.discovery import build
from oauth2client.client import flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import run

from IPython.display import display, HTML

import json
import pandas as pd
#pd.options.display.mpl_style = 'default'

from pprint import pprint

def ga():
    service = initialize_service()
    return service.data().ga()

def get_df(heading=None, **kwargs):
    service = initialize_service()
    result = service.data().ga().get(**kwargs).execute()
    df = results_wrapper(result, heading=heading)
    return df

def get_one(**kwargs):
    service = initialize_service()
    result = service.data().ga().get(**kwargs).execute()
    df = results_wrapper(result)
    if len(df.index) == 1 and len(df.columns) == 1:
        return df.iloc[0,0]
    else:
        raise ValueError(("result should contain just one value\n\n",
                         str(result)))

def scalar_type_sniff(scalar):
    try:
        scalar = float(scalar)
        if scalar == int(scalar):
            scalar = int(scalar)
    except:
        pass
    return scalar

def series_type_sniff(series):
    try:
        series = series.astype(float)
        if all(series == series.astype(int)):
            series = series.astype(int)
    except:
        pass
    return series

def results_wrapper(data,
                    print_notes=True,
                    set_index=None,
                    heading=None):

    def ga_clean(varname):
        return varname.replace('ga:', '')

    def add_df_heading(df, heading):
        return pd.concat([df], axis=1, keys=[heading])

    if print_notes:
        query = data['id']
        sampling_level = data['query'].get('samplingLevel')
        if sampling_level == 'HIGHER_PRECISION':
            sampling_level = 'h.p.'
        print(query, sampling_level)

    col_heads = [ga_clean(c.get('name')) for c in data['columnHeaders']]
    data_rows = data.get('rows')

    if not data_rows:
        print("No results")
        return None

    assert len(col_heads) == len(data_rows[0])

    data_row_dicts = [{k:v for (k, v) in zip(col_heads, row)}
                      for row in data_rows]

    df = pd.DataFrame(data_row_dicts)

    for c in df.columns:
        df[c] = series_type_sniff(df[c])

    if print_notes:
        warning = u"Result {} contain sampled data.".format(
            {True: u"DOES", False: u"does not"}
            [data['containsSampledData']])
        print(warning)

    if set_index:
        df.set_index(set_index, inplace=True)

    if heading:
        return add_df_heading(df, heading)
    else:
        return df

class ga_context:
    '''
    initialize a GA service instance with a view_id and
    a date range, to provide a shorthand for multiple related queries.

    usage:
        c = ga_context(view_id='ga:11111111', ('2015-01-01','2015-03-31'), label='Q1 2015')

        df1 = c.get() # default values: returns a dataframe with one number, total pageviews

        df2 = c.get(dimensions='ga:pageTitle') # titles by pageview count

        df3 = c.get(metrics='ga:pageviews', # default
                    sort='-ga:pageviews',   # default
                    filters='ga:pagePath=@site.com/folder,' #default=None
                            'ga:pagePath=@site.com/etc',
                    dimensions='ga:pageTitle',              #default=None
                    samplingLevel='HIGHER_PRECISION',       # default = HIGHER_PRECISION
                    )
    '''
    # Default kwargs in a context object are set as each query is made, which
    # can produce unexpected results - that functionality should be removed.
    # (but for safety the full query URL is printed, so default values can be read off.)

    def __init__(self, view_id, date_range=None, start_date=None, end_date=None, label=None):
        if date_range:
            start_date = date_range[0]
            end_date = date_range[1]

        if not start_date and end_date:
            raise ValueError('date range or start/end dates must be specified')

        self.view_id = view_id
        self.start_date = start_date
        self.end_date = end_date
        self.service = initialize_service()
        self.label = label

    def description(self):
        return (u'{l} ({s} to {e})'.format(l=self.label,
                                           s=self.start_date,
                                           e=self.end_date))

    def get(self, raw=False, show_heading=False, **kwargs):
        # default values, to be overriden by kwargs where needed
        query = dict(ids=self.view_id,
                     start_date=self.start_date,
                     end_date=self.end_date,
                     metrics='ga:pageviews',
                     filters=None,
                     sort='-ga:pageviews',
                     dimensions=None,
                     max_results=1000,
                     samplingLevel='HIGHER_PRECISION',
                    )

        # Setting up sensible defaults for fast querying:

        # default to sorting by descending pageviews, unless pageviews
        # aren't among the metrics requested
        if 'metrics' not in kwargs: # defaulting to pageviews metric
            pass
        else:
            # default to descending sort by the first metric listed
            mets = kwargs['metrics'].split(',')
            query['sort'] = '-%s' % mets[0]

        query.update(kwargs)
        get_obj = self.service.data().ga().get(**query)

        if self.label:
            print(self.description())
        result = get_obj.execute()

        if raw:
            return result
        if show_heading:
            df_out = results_wrapper(result, heading=self.description())
        else:
            df_out = results_wrapper(result)
        return df_out

    def get_one(self, **kwargs):
        '''
        returns a scalar instead of a dataframe, and ensures that only one
        value was returned by the API.
        '''
        kwargs['sort']=None
        df = self.get(**kwargs)
        if df is None:
            print('(no results)')
            return None
        if len(df.index) == 1 and len(df.columns) == 1:
            return df.iloc[0,0]
        else:
            raise ValueError(("result should contain just one value\n\n",
                              str(df)))

# Declare constants and set configuration values

# The file with the OAuth 2.0 Client details for authentication and authorization.
from os import path
CLIENT_SECRETS = path.join(path.dirname(__file__), 'csec.json')
print('OAuth 2.0 Client auth secrets file: ', CLIENT_SECRETS)

# A file to store the access token
TOKEN_FILE_NAME = path.join(path.dirname(__file__), 'analytics.dat')

# The Flow object to be used if we need to authenticate.
FLOW = flow_from_clientsecrets(CLIENT_SECRETS,
    scope='https://www.googleapis.com/auth/analytics.readonly',
    message='client secrets JSON is missing: %s' % CLIENT_SECRETS)

def prepare_credentials():
    # Retrieve existing credendials
    storage = Storage(TOKEN_FILE_NAME)
    credentials = storage.get()

    # If existing credentials are invalid and Run Auth flow
    # the run method will store any new credentials
    if credentials is None or credentials.invalid:
        credentials = run(FLOW, storage)
        #run Auth Flow and store credentials
    return credentials

def initialize_service():
    # 1. Create an http object
    http = httplib2.Http()

    # 2. Authorize the http object
    # In this tutorial we first try to retrieve stored credentials. If
    # none are found then run the Auth Flow. This is handled by the
    # prepare_credentials() function defined earlier in the tutorial
    credentials = prepare_credentials()
    http = credentials.authorize(http)  # authorize the http object

    # 3. Build the Analytics Service Object with the authorized http object
    return build('analytics', 'v3', http=http)

#-----------------------------------------------------------
# additional convenience functions

def context_comparison(context_list, query_dict,
                       set_index=None, collapse_column_labels=False):

    dfs = {c.label: c.get(**query_dict)
          for c in context_list}
    if set_index:
        for k in dfs.keys():
            dfs[k] = dfs[k].set_index(set_index)

    outdf = pd.concat(dfs, axis=1)
    if collapse_column_labels:
        outdf.columns = ['{}: {}'.format(a, b)
                         for (a, b) in outdf.columns]
    return outdf

def query_comparison(query_dict_dict, context,
                     set_index=None, collapse_column_labels=False):

    dfs = {qkey: context.get(**query)
          for qkey, query in query_dict_dict.items()}
    if set_index:
        for k in dfs.keys():
            dfs[k] = dfs[k].set_index(set_index)

    outdf = pd.concat(dfs, axis=1)
    print(outdf.columns)
    if collapse_column_labels:
        outdf.columns = ['{}: {}'.format(a, b)
                         for (a, b) in outdf.columns]
    return outdf
<|endoftext|>


<commit_before>from setuptools import setup, find_packages
import os

version = '1.0'

setup(name='faust.blog',
      version=version,
      description="Blogging facilities for faust site",
      long_description=open("README.txt").read() + "\n" +
                       open(os.path.join("docs", "HISTORY.txt")).read(),
      # Get more strings from
      # http://pypi.python.org/pypi?%3Aaction=list_classifiers
      classifiers=[
        "Framework :: Plone",
        "Programming Language :: Python",
        "Topic :: Software Development :: Libraries :: Python Modules",
        ],
      keywords='',
      author='Vorwaerts Werbung GbR',
      author_email='cb@vorwaerts-werbung.de',
      url='http://dist.vorwaerts-werbung.de',
      license='GPL',
      packages=find_packages(exclude=['ez_setup']),
      namespace_packages=['faust'],
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          'setuptools',
          'plone.app.dexterity',
          'plone.namedfile [blobs]',
          'plone.formwidget.recaptcha',
          'collective.upload',
          'sc.social.like',
          'fourdigits.portlet.twitter',
          # -*- Extra requirements: -*-
      ],
      entry_points="""
      # -*- Entry points: -*-
      [z3c.autoinclude.plugin]
      target = plone
      """,
      # The next two lines may be deleted after you no longer need
      # addcontent support from paster and before you distribute
      # your package.
      setup_requires=["PasteScript"],
      paster_plugins = ["ZopeSkel"],

      )
<commit_msg>Add grok dependency<commit_after>from setuptools import setup, find_packages
import os

version = '1.0'

setup(name='faust.blog',
      version=version,
      description="Blogging facilities for faust site",
      long_description=open("README.txt").read() + "\n" +
                       open(os.path.join("docs", "HISTORY.txt")).read(),
      # Get more strings from
      # http://pypi.python.org/pypi?%3Aaction=list_classifiers
      classifiers=[
        "Framework :: Plone",
        "Programming Language :: Python",
        "Topic :: Software Development :: Libraries :: Python Modules",
        ],
      keywords='',
      author='Vorwaerts Werbung GbR',
      author_email='cb@vorwaerts-werbung.de',
      url='http://dist.vorwaerts-werbung.de',
      license='GPL',
      packages=find_packages(exclude=['ez_setup']),
      namespace_packages=['faust'],
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          'setuptools',
          'plone.app.dexterity [grok, relations]',
          'plone.namedfile [blobs]',
          'plone.formwidget.recaptcha',
          'collective.upload',
          'sc.social.like',
          'fourdigits.portlet.twitter',
          # -*- Extra requirements: -*-
      ],
      entry_points="""
      # -*- Entry points: -*-
      [z3c.autoinclude.plugin]
      target = plone
      """,
      # The next two lines may be deleted after you no longer need
      # addcontent support from paster and before you distribute
      # your package.
      setup_requires=["PasteScript"],
      paster_plugins = ["ZopeSkel"],

      )
<|endoftext|>


<commit_before>#!/bin/env python

import sys
import traceback
from discord.ext import commands

class ErrorHandler:
    def __init__(self, bot):
        self.bot = bot

    async def on_command_error(self, ctx, error):
        """ Handle command errors more gracefully """

        if isinstance(error, commands.CommandNotFound):
            return

        if isinstance(error, commands.NotOwner):
            return await ctx.send('Sorry, only the owner of qtbot may run this command.')

        if isinstance(error, commands.CommandOnCooldown):
            return await ctx.send(f'This command is on cooldown. Please retry in `{error.retry_after:.0f}` second(s).')

        if isinstance(error, commands.MissingRequiredArgument):
            return await ctx.send(f'Command missing required argument `{error.param}`.')

        if isinstance(error, commands.MissingPermissions):
            return await ctx.send(f'Sorry you need permissions: `{",".join(error.missing_perms)}` to do that.')

        if isinstance(error, commands.BotMissingPermissions):
            return await ctx.send(f'Sorry I need permissions: `{",".join(error.missing_perms)}` to do that.')

        print(f'Ignoring exception in command {ctx.command}:', file=sys.stderr)
        traceback.print_exception(type(error), error, error.__traceback__, file=sys.stderr)

def setup(bot):
    bot.add_cog(ErrorHandler(bot))
<commit_msg>add bad arg error<commit_after>#!/bin/env python

import sys
import traceback
from discord.ext import commands

class ErrorHandler:
    def __init__(self, bot):
        self.bot = bot

    async def on_command_error(self, ctx, error):
        """ Handle command errors more gracefully """

        if isinstance(error, commands.CommandNotFound):
            return

        if isinstance(error, commands.NotOwner):
            return await ctx.send('Sorry, only the owner of qtbot may run this command.')

        if isinstance(error, commands.CommandOnCooldown):
            return await ctx.send(f'This command is on cooldown. Please retry in `{error.retry_after:.0f}` second(s).')

        if isinstance(error, commands.MissingRequiredArgument):
            return await ctx.send(f'Command missing required argument `{error.param}`.')

        if isinstance(error, commands.BadArgument):
            return await ctx.send(f'Bad argument: `{error.text}`.')

        if isinstance(error, commands.MissingPermissions):
            return await ctx.send(f'Sorry you need permissions: `{",".join(error.missing_perms)}` to do that.')

        if isinstance(error, commands.BotMissingPermissions):
            return await ctx.send(f'Sorry I need permissions: `{",".join(error.missing_perms)}` to do that.')

        print(f'Ignoring exception in command {ctx.command}:', file=sys.stderr)
        traceback.print_exception(type(error), error, error.__traceback__, file=sys.stderr)

def setup(bot):
    bot.add_cog(ErrorHandler(bot))
<|endoftext|>


<commit_before>import batch_script_utils as utils
import commands

import re

class Issue(object):
    """
    A class used to represent an issue.

    ...

    Attributes
    ----------
    verbose : bool
        Whether to print when commenting on and closing issue.
    issue : PyGithub issue
        Issue to create wrapper for.
    number : int
        The number of the issue on GitHub.
    body : dict
        The body of the issue, rehydrated into a dict.
    commands : list
        List of commands which caused the issue.
    resolved : bool
        Whether the issue has been resolved.

    Methods
    -------
    comment()
    Comments on the issue.
    close()
    Closes the issue.
    update()
    Updates the issue with the results of the commands, and updates the resolved status.
    get_commands()
    Returns the list of commands.
    test_commands()
    Tests each of the commands.
    stringify()
    Returns a formatted string.

    """

    def __init__(self, verbose, issue):
        self.verbose = verbose

        self.issue = issue
        self.number = issue.number
        self.body = {}
        bulletpoints = self.issue.body.split("\n-")[1:]
        for bulletpoint in bulletpoints:
            key, value = bulletpoint.split(":", 1)
            key = key.strip()
            value = value.strip()
            self.body[key] = value

        self.commands = []
        self.resolved = False

    def comment(self, comment):
        if self.verbose:
            print(f"Commenting on #{self.number}")
        self.issue.create_comment(comment)

    def close(self):
        if self.verbose:
            print(f"Closing #{self.number}")
        self.issue.edit(state="closed")

    def update(self):
        self.resolved = True
        comment = ""
        for command in self.commands:
            self.resolved = self.resolved and command.ran_successfully()
            comment = comment + "\n" + command.stringify()

        self.comment(comment)
        if self.resolved:
            self.close()

    def get_commands(self):
        return self.commands

    def test_commands(self):
        for command in self.commands:
            command.run()

    def stringify(self):
        return f"#{self.issue.number}\n" + "\n".join(
            [command.stringify_command() for command in self.commands]
        )

class RedcapUpdateSummaryScoresIssue(Issue):
    """
    Subclass of Issue used for redcap-update-summary-scores issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "requestError" not in self.body or "experiment_site_id" not in self.body:
            raise ValueError(
                f"#{self.number}\nrequestError or experiment_site_id not in body"
            )

        study_ids = utils.extract_unique_study_ids(self.body["requestError"])
        first_field = self.body["requestError"].split('","')[1]
        field_row = metadata[metadata["field_name"] == first_field]
        self.form = field_row["form_name"].item()
        experiment_site_id_regex = (
            f"(?:{utils.STUDY_ID_REGEX}_{utils.EVENT_REGEX}_)?(.*)-"
        )
        experiment_site_id_matches = re.match(
            experiment_site_id_regex, self.body["experiment_site_id"]
        )
        if not experiment_site_id_matches:
            raise ValueError(f"#{self.number}\nNo matches in experiment_site_id")
        instrument = experiment_site_id_matches.group(1)

        for study_id in study_ids:
            command = commands.RedcapUpdateSummaryScoresCommand(
                self.verbose, study_id, instrument
            )
            self.commands.append(command)

        if self.verbose:
            print(self.stringify())

class UpdateVisitDataIssue(Issue):
    """
    Subclass of Issue used for update_visit_data issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "requestError" in self.body:
            study_ids = utils.extract_unique_study_ids(self.body["requestError"])
            first_field = self.body["requestError"].split('","')[1]
            field_row = metadata[metadata["field_name"] == first_field]
            self.form = field_row["form_name"].item()
            imports_form = self.form
            if "limesurvey_ssaga" in self.form:
                lssaga_regex = "(lssaga[1234]_youth|lssaga[1234]_parent)"
                lssaga_matches = re.match(lssaga_regex, self.body["redcap_variable"])
                if not lssaga_matches:
                    raise ValueError(
                        f"#{self.number}\nNo lssaga form in redcap_variable"
                    )
                imports_form = lssaga_matches[0]
            for study_id in study_ids:
                command = commands.UpdateVisitDataCommand(
                    self.verbose, study_id, imports_form
                )
                self.commands.append(command)
        elif "form_name" in self.body:
            form = self.body["form_name"]
            study_ids = utils.extract_unique_study_ids(self.body["experiment_site_id"])
            if not study_ids:
                raise ValueError(f"#{self.number}\nNo study id in experiment_site_id")
            study_id = study_ids[0]
            command = commands.UpdateVisitDataCommand(self.verbose, study_id, form)
            self.commands.append(command)
        else:
            raise ValueError(f"#{self.number}\nrequestError and form_name not in body")

        if self.verbose:
            print(self.stringify())

class UpdateSummaryFormsIssue(Issue):
    """
    Subclass of Issue used for update_summary_forms issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "requestError" in self.body:
            study_ids = utils.extract_unique_study_ids(self.body["requestError"])
            first_field = self.body["requestError"].split('","')[1]
            field_row = metadata[metadata["field_name"] == first_field]
            self.form = field_row["form_name"].item()
            for study_id in study_ids:
                command = commands.UpdateSummaryFormsCommand(self.verbose, study_id)
                self.commands.append(command)
        elif "records_this_visit" in self.body:
            study_id = utils.extract_unique_study_ids(self.body["records_this_visit"])[
                0
            ]
            command = commands.UpdateSummaryFormsCommand(self.verbose, study_id)
            self.commands.append(command)
        else:
            raise ValueError(
                f"#{self.number}\nrequestError and records_this_visit not in body"
            )

        if self.verbose:
            print(self.stringify())

class ImportMRSessionsIssue(Issue):
    """
    Subclass of Issue used for import_mr_sessions issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "requestError" in self.body:
            study_ids = utils.extract_unique_study_ids(self.body["requestError"])
            first_field = self.body["requestError"].split('","')[1]
            field_row = metadata[metadata["field_name"] == first_field]
            self.form = field_row["form_name"].item()
            for study_id in study_ids:
                command = commands.ImportMRSessionsCommand(self.verbose, study_id)
                self.commands.append(command)
        elif "experiment_site_id" in self.body:
            study_ids = utils.extract_unique_study_ids(self.body["experiment_site_id"])
            if not study_ids:
                raise ValueError(f"#{self.number}\nNo study id in experiment_site_id")
            study_id = study_ids[0]
            command = commands.ImportMRSessionsCommand(self.verbose, study_id)
            self.commands.append(command)
        else:
            raise ValueError(f"#{self.number}\nrequestError not in body")

        if self.verbose:
            print(self.stringify())

class CheckNewSessionsIssue(Issue):
    """
    Subclass of Issue used for check_new_sessions issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "eid" in self.body:
            eid = self.body["eid"]
            command = commands.CheckNewSessionsCommand(self.verbose, eid)
            self.commands.append(command)
        else:
            raise ValueError(f"#{self.number}\neid not in body")

        if self.verbose:
            print(self.stringify())

class CheckPhantomScansIssue(Issue):
    """
    Subclass of Issue used for check_phantom_scans issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "experiment_id" in self.body:
            experiment_id = self.body["experiment_id"]
            command = commands.CheckPhantomScansCommand(self.verbose, experiment_id)
            self.commands.append(command)
        else:
            raise ValueError(f"#{self.number}\nexperiment_id not in body")

        if self.verbose:
            print(self.stringify())

class UpdateBulkFormsIssue(Issue):
    """
    Subclass of Issue used for update_bulk_forms issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "form" in self.body:
            self.form = self.body["form"]
        else:
            raise ValueError(f"#{self.number}\nNo form in issue body")
        if "experiment_site_id" in self.body:
            study_ids = utils.extract_unique_study_ids(self.body["experiment_site_id"])
            if not study_ids:
                raise ValueError(f"#{self.number}\nNo study id in experiment_site_id")
            study_id = study_ids[0]
            command = commands.UpdateBulkFormsCommand(self.verbose, study_id)
            self.commands.append(command)
        else:
            raise ValueError(f"#{self.number}\nNo experiment_site_id in issue body")

        if self.verbose:
            print(self.stringify())
<commit_msg>Getting first group from ssaga regex<commit_after>import batch_script_utils as utils
import commands

import re

class Issue(object):
    """
    A class used to represent an issue.

    ...

    Attributes
    ----------
    verbose : bool
        Whether to print when commenting on and closing issue.
    issue : PyGithub issue
        Issue to create wrapper for.
    number : int
        The number of the issue on GitHub.
    body : dict
        The body of the issue, rehydrated into a dict.
    commands : list
        List of commands which caused the issue.
    resolved : bool
        Whether the issue has been resolved.

    Methods
    -------
    comment()
    Comments on the issue.
    close()
    Closes the issue.
    update()
    Updates the issue with the results of the commands, and updates the resolved status.
    get_commands()
    Returns the list of commands.
    test_commands()
    Tests each of the commands.
    stringify()
    Returns a formatted string.

    """

    def __init__(self, verbose, issue):
        self.verbose = verbose

        self.issue = issue
        self.number = issue.number
        self.body = {}
        bulletpoints = self.issue.body.split("\n-")[1:]
        for bulletpoint in bulletpoints:
            key, value = bulletpoint.split(":", 1)
            key = key.strip()
            value = value.strip()
            self.body[key] = value

        self.commands = []
        self.resolved = False

    def comment(self, comment):
        if self.verbose:
            print(f"Commenting on #{self.number}")
        self.issue.create_comment(comment)

    def close(self):
        if self.verbose:
            print(f"Closing #{self.number}")
        self.issue.edit(state="closed")

    def update(self):
        self.resolved = True
        comment = ""
        for command in self.commands:
            self.resolved = self.resolved and command.ran_successfully()
            comment = comment + "\n" + command.stringify()

        self.comment(comment)
        if self.resolved:
            self.close()

    def get_commands(self):
        return self.commands

    def test_commands(self):
        for command in self.commands:
            command.run()

    def stringify(self):
        return f"#{self.issue.number}\n" + "\n".join(
            [command.stringify_command() for command in self.commands]
        )

class RedcapUpdateSummaryScoresIssue(Issue):
    """
    Subclass of Issue used for redcap-update-summary-scores issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "requestError" not in self.body or "experiment_site_id" not in self.body:
            raise ValueError(
                f"#{self.number}\nrequestError or experiment_site_id not in body"
            )

        study_ids = utils.extract_unique_study_ids(self.body["requestError"])
        first_field = self.body["requestError"].split('","')[1]
        field_row = metadata[metadata["field_name"] == first_field]
        self.form = field_row["form_name"].item()
        experiment_site_id_regex = (
            f"(?:{utils.STUDY_ID_REGEX}_{utils.EVENT_REGEX}_)?(.*)-"
        )
        experiment_site_id_matches = re.match(
            experiment_site_id_regex, self.body["experiment_site_id"]
        )
        if not experiment_site_id_matches:
            raise ValueError(f"#{self.number}\nNo matches in experiment_site_id")
        instrument = experiment_site_id_matches.group(1)

        for study_id in study_ids:
            command = commands.RedcapUpdateSummaryScoresCommand(
                self.verbose, study_id, instrument
            )
            self.commands.append(command)

        if self.verbose:
            print(self.stringify())

class UpdateVisitDataIssue(Issue):
    """
    Subclass of Issue used for update_visit_data issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "requestError" in self.body:
            study_ids = utils.extract_unique_study_ids(self.body["requestError"])
            first_field = self.body["requestError"].split('","')[1]
            field_row = metadata[metadata["field_name"] == first_field]
            self.form = field_row["form_name"].item()
            imports_form = self.form
            if "limesurvey_ssaga" in self.form:
                lssaga_regex = "(lssaga[1234]_youth|lssaga[1234]_parent)"
                lssaga_matches = re.match(lssaga_regex, self.body["redcap_variable"])
                if not lssaga_matches:
                    raise ValueError(
                        f"#{self.number}\nNo lssaga form in redcap_variable"
                    )
                imports_form = lssaga_matches.group(1)
            for study_id in study_ids:
                command = commands.UpdateVisitDataCommand(
                    self.verbose, study_id, imports_form
                )
                self.commands.append(command)
        elif "form_name" in self.body:
            form = self.body["form_name"]
            study_ids = utils.extract_unique_study_ids(self.body["experiment_site_id"])
            if not study_ids:
                raise ValueError(f"#{self.number}\nNo study id in experiment_site_id")
            study_id = study_ids[0]
            command = commands.UpdateVisitDataCommand(self.verbose, study_id, form)
            self.commands.append(command)
        else:
            raise ValueError(f"#{self.number}\nrequestError and form_name not in body")

        if self.verbose:
            print(self.stringify())

class UpdateSummaryFormsIssue(Issue):
    """
    Subclass of Issue used for update_summary_forms issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "requestError" in self.body:
            study_ids = utils.extract_unique_study_ids(self.body["requestError"])
            first_field = self.body["requestError"].split('","')[1]
            field_row = metadata[metadata["field_name"] == first_field]
            self.form = field_row["form_name"].item()
            for study_id in study_ids:
                command = commands.UpdateSummaryFormsCommand(self.verbose, study_id)
                self.commands.append(command)
        elif "records_this_visit" in self.body:
            study_id = utils.extract_unique_study_ids(self.body["records_this_visit"])[
                0
            ]
            command = commands.UpdateSummaryFormsCommand(self.verbose, study_id)
            self.commands.append(command)
        else:
            raise ValueError(
                f"#{self.number}\nrequestError and records_this_visit not in body"
            )

        if self.verbose:
            print(self.stringify())

class ImportMRSessionsIssue(Issue):
    """
    Subclass of Issue used for import_mr_sessions issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "requestError" in self.body:
            study_ids = utils.extract_unique_study_ids(self.body["requestError"])
            first_field = self.body["requestError"].split('","')[1]
            field_row = metadata[metadata["field_name"] == first_field]
            self.form = field_row["form_name"].item()
            for study_id in study_ids:
                command = commands.ImportMRSessionsCommand(self.verbose, study_id)
                self.commands.append(command)
        elif "experiment_site_id" in self.body:
            study_ids = utils.extract_unique_study_ids(self.body["experiment_site_id"])
            if not study_ids:
                raise ValueError(f"#{self.number}\nNo study id in experiment_site_id")
            study_id = study_ids[0]
            command = commands.ImportMRSessionsCommand(self.verbose, study_id)
            self.commands.append(command)
        else:
            raise ValueError(f"#{self.number}\nrequestError not in body")

        if self.verbose:
            print(self.stringify())

class CheckNewSessionsIssue(Issue):
    """
    Subclass of Issue used for check_new_sessions issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "eid" in self.body:
            eid = self.body["eid"]
            command = commands.CheckNewSessionsCommand(self.verbose, eid)
            self.commands.append(command)
        else:
            raise ValueError(f"#{self.number}\neid not in body")

        if self.verbose:
            print(self.stringify())

class CheckPhantomScansIssue(Issue):
    """
    Subclass of Issue used for check_phantom_scans issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "experiment_id" in self.body:
            experiment_id = self.body["experiment_id"]
            command = commands.CheckPhantomScansCommand(self.verbose, experiment_id)
            self.commands.append(command)
        else:
            raise ValueError(f"#{self.number}\nexperiment_id not in body")

        if self.verbose:
            print(self.stringify())

class UpdateBulkFormsIssue(Issue):
    """
    Subclass of Issue used for update_bulk_forms issues.

    """

    def __init__(self, verbose, issue, metadata):
        Issue.__init__(self, verbose, issue)

        if "form" in self.body:
            self.form = self.body["form"]
        else:
            raise ValueError(f"#{self.number}\nNo form in issue body")
        if "experiment_site_id" in self.body:
            study_ids = utils.extract_unique_study_ids(self.body["experiment_site_id"])
            if not study_ids:
                raise ValueError(f"#{self.number}\nNo study id in experiment_site_id")
            study_id = study_ids[0]
            command = commands.UpdateBulkFormsCommand(self.verbose, study_id)
            self.commands.append(command)
        else:
            raise ValueError(f"#{self.number}\nNo experiment_site_id in issue body")

        if self.verbose:
            print(self.stringify())
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function

import platform
import sys

from cryptography.hazmat.bindings.utils import build_ffi

class Binding(object):
    """
    CommonCrypto API wrapper.
    """
    _module_prefix = "cryptography.hazmat.bindings.commoncrypto."
    _modules = [
        "cf",
        "common_digest",
        "common_hmac",
        "common_key_derivation",
        "common_cryptor",
        "security",
    ]

    ffi = None
    lib = None

    def __init__(self):
        self._ensure_ffi_initialized()

    @classmethod
    def _ensure_ffi_initialized(cls):
        if cls.ffi is not None and cls.lib is not None:
            return

        cls.ffi, cls.lib = build_ffi(
            module_prefix=cls._module_prefix,
            modules=cls._modules,
        )

    @classmethod
    def is_available(cls):
        return sys.platform == "darwin" and list(map(
            int, platform.mac_ver()[0].split("."))) >= [10, 8, 0]
<commit_msg>add new link args<commit_after># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function

import platform
import sys

from cryptography.hazmat.bindings.utils import build_ffi

class Binding(object):
    """
    CommonCrypto API wrapper.
    """
    _module_prefix = "cryptography.hazmat.bindings.commoncrypto."
    _modules = [
        "cf",
        "common_digest",
        "common_hmac",
        "common_key_derivation",
        "common_cryptor",
        "security",
    ]

    ffi = None
    lib = None

    def __init__(self):
        self._ensure_ffi_initialized()

    @classmethod
    def _ensure_ffi_initialized(cls):
        if cls.ffi is not None and cls.lib is not None:
            return

        cls.ffi, cls.lib = build_ffi(
            module_prefix=cls._module_prefix,
            modules=cls._modules,
            extra_link_args=["-framework", "Security"]
        )

    @classmethod
    def is_available(cls):
        return sys.platform == "darwin" and list(map(
            int, platform.mac_ver()[0].split("."))) >= [10, 8, 0]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Django on App Engine documentation build configuration file, created by
# sphinx-quickstart on Mon Nov 14 11:22:26 2011.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.todo']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Django on App Engine'
copyright = u'2011, Wilfred Hughes, allbuttonspressed'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.3'
# The full version, including alpha/beta/rc tags.
release = '1.3'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'DjangoonAppEnginedoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'DjangoonAppEngine.tex', u'Django on App Engine Documentation',
   u'Wilfred Hughes, allbuttonspressed', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'djangoonappengine', u'Django on App Engine Documentation',
     [u'Wilfred Hughes, allbuttonspressed'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'DjangoonAppEngine', u'Django on App Engine Documentation', u'Wilfred Hughes, allbuttonspressed',
   'DjangoonAppEngine', 'One line description of project.', 'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
<commit_msg>changing name from django-appengine to django nonrel<commit_after># -*- coding: utf-8 -*-
#
# Django on App Engine documentation build configuration file, created by
# sphinx-quickstart on Mon Nov 14 11:22:26 2011.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.todo']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Django nonrel'
copyright = u'2011, Wilfred Hughes, allbuttonspressed'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.3'
# The full version, including alpha/beta/rc tags.
release = '1.3'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'DjangoNonreldoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'DjangoNonrel.tex', u'Django Nonrel',
   u'Wilfred Hughes, allbuttonspressed', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'djangononrel', u'Django Nonrel',
     [u'Wilfred Hughes, allbuttonspressed'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'DjangoNonrel', u'Django Nonrel', u'Wilfred Hughes, allbuttonspressed',
   'Djangononrel', 'One line description of project.', 'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import glob

from multiprocessing.dummy import Pool as ThreadPool
from Directory_Setup import Directory_Setup
from Run_Transdecoder import Run_Transdecoder

"""
To execute, run in bash `./Main.py *.[FILE EXTENSION]

For example, to run on all .fasta files, do `./Main.py *.fasta`
"""

def run_commands(file):
        #print file
        ds = Directory_Setup(file)
        rt = Run_Transdecoder(ds.get_new_path())

def globbing(ext):
        """
        Locate all files ending with a file extension provided to the constructor
        and pipes each of them to the Transdecoder Program.

        """
        files = glob.glob(ext)

        pool = ThreadPool(4)

        pool.map(run_commands, files)

        pool.close()
        pool.join()

        print "Done with globbing"

if __name__ == "__main__":
        print "Running TransDecoder on all %s files in current working directory" % sys.argv[1]
        print ""
        globbing(sys.argv[1])<commit_msg>add documentation and refractor functions<commit_after>#!/usr/bin/python
import sys
import glob

from multiprocessing.dummy import Pool as ThreadPool
from Directory_Setup import Directory_Setup
from Run_Transdecoder import Run_Transdecoder

"""
To execute, run in bash `./Main.py *.[FILE EXTENSION]

For example, to run on all .fasta files, do `./Main.py *.fasta`
"""

def runFileTransdecoder(file):
        """
        Pipe file to Transdecoder program and sets up output and input directory
        before processing it.
        """
        ds = Directory_Setup(file)
        rt = Run_Transdecoder(ds.get_new_path())

def parallelizeFileProcessing(ext):
        """
        Locate all files ending with a file extension provided to the constructor
        and pipes each of them to the Transdecoder Program in parallel.

        """
        files = glob.glob(ext)

        pool = ThreadPool(4)

        pool.map(runFileTransdecoder, files)

        pool.close()
        pool.join()

        print "Completed parallel processing of files"

if __name__ == "__main__":
        print "Running TransDecoder on all %s files in current working directory" % sys.argv[1]
        print ""
        parallelizeFileProcessing(sys.argv[1])<|endoftext|>


<commit_before>import uuid
from dateutil import parser
from pytz import timezone

from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from casexml.apps.case.const import CASE_INDEX_EXTENSION, UNOWNED_EXTENSION_OWNER_ID
from corehq.form_processor.exceptions import CaseNotFound
from dimagi.utils.decorators.memoized import memoized

from casexml.apps.case.mock import CaseFactory, CaseStructure, CaseIndex
from custom.enikshay.const import ENIKSHAY_TIMEZONE
from custom.enikshay.integrations.ninetyninedots.exceptions import AdherenceException
from custom.enikshay.case_utils import get_open_episode_case_from_person, get_adherence_cases_between_dates
from custom.enikshay.exceptions import ENikshayCaseNotFound
from custom.enikshay.tasks import EpisodeUpdater

class AdherenceCaseFactory(object):
    """
    Creates and updates adherence cases for a given person

    Only for use by the 99DOTS integration with eNikshay
    """
    DEFAULT_ADHERENCE_CONFIDENCE = "medium"
    ADHERENCE_CASE_TYPE = "adherence"
    DEFAULT_ADHERENCE_VALUE = "unobserved_dose"

    def __init__(self, domain, person_id):
        self.domain = domain
        self.person_id = person_id

        self.case_accessor = CaseAccessors(domain)
        self.case_factory = CaseFactory(domain)

    @property
    @memoized
    def _person_case(self):
        try:
            return self.case_accessor.get_case(self.person_id)
        except CaseNotFound:
            raise AdherenceException("No patient exists with this beneficiary ID")

    @property
    @memoized
    def _default_adherence_confidence(self):
        return self._episode_case.dynamic_case_properties().get(
            'default_adherence_confidence', self.DEFAULT_ADHERENCE_CONFIDENCE
        )

    @property
    @memoized
    def _episode_case(self):
        try:
            return get_open_episode_case_from_person(self.domain, self._person_case.case_id)
        except ENikshayCaseNotFound as e:
            raise AdherenceException(e.message)

    def create_adherence_cases(self, adherence_points):
        return self.case_factory.create_or_update_cases([
            CaseStructure(
                case_id=uuid.uuid4().hex,
                attrs={
                    "case_type": self.ADHERENCE_CASE_TYPE,
                    "owner_id": UNOWNED_EXTENSION_OWNER_ID,
                    "create": True,
                    "update": self._get_adherence_case_properties(adherence_point),
                },
                indices=[CaseIndex(
                    CaseStructure(case_id=self._episode_case.case_id, attrs={"create": False}),
                    identifier='host',
                    relationship=CASE_INDEX_EXTENSION,
                    related_type=self._episode_case.type,
                )],
                walk_related=False,
            )
            for adherence_point in adherence_points
        ])

    def _get_adherence_case_properties(self, adherence_point):
        return {
            "name": adherence_point.get("timestamp", None),
            "adherence_value": self.DEFAULT_ADHERENCE_VALUE,
            "adherence_source": adherence_point.get('adherenceSource', '99DOTS'),
            "adherence_date": self._parse_adherence_date(adherence_point["timestamp"]),
            "person_name": self._person_case.name,
            "adherence_confidence": self._default_adherence_confidence,
            "shared_number_99_dots": adherence_point["sharedNumber"],
        }

    def _parse_adherence_date(self, iso_datestring):
        tz = timezone(ENIKSHAY_TIMEZONE)
        try:
            datetime_from_adherence = parser.parse(iso_datestring)
            datetime_in_india = datetime_from_adherence.astimezone(tz)
        except ValueError:
            raise AdherenceException(
                "Adherence date should be an ISO8601 formated string with timezone information."
            )
        return datetime_in_india.date()

    def update_adherence_cases(self, start_date, end_date, confidence_level):
        try:
            adherence_cases = get_adherence_cases_between_dates(self.domain, self.person_id, start_date, end_date)
        except ENikshayCaseNotFound as e:
            raise AdherenceException(e.message)
        adherence_case_ids = [case.case_id for case in adherence_cases]
        return self.case_factory.create_or_update_cases([
            CaseStructure(
                case_id=adherence_case_id,
                attrs={
                    "create": False,
                    "update": {
                        "adherence_confidence": confidence_level
                    },
                },
                walk_related=False
            ) for adherence_case_id in adherence_case_ids
        ])

    def update_default_confidence_level(self, confidence_level):
        return self.case_factory.create_or_update_cases([
            CaseStructure(
                case_id=self._episode_case.case_id,
                attrs={
                    "create": False,
                    "update": {
                        "default_adherence_confidence": confidence_level
                    },
                },
                walk_related=False
            )
        ])

def create_adherence_cases(domain, person_id, adherence_points):
    return AdherenceCaseFactory(domain, person_id).create_adherence_cases(adherence_points)

def update_adherence_confidence_level(domain, person_id, start_date, end_date, new_confidence):
    return AdherenceCaseFactory(domain, person_id).update_adherence_cases(start_date, end_date, new_confidence)

def update_default_confidence_level(domain, person_id, new_confidence):
    return AdherenceCaseFactory(domain, person_id).update_default_confidence_level(new_confidence)

def update_episode_adherence_properties(domain, person_id):
    try:
        episode_case = get_open_episode_case_from_person(domain, person_id)
    except ENikshayCaseNotFound as e:
        raise AdherenceException(e.message)
    try:
        updater = EpisodeUpdater(domain)
        updater.update_single_case(episode_case)
    except Exception as e:
        raise AdherenceException(
            "Error calculating updates for episode case_id({}): {}".format(
                episode_case.case_id,
                e
            )
        )
<commit_msg>Add MERM info to new adherence cases<commit_after>import uuid
from dateutil import parser
from pytz import timezone

from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from casexml.apps.case.const import CASE_INDEX_EXTENSION, UNOWNED_EXTENSION_OWNER_ID
from corehq.form_processor.exceptions import CaseNotFound
from dimagi.utils.decorators.memoized import memoized

from casexml.apps.case.mock import CaseFactory, CaseStructure, CaseIndex
from custom.enikshay.const import ENIKSHAY_TIMEZONE
from custom.enikshay.integrations.ninetyninedots.exceptions import AdherenceException
from custom.enikshay.case_utils import get_open_episode_case_from_person, get_adherence_cases_between_dates
from custom.enikshay.exceptions import ENikshayCaseNotFound
from custom.enikshay.tasks import EpisodeUpdater

class AdherenceCaseFactory(object):
    """
    Creates and updates adherence cases for a given person

    Only for use by the 99DOTS integration with eNikshay
    """
    DEFAULT_ADHERENCE_CONFIDENCE = "medium"
    ADHERENCE_CASE_TYPE = "adherence"
    DEFAULT_ADHERENCE_VALUE = "unobserved_dose"

    def __init__(self, domain, person_id):
        self.domain = domain
        self.person_id = person_id

        self.case_accessor = CaseAccessors(domain)
        self.case_factory = CaseFactory(domain)

    @property
    @memoized
    def _person_case(self):
        try:
            return self.case_accessor.get_case(self.person_id)
        except CaseNotFound:
            raise AdherenceException("No patient exists with this beneficiary ID")

    @property
    @memoized
    def _default_adherence_confidence(self):
        return self._episode_case.dynamic_case_properties().get(
            'default_adherence_confidence', self.DEFAULT_ADHERENCE_CONFIDENCE
        )

    @property
    @memoized
    def _episode_case(self):
        try:
            return get_open_episode_case_from_person(self.domain, self._person_case.case_id)
        except ENikshayCaseNotFound as e:
            raise AdherenceException(e.message)

    def create_adherence_cases(self, adherence_points):
        case_structures = []
        for adherence_point in adherence_points:
            if adherence_point.get('MERM_TypeOfEvent') == "HEARTBEAT":
                continue
            case_structures.append([
                CaseStructure(
                    case_id=uuid.uuid4().hex,
                    attrs={
                        "case_type": self.ADHERENCE_CASE_TYPE,
                        "owner_id": UNOWNED_EXTENSION_OWNER_ID,
                        "create": True,
                        "update": self._get_adherence_case_properties(adherence_point),
                    },
                    indices=[CaseIndex(
                        CaseStructure(case_id=self._episode_case.case_id, attrs={"create": False}),
                        identifier='host',
                        relationship=CASE_INDEX_EXTENSION,
                        related_type=self._episode_case.type,
                    )],
                    walk_related=False,
                )
            ])
        return self.case_factory.create_or_update_cases(case_structures)

    def _get_adherence_case_properties(self, adherence_point):
        return {
            "name": adherence_point.get("timestamp", None),
            "adherence_value": self.DEFAULT_ADHERENCE_VALUE,
            "adherence_source": adherence_point.get('adherenceSource', '99DOTS'),
            "adherence_date": self._parse_adherence_date(adherence_point["timestamp"]),
            "person_name": self._person_case.name,
            "adherence_confidence": self._default_adherence_confidence,
            "shared_number_99_dots": adherence_point.get("sharedNumber"),
            "merm_imei": adherence_point.get("MERM_IMEI"),
            "merm_extra_info": adherence_point.get("MERM_ExtraInformation"),
        }

    def _parse_adherence_date(self, iso_datestring):
        tz = timezone(ENIKSHAY_TIMEZONE)
        try:
            datetime_from_adherence = parser.parse(iso_datestring)
            datetime_in_india = datetime_from_adherence.astimezone(tz)
        except ValueError:
            raise AdherenceException(
                "Adherence date should be an ISO8601 formated string with timezone information."
            )
        return datetime_in_india.date()

    def update_adherence_cases(self, start_date, end_date, confidence_level):
        try:
            adherence_cases = get_adherence_cases_between_dates(self.domain, self.person_id, start_date, end_date)
        except ENikshayCaseNotFound as e:
            raise AdherenceException(e.message)
        adherence_case_ids = [case.case_id for case in adherence_cases]
        return self.case_factory.create_or_update_cases([
            CaseStructure(
                case_id=adherence_case_id,
                attrs={
                    "create": False,
                    "update": {
                        "adherence_confidence": confidence_level
                    },
                },
                walk_related=False
            ) for adherence_case_id in adherence_case_ids
        ])

    def update_default_confidence_level(self, confidence_level):
        return self.case_factory.create_or_update_cases([
            CaseStructure(
                case_id=self._episode_case.case_id,
                attrs={
                    "create": False,
                    "update": {
                        "default_adherence_confidence": confidence_level
                    },
                },
                walk_related=False
            )
        ])

def create_adherence_cases(domain, person_id, adherence_points):
    return AdherenceCaseFactory(domain, person_id).create_adherence_cases(adherence_points)

def update_adherence_confidence_level(domain, person_id, start_date, end_date, new_confidence):
    return AdherenceCaseFactory(domain, person_id).update_adherence_cases(start_date, end_date, new_confidence)

def update_default_confidence_level(domain, person_id, new_confidence):
    return AdherenceCaseFactory(domain, person_id).update_default_confidence_level(new_confidence)

def update_episode_adherence_properties(domain, person_id):
    try:
        episode_case = get_open_episode_case_from_person(domain, person_id)
    except ENikshayCaseNotFound as e:
        raise AdherenceException(e.message)
    try:
        updater = EpisodeUpdater(domain)
        updater.update_single_case(episode_case)
    except Exception as e:
        raise AdherenceException(
            "Error calculating updates for episode case_id({}): {}".format(
                episode_case.case_id,
                e
            )
        )
<|endoftext|>


<commit_before># Alex Ciaramella and Greg Suner
# Abstract Tournament Class

#Tournament is observable while players are observers

import Message

class Tournament(Observable):
    # set up a list of players when tournament is initialized
    def __init__(self):
        self.playerList = []
	self.game

    # run the tournament
    def run(self):
        self.begin_tournament()
        while(True):
        	match = self.create_next_match()
                if match == None:
                    break
                self.play_match(match)
        self.end_tournament()

    # get a reference to the next game to be played
    def create_next_match(self):
        pass

    # register a player for the tournament by adding them to       # the  list of current players
    def register_player(self, player):
        self.playerList.append(player)

    # stores a reference to the type of game we will be playing
    def set_game(self, game):
        self.game = game

    # play the next match and return the results
    def play_match(self, match):
        self.start_match(match[0])
        result = self.play_game(match)
	self.end_match(match[0], result)

    # plays each indvidual game in the match
    def play_game(self, match):
        for i in range(0, match[1]):
            self.start_game(match[0])
            result = match.get_results(match[0])
            # need to find out definite structure of results!!
            self.end_game(result[0], result[1], result[2])

	# notifies players tournament has begun
    def begin_tournament(self):
        pass

	# Announces results of tournament to all players
    def end_tournament(self):
        pass

    # send a message containing a list of all the players in the current match
    def start_match(self, players):
        message = Message.get_match_start_message(players)
        self.notify_all(message)

    # send a message containing the result of the match
    def end_match(self,players, result):
        message = Message.get_match_end_message(players, result)
        self.notify_all(message)

    # send a message containing the players in the next game
    def start_game(self, players):
        message = Message.get_round_start_message(players)
        self.notify_all(message)

    # send a message containing the players, moves, and result of the last game
    def end_game(players, moves, result):
        message = Message.get_round_end_message(players, moves, result)
        self.notify_all(message)
<commit_msg>deleted feature_Torunament.py<commit_after><|endoftext|>


<commit_before>import json
import logging
import aiohttp

logging.basicConfig(format='[%(levelname)s] - %(message)s', level=logging.INFO)
config = json.load(open('config.json'))

class Stats:
  def __init__(self, bot):
    self.bot = bot

  async def listpost():
    session = aiohttp.ClientSession()
    payload = json.dumps({
      'server_count': len(self.bot.guilds)
    })
    if config['dbotsorg'] == False:
      logging.info('not posting to discordbots.org')
    else:
      dbl_headers = {
        'Authorization': config['dbl'],
        'Content-Type': 'application/json'
      }
      dbl_url = 'https://discordbots.org/api/bots/' + config['botid'] + '/stats'
      async with session.post(dbl_url, data=payload, headers=dbl_headers) as dbl_resp:
        if dbl_resp.status == 200:
          logging.info('dbl: posted!')
        else:
          logging.info(f'dbl: something weird happened: code {dbl_resp.statu}.')

    if config['dbotspw'] == False:
      logging.info('not posting to bots.discord.pw')
    else:
      headers = {
        'Authorization': config['dbots'],
        'Content-Type': 'application/json'
      }
      dbots_url = 'https://bots.discord.pw/api/bots/' + config['botid'] + '/stats'
      async with session.post(dbots_url, data=payload, headers=headers) as resp:
        assert resp.status == 200
        logging.info('dbots: posted!')

  async def on_guild_join( self, guild ):
    logging.info('Joined guild "' + str(guild.name) + '" ID: ' + str(guild.id))
    await listpost()

  async def on_guild_remove( self, guild ):
    logging.info('Left ' + str(guild.name))
    await listpost()

def setup(bot):
  bot.add_cog(Stats(bot))
<commit_msg>[cogs, stats] aaaaaaa<commit_after>import json
import logging
import aiohttp

logging.basicConfig(format='[%(levelname)s] - %(message)s', level=logging.INFO)
config = json.load(open('config.json'))

class Stats:
  def __init__(self, bot):
    self.bot = bot
    payload = json.dumps({
      'server_count': len(self.bot.guilds)
    })
  async def listpost(self):
    session = aiohttp.ClientSession()
    if config['dbotsorg'] == False:
      logging.info('not posting to discordbots.org')
    else:
      dbl_headers = {
        'Authorization': config['dbl'],
        'Content-Type': 'application/json'
      }
      dbl_url = 'https://discordbots.org/api/bots/' + config['botid'] + '/stats'
      async with session.post(dbl_url, data=payload, headers=dbl_headers) as dbl_resp:
        if dbl_resp.status == 200:
          logging.info('dbl: posted!')
        else:
          logging.info(f'dbl: something weird happened: code {dbl_resp.status}.')

    if config['dbotspw'] == False:
      logging.info('not posting to bots.discord.pw')
    else:
      headers = {
        'Authorization': config['dbots'],
        'Content-Type': 'application/json'
      }
      dbots_url = 'https://bots.discord.pw/api/bots/' + config['botid'] + '/stats'
      async with session.post(dbots_url, data=payload, headers=headers) as resp:
        assert resp.status == 200
        logging.info('dbots: posted!')

  async def on_guild_join( self, guild ):
    logging.info('Joined guild "' + str(guild.name) + '" ID: ' + str(guild.id))
    await listpost()

  async def on_guild_remove( self, guild ):
    logging.info('Left ' + str(guild.name))
    await listpost()

def setup(bot):
  bot.add_cog(Stats(bot))
<|endoftext|>


<commit_before>import re
import glob
import numpy as np

from PIL import Image
from tqdm import tqdm
from os.path import basename, dirname, join, splitext, isfile

def _get_file_root_name(file_path):
    file_root_name_expression = re.compile(r'/[^/]*\.', re.IGNORECASE)
    return file_root_name_expression.search(file_path).group(0)[1:-1]

def _normalization_pixel(pixel):
    return pixel/255.0

def _get_rootname(filename):
    return splitext(basename(filename))[0]

def _get_feature_label_images():
    IMAGE_SHAPE = (58, 42)
    ARRAY_FEATURE_PATH = './data/train/features.npy'
    ARRAY_LABEL_PATH = './data/train/labels.npy'
    TRAIN_GLOB_PATH = './data/train/*.tif'

    if not isfile(ARRAY_FEATURE_PATH) or not isfile(ARRAY_LABEL_PATH):
        image_paths = [filename for filename in glob.glob(TRAIN_GLOB_PATH) if not _get_rootname(filename).endswith('_mask')]
        with tqdm(desc='Reading Images from Disk', total=len(image_paths), unit='image') as progress_bar:
            features = np.empty((len(image_paths), IMAGE_SHAPE[0]*IMAGE_SHAPE[1]), dtype=np.float32)
            labels = np.empty((len(image_paths), IMAGE_SHAPE[0]*IMAGE_SHAPE[1]), dtype=np.bool)

            for file_i, filename in enumerate(image_paths):
                progress_bar.update()
                with Image.open(filename) as image:
                    image = image.resize(IMAGE_SHAPE)
                    features[file_i] = [_normalization_pixel(pixel) for pixel in image.getdata()]
                with Image.open(join(dirname(filename), _get_rootname(filename)+'_mask.tif')) as image:
                    image = image.resize(IMAGE_SHAPE)
                    labels[file_i] = image.getdata()

            with open(ARRAY_FEATURE_PATH, 'a') as feature_file:
                np.save(feature_file, features)
            with open(ARRAY_LABEL_PATH, 'a') as label_file:
                np.save(label_file, labels)
    else:
        print 'Loading Image Array...'
        features = np.load(ARRAY_FEATURE_PATH)
        labels = np.load(ARRAY_LABEL_PATH)

    return features, labels

def get_detection_data():
    new_labels = []

    features, labels = _get_feature_label_images()
    with tqdm(desc='Extracting Labels', total=len(labels), unit='image') as progress_bar:
        for label in labels:
            progress_bar.update()
            new_labels.append(True in label)

    return features, new_labels

def _find_coord_mask(mask, height, width):
    rectangel = [[0,0], [0,0]]
    mask_transpose = np.transpose(np.reshape(mask, (height, width))).flatten()

    rectangel[0][0] = next((i for i, x in enumerate(mask_transpose) if x), 0) / height
    rectangel[0][1] = next((i for i, x in enumerate(mask) if x), 0) / width
    rectangel[1][0] = width - (next((i for i, x in enumerate(reversed(mask_transpose)) if x), len(mask)) / height)
    rectangel[1][1] = height-(next((i for i, x in enumerate(reversed(mask)) if x), len(mask)) / width)

    return rectangel

def get_rectangle_masks():
    IMAGE_HEIGHT = 420
    IMAGE_WIDTH = 580
    rectangle_masks = []
    _, labels = _get_feature_label_images()

    with tqdm(desc='Extracting Labels', total=len(labels), unit='image') as progress_bar:
        for label in labels:
            progress_bar.update()
            rectangle_coords = _find_coord_mask(label, IMAGE_HEIGHT, IMAGE_WIDTH)
            if rectangle_coords[1][0] and rectangle_coords[1][1]:
                rectangle_masks.append(rectangle_coords)
    return rectangle_masks

def get_sub_samples(height, width):
    IMAGE_HEIGHT = 420
    IMAGE_WIDTH = 580
    features = []
    labels = []

    features_labels = zip(*_get_feature_label_images())
    with tqdm(desc='Extracting Features and Labels', total=len(features_labels), unit='image') as progress_bar:
        for feature, label in features_labels:
            feature = np.array(feature).reshape((IMAGE_HEIGHT, IMAGE_WIDTH))
            label = np.array(label).reshape((IMAGE_HEIGHT, IMAGE_WIDTH))
            progress_bar.update()

            # Get Inner Rectangles
            for height_i in range(0, IMAGE_HEIGHT-height/2, height/2):
                for width_i in range(0, IMAGE_WIDTH-width/2, width/2):
                    # Make sure the Rectangle size is always height by width
                    if IMAGE_HEIGHT-height_i < height:
                        height_i = IMAGE_HEIGHT-height
                    if IMAGE_WIDTH - width_i < width:
                        width_i = IMAGE_WIDTH - width

                    features.append(feature[height_i:height_i+height, width_i:width_i+width].flatten())
                    # Set the label as the percentage of masked positive pixels to number of pixels in the rectangle
                    unique_counts = dict(zip(*np.unique(
                        label[height_i:height_i+height, width_i:width_i+width],
                        return_counts=True)))
                    labels.append(float(unique_counts.get(True, 0))/(height*width))

    return features, labels
<commit_msg>Use bicubic image resizing<commit_after>import re
import glob
import numpy as np

from PIL import Image
from tqdm import tqdm
from os.path import basename, dirname, join, splitext, isfile

def _get_file_root_name(file_path):
    file_root_name_expression = re.compile(r'/[^/]*\.', re.IGNORECASE)
    return file_root_name_expression.search(file_path).group(0)[1:-1]

def _normalization_pixel(pixel):
    return pixel/255.0

def _get_rootname(filename):
    return splitext(basename(filename))[0]

def _get_feature_label_images():
    IMAGE_RESIZE_SHAPE = (58, 42)
    IMAGE_RESAMPLING = Image.BICUBIC
    ARRAY_FEATURE_PATH = './data/train/features.npy'
    ARRAY_LABEL_PATH = './data/train/labels.npy'
    TRAIN_GLOB_PATH = './data/train/*.tif'

    if not isfile(ARRAY_FEATURE_PATH) or not isfile(ARRAY_LABEL_PATH):
        image_paths = [filename for filename in glob.glob(TRAIN_GLOB_PATH) if not _get_rootname(filename).endswith('_mask')]
        with tqdm(desc='Reading Images from Disk', total=len(image_paths), unit='image') as progress_bar:
            features = np.empty((len(image_paths), IMAGE_RESIZE_SHAPE[0]*IMAGE_RESIZE_SHAPE[1]), dtype=np.float32)
            labels = np.empty((len(image_paths), IMAGE_RESIZE_SHAPE[0]*IMAGE_RESIZE_SHAPE[1]), dtype=np.bool)

            for file_i, filename in enumerate(image_paths):
                progress_bar.update()
                with Image.open(filename) as image:
                    image = image.resize(IMAGE_RESIZE_SHAPE, IMAGE_RESAMPLING)
                    features[file_i] = [_normalization_pixel(pixel) for pixel in image.getdata()]
                with Image.open(join(dirname(filename), _get_rootname(filename)+'_mask.tif')) as image:
                    image = image.resize(IMAGE_RESIZE_SHAPE, IMAGE_RESAMPLING)
                    labels[file_i] = image.getdata()

            with open(ARRAY_FEATURE_PATH, 'a') as feature_file:
                np.save(feature_file, features)
            with open(ARRAY_LABEL_PATH, 'a') as label_file:
                np.save(label_file, labels)
    else:
        print 'Loading Image Array...'
        features = np.load(ARRAY_FEATURE_PATH)
        labels = np.load(ARRAY_LABEL_PATH)

    return features, labels

def get_detection_data():
    new_labels = []

    features, labels = _get_feature_label_images()
    with tqdm(desc='Extracting Labels', total=len(labels), unit='image') as progress_bar:
        for label in labels:
            progress_bar.update()
            new_labels.append(True in label)

    return features, new_labels

def _find_coord_mask(mask, height, width):
    rectangel = [[0,0], [0,0]]
    mask_transpose = np.transpose(np.reshape(mask, (height, width))).flatten()

    rectangel[0][0] = next((i for i, x in enumerate(mask_transpose) if x), 0) / height
    rectangel[0][1] = next((i for i, x in enumerate(mask) if x), 0) / width
    rectangel[1][0] = width - (next((i for i, x in enumerate(reversed(mask_transpose)) if x), len(mask)) / height)
    rectangel[1][1] = height-(next((i for i, x in enumerate(reversed(mask)) if x), len(mask)) / width)

    return rectangel

def get_rectangle_masks():
    IMAGE_HEIGHT = 420
    IMAGE_WIDTH = 580
    rectangle_masks = []
    _, labels = _get_feature_label_images()

    with tqdm(desc='Extracting Labels', total=len(labels), unit='image') as progress_bar:
        for label in labels:
            progress_bar.update()
            rectangle_coords = _find_coord_mask(label, IMAGE_HEIGHT, IMAGE_WIDTH)
            if rectangle_coords[1][0] and rectangle_coords[1][1]:
                rectangle_masks.append(rectangle_coords)
    return rectangle_masks

def get_sub_samples(height, width):
    IMAGE_HEIGHT = 420
    IMAGE_WIDTH = 580
    features = []
    labels = []

    features_labels = zip(*_get_feature_label_images())
    with tqdm(desc='Extracting Features and Labels', total=len(features_labels), unit='image') as progress_bar:
        for feature, label in features_labels:
            feature = np.array(feature).reshape((IMAGE_HEIGHT, IMAGE_WIDTH))
            label = np.array(label).reshape((IMAGE_HEIGHT, IMAGE_WIDTH))
            progress_bar.update()

            # Get Inner Rectangles
            for height_i in range(0, IMAGE_HEIGHT-height/2, height/2):
                for width_i in range(0, IMAGE_WIDTH-width/2, width/2):
                    # Make sure the Rectangle size is always height by width
                    if IMAGE_HEIGHT-height_i < height:
                        height_i = IMAGE_HEIGHT-height
                    if IMAGE_WIDTH - width_i < width:
                        width_i = IMAGE_WIDTH - width

                    features.append(feature[height_i:height_i+height, width_i:width_i+width].flatten())
                    # Set the label as the percentage of masked positive pixels to number of pixels in the rectangle
                    unique_counts = dict(zip(*np.unique(
                        label[height_i:height_i+height, width_i:width_i+width],
                        return_counts=True)))
                    labels.append(float(unique_counts.get(True, 0))/(height*width))

    return features, labels
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Builds a Vega grammar specification from vincent.Scatter()
'''

import vincent
import random

vis = vincent.Line()
vis.tabular_data([random.randint(10, 100) for x in range(0, 101, 1)])

#Generate both the Vega JSON and a data JSON.
path = r'/vega.json'
vis.to_json(path, split_data=True, html=True)

#Make the visualization bigger, lines thicker, and do some point interpolation

vis.update_vis(height=400, width=800)

vis + (4, 'marks', 0, 'properties','enter', 'strokeWidth', 'value')
vis + ({'value': 'basis'}, 'marks', 0, 'properties', 'enter', 'interpolate')

vis.to_json(path, split_data=True, html=True)<commit_msg>Update vincent_line.py<commit_after># -*- coding: utf-8 -*-
'''
Builds a Vega grammar specification from vincent.Line()
'''

import vincent
import random

vis = vincent.Line()
vis.tabular_data([random.randint(10, 100) for x in range(0, 101, 1)])

#Generate both the Vega JSON and a data JSON.
path = r'/vega.json'
vis.to_json(path, split_data=True, html=True)

#Make the visualization bigger, lines thicker, and do some point interpolation

vis.update_vis(height=400, width=800)

vis + (4, 'marks', 0, 'properties','enter', 'strokeWidth', 'value')
vis + ({'value': 'basis'}, 'marks', 0, 'properties', 'enter', 'interpolate')

vis.to_json(path, split_data=True, html=True)
<|endoftext|>