<commit_before>import json
import logging
import os

log = logging.getLogger("config")

def _merge_dicts(self, dict1, dict2):
    if not dict1 or not dict2:
        return

    for attr in dict1:
        if attr in dict2:
            dict1[attr] = dict2[attr]

    return dict1

class Config:
    def __init__(self, path = None):
        self.__path   = path
        self.__config = {
            'servers' : dict(),
            'packages': dict()
        }

        if path:
            return

        path        = os.path.join("~", ".mctl", "config.json")
        self.__path = os.path.expanduser(path)

    def load(self):
        if not os.path.exists(self.__path):
            return False

        try:
            fp = open(self.__path, "r")
        except IOError, msg:
            log.warning("Unable to open: %s: %s", self.__path, msg)
            return False

        self.__config = json.load(fp)
        fp.close()

        if not 'servers' in self.__config:
            self.__config['servers'] = dict()

        if not 'packages' in self.__config:
            self.__config['packages'] = dict()

        return True

    def save(self):
        path = os.path.dirname(self.__path)

        if not os.path.exists(path):
            try:
                os.makedirs(path)
            except os.error, msg:
                log.error("Unable to create path: %s: %s", path, msg)
                return False

        try:
            fp = open(self.__path, "w")
        except IOError, msg:
            log.error("Unable to open: %s: %s", self.__path, msg)
            return False

        json.dump(self.__config, fp, indent = True)
        fp.close()

        return True

    def servers_get(self):
        servers = dict()

        for name in self.__config['servers']:
            server = self.__config['servers'][name]
            server = self._merge_dicts(self.server_new(), server)

            servers[name] = server

        return servers

    def packages_get(self):
        packages = dict()

        for name in self.__config['packages']:
            package = self.__config['packages'][name]
            package = self._merge_dicts(self.package_new(), package)

            packages[name] = package

        return packages

    def server_new(self):
        server = {
            'path'        : None,
            'launch'      : None,
            'timeout'     : 0,
            'log-size'    : 0,
            'log-path'    : None,
            'backup-max'  : 0,
            'backup-path' : None,
            'backup-paths': list(),
            'packages'    : list()
        }

        return server

    def server_get(self, server):
        if not server:
            return self.server_new()

        if not self.__config:
            return self.server_new()

        if not server in self.__config['servers']:
            return self.server_new()

        srv = self.__config['servers'][server]
        srv = self._merge_dicts(self.server_new(), srv)

        return srv

    def server_set(self, name, server):
        if not name or not server:
            return

        if not self.__config:
            return

        srv = self.server_new()
        srv = self._merge_dicts(srv, server)

        self.__config['servers'][name] = srv

    def server_remove(self, server):
        if not name:
            return

        if not self.__config:
            return

        if server in self.__config['servers']:
            del self.__config['servers'][server]

    def package_new(self):
        package = {
            'version': None,
            'path'   : None,
            'type'   : None,
            'url'    : None,
            'updater': None,
            'extract': list()
        }

        return package

    def package_get(self, package):
        if not package:
            return self.package_new()

        if not self.__config:
            return self.package_new()

        if not package in self.__config['packages']:
            return self.package_new()

        pkg = self.__config['packages'][package]
        pkg = self._merge_dicts(self.package_new(), pkg)

        return pkg

    def package_set(self, name, package):
        if not name or not package:
            return

        if not self.__config:
            return

        pkg = self.package_new()
        pkg = self._merge_dicts(pkg, package)

        self.__config['packages'][name] = pkg

    def package_remove(self, package):
        if not package:
            return

        if not self.__config:
            return

        if package in self.__config['packages']:
            del self.__config['packages'][package]
<commit_msg>config: _merge_dicts() is not a method of Config<commit_after>import json
import logging
import os

log = logging.getLogger("config")

def _merge_dicts(dict1, dict2):
    if not dict1 or not dict2:
        return

    for attr in dict1:
        if attr in dict2:
            dict1[attr] = dict2[attr]

    return dict1

class Config:
    def __init__(self, path = None):
        self.__path   = path
        self.__config = {
            'servers' : dict(),
            'packages': dict()
        }

        if path:
            return

        path        = os.path.join("~", ".mctl", "config.json")
        self.__path = os.path.expanduser(path)

    def load(self):
        if not os.path.exists(self.__path):
            return False

        try:
            fp = open(self.__path, "r")
        except IOError, msg:
            log.warning("Unable to open: %s: %s", self.__path, msg)
            return False

        self.__config = json.load(fp)
        fp.close()

        if not 'servers' in self.__config:
            self.__config['servers'] = dict()

        if not 'packages' in self.__config:
            self.__config['packages'] = dict()

        return True

    def save(self):
        path = os.path.dirname(self.__path)

        if not os.path.exists(path):
            try:
                os.makedirs(path)
            except os.error, msg:
                log.error("Unable to create path: %s: %s", path, msg)
                return False

        try:
            fp = open(self.__path, "w")
        except IOError, msg:
            log.error("Unable to open: %s: %s", self.__path, msg)
            return False

        json.dump(self.__config, fp, indent = True)
        fp.close()

        return True

    def servers_get(self):
        servers = dict()

        for name in self.__config['servers']:
            server = self.__config['servers'][name]
            server = _merge_dicts(self.server_new(), server)

            servers[name] = server

        return servers

    def packages_get(self):
        packages = dict()

        for name in self.__config['packages']:
            package = self.__config['packages'][name]
            package = _merge_dicts(self.package_new(), package)

            packages[name] = package

        return packages

    def server_new(self):
        server = {
            'path'        : None,
            'launch'      : None,
            'timeout'     : 0,
            'log-size'    : 0,
            'log-path'    : None,
            'backup-max'  : 0,
            'backup-path' : None,
            'backup-paths': list(),
            'packages'    : list()
        }

        return server

    def server_get(self, server):
        if not server:
            return self.server_new()

        if not self.__config:
            return self.server_new()

        if not server in self.__config['servers']:
            return self.server_new()

        srv = self.__config['servers'][server]
        srv = _merge_dicts(self.server_new(), srv)

        return srv

    def server_set(self, name, server):
        if not name or not server:
            return

        if not self.__config:
            return

        srv = _merge_dicts(self.server_new(), server)
        self.__config['servers'][name] = srv

    def server_remove(self, server):
        if not name:
            return

        if not self.__config:
            return

        if server in self.__config['servers']:
            del self.__config['servers'][server]

    def package_new(self):
        package = {
            'version': None,
            'path'   : None,
            'type'   : None,
            'url'    : None,
            'updater': None,
            'extract': list()
        }

        return package

    def package_get(self, package):
        if not package:
            return self.package_new()

        if not self.__config:
            return self.package_new()

        if not package in self.__config['packages']:
            return self.package_new()

        pkg = self.__config['packages'][package]
        pkg = _merge_dicts(self.package_new(), pkg)

        return pkg

    def package_set(self, name, package):
        if not name or not package:
            return

        if not self.__config:
            return

        pkg = _merge_dicts(self.package_new(), package)
        self.__config['packages'][name] = pkg

    def package_remove(self, package):
        if not package:
            return

        if not self.__config:
            return

        if package in self.__config['packages']:
            del self.__config['packages'][package]
<|endoftext|>


<commit_before>import argparse
import json

import pinder

from twisted.words import service

rooms = {}
irc_users = {}

config_file = open('config.json')
configuration = dict(json.loads(config_file.read()))
users = configuration['users']

parser = argparse.ArgumentParser(description='Campfire to IRC Proxy')
parser.add_argument('-d', '--debug', action='store_true')
args = parser.parse_args()

# connect to Campfire
campfire = pinder.Campfire(configuration['domain'], configuration['api_key'])
# Initialize the Cred authentication system used by the IRC server.
irc_realm = service.InMemoryWordsRealm('LuckyStrike')
<commit_msg>Add ability to put config.json in ~/.luckystrike/<commit_after>import argparse
import json
from os import path

import pinder

from twisted.words import service

rooms = {}
irc_users = {}

if path.exists(path.expanduser('~') + '/.luckystrike/config.json'):
    with open(path.expanduser('~') + '/.luckystrike/config.json') as config_file:
        configuration = dict(json.loads(config_file.read()))
else:
    with open('config.json') as config_file:
        configuration = dict(json.loads(config_file.read()))

users = configuration['users']

parser = argparse.ArgumentParser(description='Campfire to IRC Proxy')
parser.add_argument('-d', '--debug', action='store_true')
args = parser.parse_args()

# connect to Campfire
campfire = pinder.Campfire(configuration['domain'], configuration['api_key'])
# Initialize the Cred authentication system used by the IRC server.
irc_realm = service.InMemoryWordsRealm('LuckyStrike')
<|endoftext|>


<commit_before>"""
This file is part of Ludolph: VCS integration plugin
Copyright (C) 2017 Erigones, s. r. o.

See the LICENSE file for copying permission.
"""
import logging
import hmac
import hashlib

from ludolph_vcs import __version__
from ludolph.plugins.plugin import LudolphPlugin
from ludolph.web import webhook, request, abort

logger = logging.getLogger(__name__)

class Github(LudolphPlugin):
    """
    Ludolph: GitHub integration.
    """
    __version__ = __version__
    _secret_token = None

    def __post_init__(self):
        self._secret_token = self.config.get('secret_token')

    def _room_message(self, msg):
        self.xmpp.msg_send(self.xmpp.room, '\n'.join(msg), mtype='groupchat')

    def _verify_github_signature(self):
        """Verify GitHub signature"""
        if self._secret_token:
            payload = request.body.read()
            signature = request.headers.get('X-Hub-Signature', '')
            mac = hmac.new(self._secret_token, msg=payload, digestmod=hashlib.sha1)

            if not hmac.compare_digest('sha1=' + mac.hexdigest(), signature):
                logger.error('Invalid GitHub secret token')
                abort('403', 'Invalid GitHub secret token')

    def _event_push(self, data):
        data['branch'] = data.get('ref', '').split('/', 2)[-1]

        msg = ['**[{repository[name]}]** The {branch} branch has been updated by '
               '{pusher[name]}: \n\t {repository[html_url]}'.format(**data)]

        for commit in data.get('commits', []):
            commit['message'] = commit.get('message', '').strip()
            msg.append('\t * {id:.8}: {message} ({author[name]})'.format(**commit))

        self._room_message(msg)

        return 'OK'

    def _event_repo_fork(self, data):
        msg = ['**[{repository[name]}]** Yeeey {sender[login]} has forked a repo!'.format(**data)]

        self._room_message(msg)

        return 'OK'

    def _event_issue_handler(self, data):
        msg = ['**[{repository[name]}]** {action} issue: #{issue[number]} {issue[title]} '
               '\n\t {issue[html_url]}'.format(**data)]

        self._room_message(msg)

        return 'OK'

    def _event_issue_comment(self, data):
        msg = ['**[{repository[name]}]** {comment[user][login]} {action} issue comment: '
               '#{issue[number]} {issue[title]} \n\t {comment[body]}'.format(**data)]

        self._room_message(msg)

        return 'OK'

    @webhook('/github-web-hook', methods=('POST',))
    def github_web_hook(self):
        event = request.headers.get('X-GitHub-Event', None)
        logger.info('Incoming GitHub Event: %s', event)
        self._verify_github_signature()

        data = request.json
        logger.debug('GitHub %s payload: %s', event, data)

        if event == 'push':
            return self._event_push(data)
        elif event == 'issues':
            return self._event_issue_handler(data)
        elif event == 'issue_comment':
            return self._event_issue_comment(data)
        elif event == 'fork':
            return self._event_repo_fork(data)
        elif event == 'ping':
            self._room_message(['**[GitHub]** Ping received!'])
            return 'OK'

        logger.error('Unsupported GitHub Web Hook request (%s)', event)
        abort(400, 'Unsupported GitHub Web Hook request (%s)' % event)
<commit_msg>Added reference to assignee (#4)<commit_after>"""
This file is part of Ludolph: VCS integration plugin
Copyright (C) 2017 Erigones, s. r. o.

See the LICENSE file for copying permission.
"""
import logging
import hmac
import hashlib

from ludolph_vcs import __version__
from ludolph.plugins.plugin import LudolphPlugin
from ludolph.web import webhook, request, abort

logger = logging.getLogger(__name__)

class Github(LudolphPlugin):
    """
    Ludolph: GitHub integration.
    """
    __version__ = __version__
    _secret_token = None

    def __post_init__(self):
        self._secret_token = self.config.get('secret_token')

    def _room_message(self, msg):
        self.xmpp.msg_send(self.xmpp.room, '\n'.join(msg), mtype='groupchat')

    def _verify_github_signature(self):
        """Verify GitHub signature"""
        if self._secret_token:
            payload = request.body.read()
            signature = request.headers.get('X-Hub-Signature', '')
            mac = hmac.new(self._secret_token, msg=payload, digestmod=hashlib.sha1)

            if not hmac.compare_digest('sha1=' + mac.hexdigest(), signature):
                logger.error('Invalid GitHub secret token')
                abort('403', 'Invalid GitHub secret token')

    def _event_push(self, data):
        data['branch'] = data.get('ref', '').split('/', 2)[-1]

        msg = ['**[{repository[name]}]** The {branch} branch has been updated by '
               '{pusher[name]}: \n\t {repository[html_url]}'.format(**data)]

        for commit in data.get('commits', []):
            commit['message'] = commit.get('message', '').strip()
            msg.append('\t * {id:.8}: {message} ({author[name]})'.format(**commit))

        self._room_message(msg)

        return 'OK'

    def _event_repo_fork(self, data):
        msg = ['**[{repository[name]}]** Yeeey {sender[login]} has forked a repo!'.format(**data)]

        self._room_message(msg)

        return 'OK'

    def _event_issue_handler(self, data):

        if data.get('assignee', None) and 'login' in data['assignee']:
            data['to_whom'] = ' to {assignee[login]}'.format(**data)
        else:
            data['to_whom'] = ''

        msg = ['**[{repository[name]}]** {action} issue{to_whom}: #{issue[number]} {issue[title]} '
               '\n\t {issue[html_url]}'.format(**data)]

        self._room_message(msg)

        return 'OK'

    def _event_issue_comment(self, data):
        msg = ['**[{repository[name]}]** {comment[user][login]} {action} issue comment: '
               '#{issue[number]} {issue[title]} \n\t {comment[body]}'.format(**data)]

        self._room_message(msg)

        return 'OK'

    @webhook('/github-web-hook', methods=('POST',))
    def github_web_hook(self):
        event = request.headers.get('X-GitHub-Event', None)
        logger.info('Incoming GitHub Event: %s', event)
        self._verify_github_signature()

        data = request.json
        logger.debug('GitHub %s payload: %s', event, data)

        if event == 'push':
            return self._event_push(data)
        elif event == 'issues':
            return self._event_issue_handler(data)
        elif event == 'issue_comment':
            return self._event_issue_comment(data)
        elif event == 'fork':
            return self._event_repo_fork(data)
        elif event == 'ping':
            self._room_message(['**[GitHub]** Ping received!'])
            return 'OK'

        logger.error('Unsupported GitHub Web Hook request (%s)', event)
        abort(400, 'Unsupported GitHub Web Hook request (%s)' % event)
<|endoftext|>


<commit_before>from datetime import timedelta

from django.db import models
from django.contrib.auth.models import User
from django import forms
from django.core.validators import RegexValidator

# Create your models here.

# assumes NetID conforms to personal or shared NetID requirements
netid_validator = RegexValidator(
    regex='^[a-z][0-9a-z]{0,7}$', message='NetID must be valid')

class Platform(models.Model):
    name = models.CharField(max_length=15)
    app_store = models.CharField(max_length=30)

    def __str__(self):
        return self.app_store

class Sponsor(models.Model):
    first_name = models.CharField(max_length=50)
    last_name = models.CharField(max_length=50)
    netid = models.CharField(max_length=16, validators=[netid_validator])
    email = models.EmailField(max_length=256)
    title = models.CharField(max_length=50)
    department = models.CharField(max_length=30)
    unit = models.CharField(max_length=30)

    def __str__(self):
        return self.first_name + ' ' + self.last_name

class Manager(models.Model):
    first_name = models.CharField(max_length=50)
    last_name = models.CharField(max_length=50)
    netid = models.CharField(
        max_length=16,
        validators=[netid_validator],
        error_messages={'required': 'Please enter a valid NetID'}
    )
    email = models.EmailField(max_length=256)

    def __str__(self):
        return self.first_name + ' ' + self.last_name

AGREEMENT_CHOICES = [
    (True, 'Agreed'),
    (False, 'Denied'),
]

class Agreement(models.Model):
    app = models.ForeignKey('App', on_delete=models.CASCADE)
    agree = models.BooleanField(choices=AGREEMENT_CHOICES)
    agree_time = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return self.app.name

class App(models.Model):
    name = models.CharField(max_length=50)
    primary_language = models.CharField(max_length=20)
    platform = models.ManyToManyField(Platform)
    app_manager = models.ForeignKey(Manager, on_delete=models.CASCADE)
    manager_contact = models.CharField(default=str(Manager.email),
                                       editable=False)
    app_sponsor = models.ForeignKey(Sponsor, on_delete=models.CASCADE)
    sponsor_contact = models.CharField(default=str(Sponsor.email),
                                       editable=False)
    requestor = models.ForeignKey(User, on_delete=models.CASCADE)
    request_date = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return self.name

    def sponsor_contact(self):
        return self.app_sponsor.email

    def manager_contact(self):
        return self.app_manager.email

    def app_platform(self):
        return ", ".join([p.app_store for p in self.platform.all()])

    def status(self):
        if not Agreement.objects.filter(app__name=self.name).exists():
            return "Pending"
        else:
            dates = Agreement.objects.filter(app__name=self.name)
            agreements = []
            for date in dates:
                agreements.append(date)

            # method to sort agreements by agree time
            def time(agreement):
                return agreement.agree_time

            agreements.sort(key=time)
            # time adjusted back 7 hours due to disparity (temp fix?)
            time = (agreements[-1].agree_time - timedelta(hours=7)).strftime('%b %d, %Y, %I:%M %p')
            if agreements[-1].agree:
                return "Agreed on " + time
            else:
                return "Denied on " + time

    sponsor_contact = property(sponsor_contact)
    manager_contact = property(manager_contact)
    platforms = property(app_platform)

class SponsorForm(forms.ModelForm):
    class Meta:
        model = Sponsor
        fields = "__all__"
        labels = {
            "netid": "UW NetID"
        }

class ManagerForm(forms.ModelForm):
    class Meta:
        model = Manager
        fields = "__all__"
        labels = {
            "netid": "UW NetID"
        }

class AppForm(forms.ModelForm):
    class Meta:
        model = App
        fields = ["name", "primary_language", "platform"]
        labels = {
            "name": "Application Name",
            "platform": "Distribution Platform"
        }

    def __init__(self, *args, **kwargs):
        super(AppForm, self).__init__(*args, **kwargs)
        self.fields["primary_language"] \
            .widget.attrs["placeholder"] = "e.g. English"
        self.fields["platform"].widget = forms.CheckboxSelectMultiple()
        self.fields["platform"].queryset = Platform.objects.all()
<commit_msg>fix style<commit_after>from datetime import timedelta

from django.db import models
from django.contrib.auth.models import User
from django import forms
from django.core.validators import RegexValidator

# Create your models here.

# assumes NetID conforms to personal or shared NetID requirements
netid_validator = RegexValidator(
    regex='^[a-z][0-9a-z]{0,7}$', message='NetID must be valid')

class Platform(models.Model):
    name = models.CharField(max_length=15)
    app_store = models.CharField(max_length=30)

    def __str__(self):
        return self.app_store

class Sponsor(models.Model):
    first_name = models.CharField(max_length=50)
    last_name = models.CharField(max_length=50)
    netid = models.CharField(max_length=16, validators=[netid_validator])
    email = models.EmailField(max_length=256)
    title = models.CharField(max_length=50)
    department = models.CharField(max_length=30)
    unit = models.CharField(max_length=30)

    def __str__(self):
        return self.first_name + ' ' + self.last_name

class Manager(models.Model):
    first_name = models.CharField(max_length=50)
    last_name = models.CharField(max_length=50)
    netid = models.CharField(
        max_length=16,
        validators=[netid_validator],
        error_messages={'required': 'Please enter a valid NetID'}
    )
    email = models.EmailField(max_length=256)

    def __str__(self):
        return self.first_name + ' ' + self.last_name

AGREEMENT_CHOICES = [
    (True, 'Agreed'),
    (False, 'Denied'),
]

class Agreement(models.Model):
    app = models.ForeignKey('App', on_delete=models.CASCADE)
    agree = models.BooleanField(choices=AGREEMENT_CHOICES)
    agree_time = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return self.app.name

class App(models.Model):
    name = models.CharField(max_length=50)
    primary_language = models.CharField(max_length=20)
    platform = models.ManyToManyField(Platform)
    app_manager = models.ForeignKey(Manager, on_delete=models.CASCADE)
    manager_contact = models.CharField(default=str(Manager.email),
                                       editable=False)
    app_sponsor = models.ForeignKey(Sponsor, on_delete=models.CASCADE)
    sponsor_contact = models.CharField(default=str(Sponsor.email),
                                       editable=False)
    requestor = models.ForeignKey(User, on_delete=models.CASCADE)
    request_date = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return self.name

    def sponsor_contact(self):
        return self.app_sponsor.email

    def manager_contact(self):
        return self.app_manager.email

    def app_platform(self):
        return ", ".join([p.app_store for p in self.platform.all()])

    def status(self):
        if not Agreement.objects.filter(app__name=self.name).exists():
            return "Pending"
        else:
            dates = Agreement.objects.filter(app__name=self.name)
            agreements = []
            for date in dates:
                agreements.append(date)

            # method to sort agreements by agree time
            def time(agreement):
                return agreement.agree_time

            agreements.sort(key=time)
            # time adjusted back 7 hours due to disparity (temp fix?)
            time = (agreements[-1].agree_time - timedelta(hours=7))\
                .strftime('%b %d, %Y, %I:%M %p')
            if agreements[-1].agree:
                return "Agreed on " + time
            else:
                return "Denied on " + time

    sponsor_contact = property(sponsor_contact)
    manager_contact = property(manager_contact)
    platforms = property(app_platform)

class SponsorForm(forms.ModelForm):
    class Meta:
        model = Sponsor
        fields = "__all__"
        labels = {
            "netid": "UW NetID"
        }

class ManagerForm(forms.ModelForm):
    class Meta:
        model = Manager
        fields = "__all__"
        labels = {
            "netid": "UW NetID"
        }

class AppForm(forms.ModelForm):
    class Meta:
        model = App
        fields = ["name", "primary_language", "platform"]
        labels = {
            "name": "Application Name",
            "platform": "Distribution Platform"
        }

    def __init__(self, *args, **kwargs):
        super(AppForm, self).__init__(*args, **kwargs)
        self.fields["primary_language"] \
            .widget.attrs["placeholder"] = "e.g. English"
        self.fields["platform"].widget = forms.CheckboxSelectMultiple()
        self.fields["platform"].queryset = Platform.objects.all()
<|endoftext|>


<commit_before># Copyright (c) 2015 Gouthaman Balaraman
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

"""
Support for SQLAlchmey. Provides SQLAlchemyTarget for storing in databases
supported by SQLAlchemy. The user would be responsible for installing the
required database driver to connect using SQLAlchemy.

Minimal example of a job to copy data to database using SQLAlchemy is as shown
below::

    from sqlalchemy import String
    import luigi
    from luigi.contrib import sqla

    class SQLATask(sqla.CopyToTable):
        # columns defines the table schema, with each element corresponding
        # to a column in the format (args, kwargs) which will be sent to
        # the sqlalchemy.Column(*args, **kwargs)
        columns = [
            (["item", String(64)], {"primary_key": True}),
            (["property", String(64)], {})
        ]
        connection_string = "sqlite://"  # in memory SQLite database
        table = "item_property"  # name of the table to store data

        def rows(self):
            for row in [("item1" "property1"), ("item2", "property2")]:
                yield row

    if __name__ == '__main__':
        task = SQLATask()
        luigi.build([task], local_scheduler=True)

If the target table where the data needs to be copied already exists, then
the column schema definition can be skipped and instead the reflect flag
can be set as True. Here is a modified version of the above example::

    from sqlalchemy import String
    import luigi
    from luigi.contrib import sqla

    class SQLATask(sqla.CopyToTable):
        # If database table is already created, then the schema can be loaded
        # by setting the reflect flag to True
        reflect = True
        connection_string = "sqlite://"  # in memory SQLite database
        table = "item_property"  # name of the table to store data

        def rows(self):
            for row in [("item1" "property1"), ("item2", "property2")]:
                yield row

    if __name__ == '__main__':
        task = SQLATask()
        luigi.build([task], local_scheduler=True)

In the above examples, the data that needs to be copied was directly provided by
overriding the rows method. Alternately, if the data comes from another task, the
modified example would look as shown below::

    from sqlalchemy import String
    import luigi
    from luigi.contrib import sqla
    from luigi.mock import MockFile

    class BaseTask(luigi.Task):

        def output(self):
            return MockFile("BaseTask")

        def run(self):
            out = self.output().open("w")
            TASK_LIST = ["item%d\tproperty%d\n" % (i, i) for i in range(10)]
            for task in TASK_LIST:
                out.write(task)
            out.close()

    class SQLATask(sqla.CopyToTable):
        # columns defines the table schema, with each element corresponding
        # to a column in the format (args, kwargs) which will be sent to
        # the sqlalchemy.Column(*args, **kwargs)
        columns = [
            (["item", String(64)], {"primary_key": True}),
            (["property", String(64)], {})
        ]
        connection_string = "sqlite://"  # in memory SQLite database
        table = "item_property"  # name of the table to store data

        def requires(self):
            return BaseTask()

    if __name__ == '__main__':
        task1, task2 = SQLATask(), BaseTask()
        luigi.build([task1, task2], local_scheduler=True)

In the above example, the output from `BaseTask` is copied into the
database. Here we did not have to implement the `rows` method because
by default `rows` implementation assumes every line is a row with
column values separated by a tab. One can define `column_separator`
option for the task if the values are say comma separated instead of
tab separated.

The other option to `sqla.CopyToTable` that can be of help with performance aspect is the
`chunk_size`. The default is 5000. This is the number of rows that will be inserted in
a transaction at a time. Depending on the size of the inserts, this value can be tuned
for performance.

Author: Gouthaman Balaraman
Date: 01/02/2015
"""

import abc
import logging
import luigi
import datetime
import itertools
import sqlalchemy

logger = logging.getLogger('luigi-interface')

class SQLAlchemyTarget(luigi.Target):
    """Database target using SQLAlchemy. This will rarely have to be
    directly instantiated by the user. Typical usage would be to override
    `luigi.contrib.sqla.CopyToTable` class to create a task to write to
    the database."""
    marker_table = None

    def __init__(self, connection_string, target_table, update_id, echo=False):
        """ Constructor for the SQLAlchemyTarget
        :param connection_string: (str) SQLAlchemy connection string
        :param target_table: (str) The table name for the data
        :param update_id: (str) An identifier for this data set
        :param echo: (bool) Flag to setup SQLAlchemy logging
        :return:
        """
        self.target_table = target_table
        self.update_id = update_id
        self.engine = sqlalchemy.create_engine(connection_string, echo=echo)
        self.marker_table_bound = None

    def touch(self):
        """Mark this update as complete.
        """
        if self.marker_table_bound is None:
            self.create_marker_table()

        table = self.marker_table_bound
        with self.engine.begin() as conn:
            id_exists = self.exists()
            if not id_exists:
                ins = table.insert().values(update_id=self.update_id, target_table=self.target_table)
            else:
                ins = table.update().values(update_id=self.update_id, target_table=self.target_table,
                                            inserted=datetime.datetime.now())
            conn.execute(ins)
        assert self.exists()

    def exists(self):
        row = None
        if self.marker_table_bound is None:
            self.create_marker_table()
        with self.engine.begin() as conn:
            table = self.marker_table_bound
            s = sqlalchemy.select([table]).where(table.c.update_id == self.update_id).limit(1)
            row = conn.execute(s).fetchone()
        return row is not None

    def create_marker_table(self):
        """Create marker table if it doesn't exist.

        Using a separate connection since the transaction might have to be reset"""
        if self.marker_table is None:
            self.marker_table = luigi.configuration.get_config().get('sqlalchemy', 'marker-table', 'table_updates')

        with self.engine.begin() as con:
            metadata = sqlalchemy.MetaData()
            if not con.dialect.has_table(con, self.marker_table):
                self.marker_table_bound = sqlalchemy.Table(
                    self.marker_table, metadata,
                    sqlalchemy.Column("update_id", sqlalchemy.String(128), primary_key=True),
                    sqlalchemy.Column("target_table", sqlalchemy.String(128)),
                    sqlalchemy.Column("inserted", sqlalchemy.DateTime, default=datetime.datetime.now()))
                metadata.create_all(self.engine)
            else:
                metadata.reflect(bind=self.engine)
                self.marker_table_bound = metadata.tables[self.marker_table]

    def open(self, mode):
        raise NotImplementedError("Cannot open() SQLAlchemyTarget")

class CopyToTable(luigi.Task):
    """
    An abstract task for inserting a data set into SQLAlchemy RDBMS

    Usage:
    Subclass and override the required `connection_string`, `table` and `columns` attributes.
    """
    echo = False

    @abc.abstractmethod
    def connection_string(self):
        return None

    @abc.abstractproperty
    def table(self):
        return None

    # specify the columns that define the schema. The format for the columns is a list
    # of tuples. For example :
    # columns = [
    #            (["id", sqlalchemy.Integer], dict(primary_key=True)),
    #            (["name", sqlalchemy.String(64)], {}),
    #            (["value", sqlalchemy.String(64)], {})
    #        ]
    # The tuple (args_list, kwargs_dict) here is the args and kwargs
    # that need to be passed to sqlalchemy.Column(*args, **kwargs).
    # If the tables have already been setup by another process, then you can
    # completely ignore the columns. Instead set the reflect value to True below
    columns = []

    # options
    column_separator = "\t"  # how columns are separated in the file copied into postgres
    chunk_size = 5000   # default chunk size for insert
    reflect = False  # Set this to true only if the table has already been created by alternate means

    def create_table(self, engine):
        """ Override to provide code for creating the target table.

        By default it will be created using types specified in columns. If the table
        exists, then it binds to the existing table.

        If overridden, use the provided connection object for setting up the table in order to
        create the table and insert data using the same transaction.
        """
        def construct_sqla_columns(columns):
            retval = [sqlalchemy.Column(*c[0], **c[1]) for c in columns]
            return retval

        needs_setup = (len(self.columns) == 0) or (False in [len(c) == 2 for c in self.columns]) if not self.reflect else False
        if needs_setup:
            # only names of columns specified, no types
            raise NotImplementedError("create_table() not implemented for %r and columns types not specified" % self.table)
        else:
            # if columns is specified as (name, type) tuples
            with engine.begin() as con:
                metadata = sqlalchemy.MetaData()
                try:
                    if not con.dialect.has_table(con, self.table):
                        sqla_columns = construct_sqla_columns(self.columns)
                        self.table_bound = sqlalchemy.Table(self.table, metadata, *sqla_columns)
                        metadata.create_all(engine)
                    else:
                        metadata.reflect(bind=engine)
                        self.table_bound = metadata.tables[self.table]
                except Exception as e:
                    logger.exception(self.table + str(e))

    def update_id(self):
        """This update id will be a unique identifier for this insert on this table."""
        return self.task_id

    def output(self):
        return SQLAlchemyTarget(
            connection_string=self.connection_string,
            target_table=self.table,
            update_id=self.update_id(),
            echo=self.echo
            )

    def rows(self):
        """Return/yield tuples or lists corresponding to each row to be inserted. This method
         can be overridden for custom file types or formats."""
        with self.input().open('r') as fobj:
            for line in fobj:
                yield line.strip("\n").split(self.column_separator)

    def run(self):
        logger.info("Running task copy to table for update id %s for table %s" % (self.update_id(), self.table))
        output = self.output()
        self.create_table(output.engine)
        with output.engine.begin() as conn:
            rows = iter(self.rows())
            ins_rows = [dict(zip((c.key for c in self.table_bound.c), row))
                        for row in itertools.islice(rows, self.chunk_size)]
            while ins_rows:
                ins = self.table_bound.insert()
                conn.execute(ins, ins_rows)
                ins_rows = [dict(zip((c.key for c in self.table_bound.c), row))
                            for row in itertools.islice(rows, self.chunk_size)]
                logger.info("Finished inserting %d rows into SQLAlchemy target" % len(ins_rows))
        output.touch()
        logger.info("Finished inserting rows into SQLAlchemy target")

<commit_msg>removed some empty lines to see if sphinx formatting works<commit_after># Copyright (c) 2015 Gouthaman Balaraman
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

"""
Support for SQLAlchmey. Provides SQLAlchemyTarget for storing in databases
supported by SQLAlchemy. The user would be responsible for installing the
required database driver to connect using SQLAlchemy.

Minimal example of a job to copy data to database using SQLAlchemy is as shown
below::

    from sqlalchemy import String
    import luigi
    from luigi.contrib import sqla

    class SQLATask(sqla.CopyToTable):
        # columns defines the table schema, with each element corresponding
        # to a column in the format (args, kwargs) which will be sent to
        # the sqlalchemy.Column(*args, **kwargs)
        columns = [
            (["item", String(64)], {"primary_key": True}),
            (["property", String(64)], {})
        ]
        connection_string = "sqlite://"  # in memory SQLite database
        table = "item_property"  # name of the table to store data

        def rows(self):
            for row in [("item1" "property1"), ("item2", "property2")]:
                yield row

    if __name__ == '__main__':
        task = SQLATask()
        luigi.build([task], local_scheduler=True)

If the target table where the data needs to be copied already exists, then
the column schema definition can be skipped and instead the reflect flag
can be set as True. Here is a modified version of the above example::

    from sqlalchemy import String
    import luigi
    from luigi.contrib import sqla

    class SQLATask(sqla.CopyToTable):
        # If database table is already created, then the schema can be loaded
        # by setting the reflect flag to True
        reflect = True
        connection_string = "sqlite://"  # in memory SQLite database
        table = "item_property"  # name of the table to store data

        def rows(self):
            for row in [("item1" "property1"), ("item2", "property2")]:
                yield row

    if __name__ == '__main__':
        task = SQLATask()
        luigi.build([task], local_scheduler=True)

In the above examples, the data that needs to be copied was directly provided by
overriding the rows method. Alternately, if the data comes from another task, the
modified example would look as shown below::

    from sqlalchemy import String
    import luigi
    from luigi.contrib import sqla
    from luigi.mock import MockFile

    class BaseTask(luigi.Task):
        def output(self):
            return MockFile("BaseTask")

        def run(self):
            out = self.output().open("w")
            TASK_LIST = ["item%d\tproperty%d\n" % (i, i) for i in range(10)]
            for task in TASK_LIST:
                out.write(task)
            out.close()

    class SQLATask(sqla.CopyToTable):
        # columns defines the table schema, with each element corresponding
        # to a column in the format (args, kwargs) which will be sent to
        # the sqlalchemy.Column(*args, **kwargs)
        columns = [
            (["item", String(64)], {"primary_key": True}),
            (["property", String(64)], {})
        ]
        connection_string = "sqlite://"  # in memory SQLite database
        table = "item_property"  # name of the table to store data

        def requires(self):
            return BaseTask()

    if __name__ == '__main__':
        task1, task2 = SQLATask(), BaseTask()
        luigi.build([task1, task2], local_scheduler=True)

In the above example, the output from `BaseTask` is copied into the
database. Here we did not have to implement the `rows` method because
by default `rows` implementation assumes every line is a row with
column values separated by a tab. One can define `column_separator`
option for the task if the values are say comma separated instead of
tab separated.

The other option to `sqla.CopyToTable` that can be of help with performance aspect is the
`chunk_size`. The default is 5000. This is the number of rows that will be inserted in
a transaction at a time. Depending on the size of the inserts, this value can be tuned
for performance.

Author: Gouthaman Balaraman
Date: 01/02/2015
"""

import abc
import logging
import luigi
import datetime
import itertools
import sqlalchemy

logger = logging.getLogger('luigi-interface')

class SQLAlchemyTarget(luigi.Target):
    """Database target using SQLAlchemy. This will rarely have to be
    directly instantiated by the user. Typical usage would be to override
    `luigi.contrib.sqla.CopyToTable` class to create a task to write to
    the database."""
    marker_table = None

    def __init__(self, connection_string, target_table, update_id, echo=False):
        """ Constructor for the SQLAlchemyTarget
        :param connection_string: (str) SQLAlchemy connection string
        :param target_table: (str) The table name for the data
        :param update_id: (str) An identifier for this data set
        :param echo: (bool) Flag to setup SQLAlchemy logging
        :return:
        """
        self.target_table = target_table
        self.update_id = update_id
        self.engine = sqlalchemy.create_engine(connection_string, echo=echo)
        self.marker_table_bound = None

    def touch(self):
        """Mark this update as complete.
        """
        if self.marker_table_bound is None:
            self.create_marker_table()

        table = self.marker_table_bound
        with self.engine.begin() as conn:
            id_exists = self.exists()
            if not id_exists:
                ins = table.insert().values(update_id=self.update_id, target_table=self.target_table)
            else:
                ins = table.update().values(update_id=self.update_id, target_table=self.target_table,
                                            inserted=datetime.datetime.now())
            conn.execute(ins)
        assert self.exists()

    def exists(self):
        row = None
        if self.marker_table_bound is None:
            self.create_marker_table()
        with self.engine.begin() as conn:
            table = self.marker_table_bound
            s = sqlalchemy.select([table]).where(table.c.update_id == self.update_id).limit(1)
            row = conn.execute(s).fetchone()
        return row is not None

    def create_marker_table(self):
        """Create marker table if it doesn't exist.

        Using a separate connection since the transaction might have to be reset"""
        if self.marker_table is None:
            self.marker_table = luigi.configuration.get_config().get('sqlalchemy', 'marker-table', 'table_updates')

        with self.engine.begin() as con:
            metadata = sqlalchemy.MetaData()
            if not con.dialect.has_table(con, self.marker_table):
                self.marker_table_bound = sqlalchemy.Table(
                    self.marker_table, metadata,
                    sqlalchemy.Column("update_id", sqlalchemy.String(128), primary_key=True),
                    sqlalchemy.Column("target_table", sqlalchemy.String(128)),
                    sqlalchemy.Column("inserted", sqlalchemy.DateTime, default=datetime.datetime.now()))
                metadata.create_all(self.engine)
            else:
                metadata.reflect(bind=self.engine)
                self.marker_table_bound = metadata.tables[self.marker_table]

    def open(self, mode):
        raise NotImplementedError("Cannot open() SQLAlchemyTarget")

class CopyToTable(luigi.Task):
    """
    An abstract task for inserting a data set into SQLAlchemy RDBMS

    Usage:
    Subclass and override the required `connection_string`, `table` and `columns` attributes.
    """
    echo = False

    @abc.abstractmethod
    def connection_string(self):
        return None

    @abc.abstractproperty
    def table(self):
        return None

    # specify the columns that define the schema. The format for the columns is a list
    # of tuples. For example :
    # columns = [
    #            (["id", sqlalchemy.Integer], dict(primary_key=True)),
    #            (["name", sqlalchemy.String(64)], {}),
    #            (["value", sqlalchemy.String(64)], {})
    #        ]
    # The tuple (args_list, kwargs_dict) here is the args and kwargs
    # that need to be passed to sqlalchemy.Column(*args, **kwargs).
    # If the tables have already been setup by another process, then you can
    # completely ignore the columns. Instead set the reflect value to True below
    columns = []

    # options
    column_separator = "\t"  # how columns are separated in the file copied into postgres
    chunk_size = 5000   # default chunk size for insert
    reflect = False  # Set this to true only if the table has already been created by alternate means

    def create_table(self, engine):
        """ Override to provide code for creating the target table.

        By default it will be created using types specified in columns. If the table
        exists, then it binds to the existing table.

        If overridden, use the provided connection object for setting up the table in order to
        create the table and insert data using the same transaction.
        """
        def construct_sqla_columns(columns):
            retval = [sqlalchemy.Column(*c[0], **c[1]) for c in columns]
            return retval

        needs_setup = (len(self.columns) == 0) or (False in [len(c) == 2 for c in self.columns]) if not self.reflect else False
        if needs_setup:
            # only names of columns specified, no types
            raise NotImplementedError("create_table() not implemented for %r and columns types not specified" % self.table)
        else:
            # if columns is specified as (name, type) tuples
            with engine.begin() as con:
                metadata = sqlalchemy.MetaData()
                try:
                    if not con.dialect.has_table(con, self.table):
                        sqla_columns = construct_sqla_columns(self.columns)
                        self.table_bound = sqlalchemy.Table(self.table, metadata, *sqla_columns)
                        metadata.create_all(engine)
                    else:
                        metadata.reflect(bind=engine)
                        self.table_bound = metadata.tables[self.table]
                except Exception as e:
                    logger.exception(self.table + str(e))

    def update_id(self):
        """This update id will be a unique identifier for this insert on this table."""
        return self.task_id

    def output(self):
        return SQLAlchemyTarget(
            connection_string=self.connection_string,
            target_table=self.table,
            update_id=self.update_id(),
            echo=self.echo
            )

    def rows(self):
        """Return/yield tuples or lists corresponding to each row to be inserted. This method
         can be overridden for custom file types or formats."""
        with self.input().open('r') as fobj:
            for line in fobj:
                yield line.strip("\n").split(self.column_separator)

    def run(self):
        logger.info("Running task copy to table for update id %s for table %s" % (self.update_id(), self.table))
        output = self.output()
        self.create_table(output.engine)
        with output.engine.begin() as conn:
            rows = iter(self.rows())
            ins_rows = [dict(zip((c.key for c in self.table_bound.c), row))
                        for row in itertools.islice(rows, self.chunk_size)]
            while ins_rows:
                ins = self.table_bound.insert()
                conn.execute(ins, ins_rows)
                ins_rows = [dict(zip((c.key for c in self.table_bound.c), row))
                            for row in itertools.islice(rows, self.chunk_size)]
                logger.info("Finished inserting %d rows into SQLAlchemy target" % len(ins_rows))
        output.touch()
        logger.info("Finished inserting rows into SQLAlchemy target")

<|endoftext|>


<commit_before>#!venv/bin/python3
import logging

from FlaskMedia import db
from FlaskMedia.models import Movie, Series, Episode

logger = logging.getLogger(__name__)
logging.basicConfig(filename='media_write.log',level=logging.DEBUG,
                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

def write_movie(title, plot, year, released, last_updated_utc, rated, runtime,
                IMDB_id, IMDB_rating, IMDB_rating_count, metascore, awards,
                language, country, poster):

    movie = Movie(title=title, plot=plot, year=year, released=released,
                  last_updated_utc=last_updated_utc, rated=rated,
                  runtime=runtime, IMDB_id=IMDB_id, IMDB_rating=IMDB_rating,
                  IMDB_rating_count=IMDB_rating_count, metascore=metascore,
                  awards=awards, language=language, country=country,
                  poster=poster)

    # attempted to use try/except to catch errors but did not work on SQL errors
    if not Movie.query.filter_by(title=title, year=year).first():
        db.session.add(movie)
        logger.info("Wrote Movie: %s" % movie)
        db.session.commit()
    else:
        logger.warning("Movie %s already in database." % movie)

def write_ep_short(season, ep, title, plot, first_aired, last_updated_utc, TVDB_id, series_id):
    # make
    new_ep=Episode(season=season, episode=ep, title=title, plot=plot,
                   first_aired=first_aired, last_updated_utc=last_updated_utc,
                   TVDB_id=TVDB_id, series_id=series_id)
    # attempted to use try/except to catch errors but did not work on SQL errors
    if not Episode.query.filter_by(season=season, episode=episode, series_id=series_id).first():
        db.session.add(new_ep)
        logger.info("Wrote Episode: %s" % new_ep)
        db.session.commit()
    else:
        logger.warning("Episode %s already in database." % new_ep)

def write_ep_long(season, ep, title, plot, first_aired, last_updated_utc,
                  TVDB_id, IMDB_id, abs_num, TVDB_rating, TVDB_rating_count,
                  TVDB_last_update, series_id):
    # make
    new_ep=Episode(season=season, episode=ep, title=title, plot=plot,
                   first_aired=first_aired, last_updated_utc=last_updated_utc,
                   TVDB_id=TVDB_id,IMDB_id=IMDB_id, abs_num=abs_num,
                   TVDB_rating=TVDB_rating, TVDB_rating_count=TVDB_rating_count,
                   TVDB_last_update=TVDB_last_update, series_id=series_id)
    # attempted to use try/except to catch errors but did not work on SQL errors
    if not Episode.query.filter_by(season=season, episode=ep, series_id=series_id).first():
        db.session.add(new_ep)
        logger.info("Wrote Episode: %s" % new_ep)
        db.session.commit()
    else:
        logger.warning("Episode %s already in database." % new_ep)

def write_series(title, plot, rated, IMDB_id, TVDB_rating, TVDB_rating_count,
                 TVDB_last_update, first_aired, last_updated_utc, poster,
                 network):

    new_series=Series(title=title, plot=plot, rated=rated, IMDB_id=IMDB_id,
                      TVDB_rating=TVDB_rating, TVDB_rating_count=TVDB_rating_count,
                      TVDB_last_update=TVDB_last_update, first_aired=first_aired,
                      last_updated_utc=last_updated_utc, poster=poster, network=network)
    # attempted to use try/except to catch errors but did not work on SQL errors
    if not Series.query.filter_by(title=title).first():
        db.session.add(new_series)
        logger.info("Wrote Series: %s" % title)
        db.session.commit()
    else:
        logger.warning("Series %s already in database." % new_series)
<commit_msg>removing sep log file<commit_after>#!venv/bin/python3
import logging

from FlaskMedia import db
from FlaskMedia.models import Movie, Series, Episode

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

def write_movie(title, plot, year, released, last_updated_utc, rated, runtime,
                IMDB_id, IMDB_rating, IMDB_rating_count, metascore, awards,
                language, country, poster):

    movie = Movie(title=title, plot=plot, year=year, released=released,
                  last_updated_utc=last_updated_utc, rated=rated,
                  runtime=runtime, IMDB_id=IMDB_id, IMDB_rating=IMDB_rating,
                  IMDB_rating_count=IMDB_rating_count, metascore=metascore,
                  awards=awards, language=language, country=country,
                  poster=poster)

    # attempted to use try/except to catch errors but did not work on SQL errors
    if not Movie.query.filter_by(title=title, year=year).first():
        db.session.add(movie)
        logger.info("Wrote Movie: %s" % movie)
        db.session.commit()
    else:
        logger.warning("Movie %s already in database." % movie)

def write_ep_short(season, ep, title, plot, first_aired, last_updated_utc, TVDB_id, series_id):
    # make
    new_ep=Episode(season=season, episode=ep, title=title, plot=plot,
                   first_aired=first_aired, last_updated_utc=last_updated_utc,
                   TVDB_id=TVDB_id, series_id=series_id)
    # attempted to use try/except to catch errors but did not work on SQL errors
    if not Episode.query.filter_by(season=season, episode=episode, series_id=series_id).first():
        db.session.add(new_ep)
        logger.info("Wrote Episode: %s" % new_ep)
        db.session.commit()
    else:
        logger.warning("Episode %s already in database." % new_ep)

def write_ep_long(season, ep, title, plot, first_aired, last_updated_utc,
                  TVDB_id, IMDB_id, abs_num, TVDB_rating, TVDB_rating_count,
                  TVDB_last_update, series_id):
    # make
    new_ep=Episode(season=season, episode=ep, title=title, plot=plot,
                   first_aired=first_aired, last_updated_utc=last_updated_utc,
                   TVDB_id=TVDB_id,IMDB_id=IMDB_id, abs_num=abs_num,
                   TVDB_rating=TVDB_rating, TVDB_rating_count=TVDB_rating_count,
                   TVDB_last_update=TVDB_last_update, series_id=series_id)
    # attempted to use try/except to catch errors but did not work on SQL errors
    if not Episode.query.filter_by(season=season, episode=ep, series_id=series_id).first():
        db.session.add(new_ep)
        logger.info("Wrote Episode: %s" % new_ep)
        db.session.commit()
    else:
        logger.warning("Episode %s already in database." % new_ep)

def write_series(title, plot, rated, IMDB_id, TVDB_rating, TVDB_rating_count,
                 TVDB_last_update, first_aired, last_updated_utc, poster,
                 network):

    new_series=Series(title=title, plot=plot, rated=rated, IMDB_id=IMDB_id,
                      TVDB_rating=TVDB_rating, TVDB_rating_count=TVDB_rating_count,
                      TVDB_last_update=TVDB_last_update, first_aired=first_aired,
                      last_updated_utc=last_updated_utc, poster=poster, network=network)
    # attempted to use try/except to catch errors but did not work on SQL errors
    if not Series.query.filter_by(title=title).first():
        db.session.add(new_series)
        logger.info("Wrote Series: %s" % title)
        db.session.commit()
    else:
        logger.warning("Series %s already in database." % new_series)
<|endoftext|>


<commit_before>SUCCESS = "SUCCESS"
FAILURE = "FAILURE"
START = "START"<commit_msg>More event types, and label formatting<commit_after>SUCCESS = "SUCCESS"
FAILURE = "FAILURE"
MISSING = "MISSING"
START = "START"
PROCESSING_TIME = "PROCESSING_TIME"

def event_label(event):
    """Format event constants for Slack

    e.g. from `MISSING` to `*Missing*`
    """
    return "*{}*".format(event.lower().title())<|endoftext|>


<commit_before>from os import environ

# Memcache addon environment variables.
# See: https://addons.heroku.com/memcache
MEMCACHE_ENV_VARS = (
    'MEMCACHE_PASSWORD',
    'MEMCACHE_SERVERS',
    'MEMCACHE_USERNAME',
)

# MemCachier addon environment variables.
# See: https://addons.heroku.com/memcachier
MEMCACHIER_ENV_VARS = (
    'MEMCACHIER_PASSWORD',
    'MEMCACHIER_SERVERS',
    'MEMCACHIER_USERNAME',
)

def memcacheify(timeout=500):
    """Return a fully configured Django ``CACHES`` setting. We do this by
    analyzing all environment variables on Heorku, scanning for an available
    memcache addon, and then building the settings dict properly.

    If no memcache servers can be found, we'll revert to building a local
    memory cache.

    Returns a fully configured caches dict.
    """
    caches = {}

    if all((environ.get(e, '') for e in MEMCACHE_ENV_VARS)):
        caches['default'] = {
            'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
            'BINARY': True,
            'LOCATION': 'localhost:11211',
            'OPTIONS': {
                'ketama': True,
                'tcp_nodelay': True,
            },
            'TIMEOUT': timeout,
        }
    elif all((environ.get(e, '') for e in MEMCACHIER_ENV_VARS)):
        environ['MEMCACHE_SERVERS'] = environ.get('MEMCACHIER_SERVERS').replace(',', ';')
        environ['MEMCACHE_USERNAME'] = environ.get('MEMCACHIER_USERNAME')
        environ['MEMCACHE_PASSWORD'] = environ.get('MEMCACHIER_PASSWORD')
        caches['default'] = {
            'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
            'BINARY': True,
            'LOCATION': environ.get('MEMCACHIER_SERVERS').replace(',', ';'),
            'OPTIONS': {
                'ketama': True,
                'tcp_nodelay': True,
            },
            'TIMEOUT': timeout,
        }
    elif environ.get('MEMCACHEIFY_USE_LOCAL', False):
        caches['default'] = {
            'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
        }
    else:
        caches['default'] = {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        }

    return caches
<commit_msg>Support for Memcached Cloud<commit_after>from os import environ

# Memcache addon environment variables.
# See: https://addons.heroku.com/memcache
MEMCACHE_ENV_VARS = (
    'MEMCACHE_PASSWORD',
    'MEMCACHE_SERVERS',
    'MEMCACHE_USERNAME',
)

# MemCachier addon environment variables.
# See: https://addons.heroku.com/memcachier
MEMCACHIER_ENV_VARS = (
    'MEMCACHIER_PASSWORD',
    'MEMCACHIER_SERVERS',
    'MEMCACHIER_USERNAME',
)

# Memcached Cloud
# See: https://addons.heroku.com/memcachedcloud
MEMCACHEDCLOUD_ENV_VARS = (
    'MEMCACHEDCLOUD_PASSWORD',
    'MEMCACHEDCLOUD_SERVERS',
    'MEMCACHEDCLOUD_USERNAME',
)

def memcacheify(timeout=500):
    """Return a fully configured Django ``CACHES`` setting. We do this by
    analyzing all environment variables on Heorku, scanning for an available
    memcache addon, and then building the settings dict properly.

    If no memcache servers can be found, we'll revert to building a local
    memory cache.

    Returns a fully configured caches dict.
    """
    caches = {}

    if all((environ.get(e, '') for e in MEMCACHE_ENV_VARS)):
        caches['default'] = {
            'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
            'BINARY': True,
            'LOCATION': 'localhost:11211',
            'OPTIONS': {
                'ketama': True,
                'tcp_nodelay': True,
            },
            'TIMEOUT': timeout,
        }
    elif all((environ.get(e, '') for e in MEMCACHIER_ENV_VARS)):
        environ['MEMCACHE_SERVERS'] = environ.get('MEMCACHIER_SERVERS').replace(',', ';')
        environ['MEMCACHE_USERNAME'] = environ.get('MEMCACHIER_USERNAME')
        environ['MEMCACHE_PASSWORD'] = environ.get('MEMCACHIER_PASSWORD')
        caches['default'] = {
            'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
            'BINARY': True,
            'LOCATION': environ.get('MEMCACHIER_SERVERS').replace(',', ';'),
            'OPTIONS': {
                'ketama': True,
                'tcp_nodelay': True,
            },
            'TIMEOUT': timeout,
        }
    elif all((environ.get(e, '') for e in MEMCACHEDCLOUD_ENV_VARS)):
        environ['MEMCACHE_SERVERS'] = environ.get('MEMCACHEDCLOUD_SERVERS').replace(',', ';')
        environ['MEMCACHE_USERNAME'] = environ.get('MEMCACHEDCLOUD_USERNAME')
        environ['MEMCACHE_PASSWORD'] = environ.get('MEMCACHEDCLOUD_PASSWORD')
        caches['default'] = {
            'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
            'BINARY': True,
            'LOCATION': environ.get('MEMCACHEDCLOUD_SERVERS').replace(',', ';'),
            'OPTIONS': {
                'ketama': True,
                'tcp_nodelay': True,
            },
            'TIMEOUT': timeout,
        }
    elif environ.get('MEMCACHEIFY_USE_LOCAL', False):
        caches['default'] = {
            'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
        }
    else:
        caches['default'] = {
            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        }

    return caches
<|endoftext|>


<commit_before>"""
Command line interface implementation.

Andrew DeOrio <awdeorio@umich.edu>
"""
from __future__ import print_function
import sys
import socket
import configparser
import smtplib
import textwrap
import jinja2
import click
from .template_message import TemplateMessage
from .sendmail_client import SendmailClient
from .utils import MailmergeError

# Python 2 pathlib support requires backport
try:
    from pathlib2 import Path
except ImportError:
    from pathlib import Path

# Python 2 UTF8 support requires csv backport
try:
    from backports import csv
except ImportError:
    import csv

@click.command(context_settings={"help_option_names": ['-h', '--help']})
@click.version_option()  # Auto detect version from setup.py
@click.option(
    "--sample", is_flag=True, default=False,
    help="Create sample database, template email, and config",
)
@click.option(
    "--dry-run/--no-dry-run", default=True,
    help="Don't send email, just print (True)",
)
@click.option(
    "--limit", is_flag=False, default=1,
    type=click.IntRange(0, None),
    help="Limit the number of messages (1)",
)
@click.option(
    "--no-limit", is_flag=True, default=False,
    help="Do not limit the number of messages",
)
@click.option(
    "--template", "template_path",
    default="mailmerge_template.txt",
    type=click.Path(),
    help="template email file name (mailmerge_template.txt)"
)
@click.option(
    "--database", "database_path",
    default="mailmerge_database.csv",
    type=click.Path(),
    help="database CSV file name (mailmerge_database.csv)",
)
@click.option(
    "--config", "config_path",
    default="mailmerge_server.conf",
    type=click.Path(),
    help="configuration file name (mailmerge_server.conf)",
)
def main(sample, dry_run, limit, no_limit,
         database_path, template_path, config_path):
    """
    Mailmerge is a simple, command line mail merge tool.

    For examples and formatting features, see:
    https://github.com/awdeorio/mailmerge
    """
    # We need an argument for each command line option.  That also means a lot
    # of local variables.
    # pylint: disable=too-many-arguments, too-many-locals

    # Convert paths from string to Path objects
    # https://github.com/pallets/click/issues/405
    template_path = Path(template_path)
    database_path = Path(database_path)
    config_path = Path(config_path)

    check_input_files(template_path, database_path, config_path, sample)

    # No limit is an alias for limit=-1
    if no_limit:
        limit = -1

    try:
        template_message = TemplateMessage(template_path)
        csv_database = read_csv_database(database_path)
        sendmail_client = SendmailClient(config_path, dry_run)
        for i, row in enumerate_limit(csv_database, limit):
            sender, recipients, message = template_message.render(row)
            sendmail_client.sendmail(sender, recipients, message)
            print(">>> message {}".format(i))
            print(message.as_string())
            for filename in get_attachment_filenames(message):
                print(">>> attached {}".format(filename))
            print(">>> sent message {}".format(i))
    except jinja2.exceptions.TemplateError as err:
        sys.exit(">>> Error in Jinja2 template: {}".format(err))
    except csv.Error as err:
        sys.exit(">>> Error reading CSV file: {}".format(err))
    except smtplib.SMTPAuthenticationError as err:
        sys.exit(">>> Authentication error: {}".format(err))
    except configparser.Error as err:
        sys.exit(
            ">>> Error reading config file {filename}: {message}"
            .format(filename=config_path, message=err)
        )
    except smtplib.SMTPException as err:
        sys.exit(">>> Error sending message", err, sep=' ', file=sys.stderr)
    except socket.error:
        sys.exit(">>> Error connecting to server")
    except MailmergeError as err:
        sys.exit(">>> {}".format(err))

    # Hints for user
    if not no_limit:
        print(
            ">>> Limit was {} messages.  "
            "To remove the limit, use the --no-limit option."
            .format(limit)
        )
    if dry_run:
        print(
            ">>> This was a dry run.  "
            "To send messages, use the --no-dry-run option."
        )

if __name__ == "__main__":
    # No value for parameter, that's how click works
    # pylint: disable=no-value-for-parameter
    main()

def check_input_files(template_path, database_path, config_path, sample):
    """Check if input files are present and hint the user."""
    if sample:
        create_sample_input_files(template_path, database_path, config_path)
        sys.exit(0)

    if not template_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find template {template_path}
            Create a sample (--sample) or specify a file (--template)

            See https://github.com/awdeorio/mailmerge for examples.
        """.format(template_path=template_path)))

    if not database_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find database {database_path}
            Create a sample (--sample) or specify a file (--database)

            See https://github.com/awdeorio/mailmerge for examples.
        """.format(database_path=database_path)))

    if not config_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find config {config_path}
            Create a sample (--sample) or specify a file (--config)

            See https://github.com/awdeorio/mailmerge for examples.
        """.format(config_path=config_path)))

def create_sample_input_files(template_path, database_path, config_path):
    """Create sample template, database and server config."""
    for path in [template_path, database_path, config_path]:
        if path.exists():
            sys.exit("Error: file exists: {}".format(path))
    with template_path.open("w") as template_file:
        template_file.write(textwrap.dedent(u"""\
            TO: {{email}}
            SUBJECT: Testing mailmerge
            FROM: My Self <myself@mydomain.com>

            Hi, {{name}},

            Your number is {{number}}.
        """))
    with database_path.open("w") as database_file:
        database_file.write(textwrap.dedent(u"""\
            email,name,number
            myself@mydomain.com,"Myself",17
            bob@bobdomain.com,"Bob",42
        """))
    with config_path.open("w") as config_file:
        config_file.write(textwrap.dedent(u"""\
            # Example: GMail, with SSL/TLS security
            [smtp_server]
            host = smtp.gmail.com
            port = 465
            security = SSL/TLS
            username = YOUR_USERNAME_HERE

            # Example: University of Michigan, with SSL/TLS security
            # [smtp_server]
            # host = smtp.mail.umich.edu
            # port = 465
            # security = SSL/TLS
            # username = YOUR_USERNAME_HERE

            # Example: University of Michigan EECS Dept., with STARTTLS security
            # [smtp_server]
            # host = newman.eecs.umich.edu
            # port = 25
            # security = STARTTLS
            # username = YOUR_USERNAME_HERE

            # Example: University of Michigan EECS Dept., with no security
            # [smtp_server]
            # host = newman.eecs.umich.edu
            # port = 25
        """))
    print(textwrap.dedent("""\
        Created sample template email "{template_path}"
        Created sample database "{database_path}"
        Created sample config file "{config_path}"
        Edit these files, then run mailmerge again.
    """.format(
        template_path=template_path,
        database_path=database_path,
        config_path=config_path,
    )))

def read_csv_database(database_path):
    """Read database CSV file, providing one line at a time."""
    # Modify the default dialect to be strict.  This will trigger errors for
    # things like unclosed quotes.
    class StrictExcel(csv.excel):
        """Strict version of default dialect."""

        # pylint: disable=too-few-public-methods
        strict = True

    # Open file and read using strict dialect
    with database_path.open("r") as database_file:
        reader = csv.DictReader(database_file, dialect=StrictExcel)
        for row in reader:
            yield row

def enumerate_limit(iterable, limit):
    """Enumerate iterable, stopping after limit iterations.

    When limit == -1, enumerate entire iterable.
    """
    for i, value in enumerate(iterable):
        if limit != -1 and i >= limit:
            return
        yield i, value

def get_attachment_filenames(message):
    """Return a list of attachment filenames."""
    if message.get_content_maintype() != "multipart":
        return []

    filenames = []
    for part in message.walk():
        if part.get_content_maintype() == "multipart":
            continue
        if part.get_content_maintype() == "text":
            continue
        if part.get("Content-Disposition") == "inline":
            continue
        if part.get("Content-Disposition") is None:
            continue
        filenames.append(part.get_filename())
    return filenames
<commit_msg>correct pluralization<commit_after>"""
Command line interface implementation.

Andrew DeOrio <awdeorio@umich.edu>
"""
from __future__ import print_function
import sys
import socket
import configparser
import smtplib
import textwrap
import jinja2
import click
from .template_message import TemplateMessage
from .sendmail_client import SendmailClient
from .utils import MailmergeError

# Python 2 pathlib support requires backport
try:
    from pathlib2 import Path
except ImportError:
    from pathlib import Path

# Python 2 UTF8 support requires csv backport
try:
    from backports import csv
except ImportError:
    import csv

@click.command(context_settings={"help_option_names": ['-h', '--help']})
@click.version_option()  # Auto detect version from setup.py
@click.option(
    "--sample", is_flag=True, default=False,
    help="Create sample database, template email, and config",
)
@click.option(
    "--dry-run/--no-dry-run", default=True,
    help="Don't send email, just print (True)",
)
@click.option(
    "--limit", is_flag=False, default=1,
    type=click.IntRange(0, None),
    help="Limit the number of messages (1)",
)
@click.option(
    "--no-limit", is_flag=True, default=False,
    help="Do not limit the number of messages",
)
@click.option(
    "--template", "template_path",
    default="mailmerge_template.txt",
    type=click.Path(),
    help="template email file name (mailmerge_template.txt)"
)
@click.option(
    "--database", "database_path",
    default="mailmerge_database.csv",
    type=click.Path(),
    help="database CSV file name (mailmerge_database.csv)",
)
@click.option(
    "--config", "config_path",
    default="mailmerge_server.conf",
    type=click.Path(),
    help="configuration file name (mailmerge_server.conf)",
)
def main(sample, dry_run, limit, no_limit,
         database_path, template_path, config_path):
    """
    Mailmerge is a simple, command line mail merge tool.

    For examples and formatting features, see:
    https://github.com/awdeorio/mailmerge
    """
    # We need an argument for each command line option.  That also means a lot
    # of local variables.
    # pylint: disable=too-many-arguments, too-many-locals

    # Convert paths from string to Path objects
    # https://github.com/pallets/click/issues/405
    template_path = Path(template_path)
    database_path = Path(database_path)
    config_path = Path(config_path)

    check_input_files(template_path, database_path, config_path, sample)

    # No limit is an alias for limit=-1
    if no_limit:
        limit = -1

    try:
        template_message = TemplateMessage(template_path)
        csv_database = read_csv_database(database_path)
        sendmail_client = SendmailClient(config_path, dry_run)
        for i, row in enumerate_limit(csv_database, limit):
            sender, recipients, message = template_message.render(row)
            sendmail_client.sendmail(sender, recipients, message)
            print(">>> message {}".format(i))
            print(message.as_string())
            for filename in get_attachment_filenames(message):
                print(">>> attached {}".format(filename))
            print(">>> sent message {}".format(i))
    except jinja2.exceptions.TemplateError as err:
        sys.exit(">>> Error in Jinja2 template: {}".format(err))
    except csv.Error as err:
        sys.exit(">>> Error reading CSV file: {}".format(err))
    except smtplib.SMTPAuthenticationError as err:
        sys.exit(">>> Authentication error: {}".format(err))
    except configparser.Error as err:
        sys.exit(
            ">>> Error reading config file {filename}: {message}"
            .format(filename=config_path, message=err)
        )
    except smtplib.SMTPException as err:
        sys.exit(">>> Error sending message", err, sep=' ', file=sys.stderr)
    except socket.error:
        sys.exit(">>> Error connecting to server")
    except MailmergeError as err:
        sys.exit(">>> {}".format(err))

    # Hints for user
    if not no_limit:
        print(
            ">>> Limit was {limit} message{pluralizer}.  "
            "To remove the limit, use the --no-limit option."
            .format(limit=limit, pluralizer=("" if limit == 1 else "s"))
        )
    if dry_run:
        print(
            ">>> This was a dry run.  "
            "To send messages, use the --no-dry-run option."
        )

if __name__ == "__main__":
    # No value for parameter, that's how click works
    # pylint: disable=no-value-for-parameter
    main()

def check_input_files(template_path, database_path, config_path, sample):
    """Check if input files are present and hint the user."""
    if sample:
        create_sample_input_files(template_path, database_path, config_path)
        sys.exit(0)

    if not template_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find template {template_path}
            Create a sample (--sample) or specify a file (--template)

            See https://github.com/awdeorio/mailmerge for examples.
        """.format(template_path=template_path)))

    if not database_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find database {database_path}
            Create a sample (--sample) or specify a file (--database)

            See https://github.com/awdeorio/mailmerge for examples.
        """.format(database_path=database_path)))

    if not config_path.exists():
        sys.exit(textwrap.dedent(u"""
            Error: can't find config {config_path}
            Create a sample (--sample) or specify a file (--config)

            See https://github.com/awdeorio/mailmerge for examples.
        """.format(config_path=config_path)))

def create_sample_input_files(template_path, database_path, config_path):
    """Create sample template, database and server config."""
    for path in [template_path, database_path, config_path]:
        if path.exists():
            sys.exit("Error: file exists: {}".format(path))
    with template_path.open("w") as template_file:
        template_file.write(textwrap.dedent(u"""\
            TO: {{email}}
            SUBJECT: Testing mailmerge
            FROM: My Self <myself@mydomain.com>

            Hi, {{name}},

            Your number is {{number}}.
        """))
    with database_path.open("w") as database_file:
        database_file.write(textwrap.dedent(u"""\
            email,name,number
            myself@mydomain.com,"Myself",17
            bob@bobdomain.com,"Bob",42
        """))
    with config_path.open("w") as config_file:
        config_file.write(textwrap.dedent(u"""\
            # Example: GMail, with SSL/TLS security
            [smtp_server]
            host = smtp.gmail.com
            port = 465
            security = SSL/TLS
            username = YOUR_USERNAME_HERE

            # Example: University of Michigan, with SSL/TLS security
            # [smtp_server]
            # host = smtp.mail.umich.edu
            # port = 465
            # security = SSL/TLS
            # username = YOUR_USERNAME_HERE

            # Example: University of Michigan EECS Dept., with STARTTLS security
            # [smtp_server]
            # host = newman.eecs.umich.edu
            # port = 25
            # security = STARTTLS
            # username = YOUR_USERNAME_HERE

            # Example: University of Michigan EECS Dept., with no security
            # [smtp_server]
            # host = newman.eecs.umich.edu
            # port = 25
        """))
    print(textwrap.dedent("""\
        Created sample template email "{template_path}"
        Created sample database "{database_path}"
        Created sample config file "{config_path}"
        Edit these files, then run mailmerge again.
    """.format(
        template_path=template_path,
        database_path=database_path,
        config_path=config_path,
    )))

def read_csv_database(database_path):
    """Read database CSV file, providing one line at a time."""
    # Modify the default dialect to be strict.  This will trigger errors for
    # things like unclosed quotes.
    class StrictExcel(csv.excel):
        """Strict version of default dialect."""

        # pylint: disable=too-few-public-methods
        strict = True

    # Open file and read using strict dialect
    with database_path.open("r") as database_file:
        reader = csv.DictReader(database_file, dialect=StrictExcel)
        for row in reader:
            yield row

def enumerate_limit(iterable, limit):
    """Enumerate iterable, stopping after limit iterations.

    When limit == -1, enumerate entire iterable.
    """
    for i, value in enumerate(iterable):
        if limit != -1 and i >= limit:
            return
        yield i, value

def get_attachment_filenames(message):
    """Return a list of attachment filenames."""
    if message.get_content_maintype() != "multipart":
        return []

    filenames = []
    for part in message.walk():
        if part.get_content_maintype() == "multipart":
            continue
        if part.get_content_maintype() == "text":
            continue
        if part.get("Content-Disposition") == "inline":
            continue
        if part.get("Content-Disposition") is None:
            continue
        filenames.append(part.get_filename())
    return filenames
<|endoftext|>