<commit_before>"""
Mixins to bind the local models to Axilent calendar functionality.
"""
from djax.calendar.models import CalendarEvent, CalendarResource
from django.contrib.contenttypes.models import ContentType

class AxilentCalendarEvent(object):
    """
    Adds calendar event sync functionality for the local model.
    """
    def push_event(self,start,end,recurrence_quantity=None,recurrence_unit=None,recurrence_end=None,resources=None):
        """
        Pushes the associated event to Axilent.  Should be called AFTER saving the local event model.
        """
        cal_event = None
        try:
            cal_event = CalendarEvent.objects.get_event_for_model(self)
            cal_event.start = start
            cal_event.end = end
            cal_event.recurrence_quantity = recurrence_quantity
            cal_event.recurrence_unit = recurrence_unit
            cal_event.recurrence_end = recurrence_end
            cal_event.save()
        except CalendarEvent.DoesNotExist:
            ctype = ContentType.objects.get_for_model(self)

            cal_event = CalendarEvent.objects.create(calendar=self.Axilent.calendar,
                                                     start=start,
                                                     end=end,
                                                     recurrence_quantity=recurrence_quantity,
                                                     recurrence_unit=recurrence_unit,
                                                     recurrence_end=recurrence_end,
                                                     local_content_type=ctype,
                                                     local_id=self.pk)

        cal_event.push_to_axilent(resource_models=resources)

    def delete_event(self):
        """
        Deletes the associated event in the Axilent calendar.
        """
        try:
            cal_event = CalendarEvent.objects.get_event_for_model(self)
            cal_event.delete_event() # deletes on server
            cal_event.delete()
            return True
        except CalendarEvent.DoesNotExist:
            return False

    def get_event(self):
        """
        Gets the matching calendar event, or None if the event has not yet been pushed.
        """
        try:
            return CalendarEvent.objects.get_event_for_model(self)
        except CalendarEvent.DoesNotExist:
            return None

class AxilentCalendarResource(object):
    """
    A mixin for local models that are to be considered calendar resources.
    """
    def push_resource(self):
        """
        Creates the equivilent resource object on Axilent corresponding to the local model.
        """
        resource = CalendarResource.objects.resource_for_model(self)
        return resource.sync()

    def resource_profile(self):
        """
        Gets the resource profile for this resource.
        """
        try:
            return CalendarResource.objects.resource_for_model(self).profile
        except CalendarResource.DoesNotExist:
            return None

    def check_availability(self,calendar,start,end):
        """
        Checks if this resource is available during the specified date range.
        """
        return CalendarEvent.objects.check_availability(calendar,start,end,self)
<commit_msg>Resources need to be a lists.<commit_after>"""
Mixins to bind the local models to Axilent calendar functionality.
"""
from djax.calendar.models import CalendarEvent, CalendarResource
from django.contrib.contenttypes.models import ContentType

class AxilentCalendarEvent(object):
    """
    Adds calendar event sync functionality for the local model.
    """
    def push_event(self,start,end,recurrence_quantity=None,recurrence_unit=None,recurrence_end=None,resources=None):
        """
        Pushes the associated event to Axilent.  Should be called AFTER saving the local event model.
        """
        cal_event = None
        try:
            cal_event = CalendarEvent.objects.get_event_for_model(self)
            cal_event.start = start
            cal_event.end = end
            cal_event.recurrence_quantity = recurrence_quantity
            cal_event.recurrence_unit = recurrence_unit
            cal_event.recurrence_end = recurrence_end
            cal_event.save()
        except CalendarEvent.DoesNotExist:
            ctype = ContentType.objects.get_for_model(self)

            cal_event = CalendarEvent.objects.create(calendar=self.Axilent.calendar,
                                                     start=start,
                                                     end=end,
                                                     recurrence_quantity=recurrence_quantity,
                                                     recurrence_unit=recurrence_unit,
                                                     recurrence_end=recurrence_end,
                                                     local_content_type=ctype,
                                                     local_id=self.pk)

        cal_event.push_to_axilent(resource_models=resources)

    def delete_event(self):
        """
        Deletes the associated event in the Axilent calendar.
        """
        try:
            cal_event = CalendarEvent.objects.get_event_for_model(self)
            cal_event.delete_event() # deletes on server
            cal_event.delete()
            return True
        except CalendarEvent.DoesNotExist:
            return False

    def get_event(self):
        """
        Gets the matching calendar event, or None if the event has not yet been pushed.
        """
        try:
            return CalendarEvent.objects.get_event_for_model(self)
        except CalendarEvent.DoesNotExist:
            return None

class AxilentCalendarResource(object):
    """
    A mixin for local models that are to be considered calendar resources.
    """
    def push_resource(self):
        """
        Creates the equivilent resource object on Axilent corresponding to the local model.
        """
        resource = CalendarResource.objects.resource_for_model(self)
        return resource.sync()

    def resource_profile(self):
        """
        Gets the resource profile for this resource.
        """
        try:
            return CalendarResource.objects.resource_for_model(self).profile
        except CalendarResource.DoesNotExist:
            return None

    def check_availability(self,calendar,start,end):
        """
        Checks if this resource is available during the specified date range.
        """
        return CalendarEvent.objects.check_availability(calendar,start,end,[self])
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# The GitRepoDriver provides the following:
#
# 1- A getpackages function based on finding all directories inside a specific
#    git repo, where each directory represents a package
#
# 2- A getinfo function based on a single-distgit repo paradigm
#
# 3- A preprocess function using renderspec on any *.spec.j2 file found in the
#    distgit.

import logging
import os
import re
import sh

from dlrn.db import Commit
from dlrn.drivers.pkginfo import PkgInfoDriver
from dlrn.repositories import getsourcebranch
from dlrn.repositories import refreshrepo
from pymod2pkg import module2upstream

logging.basicConfig(level=logging.ERROR)
logger = logging.getLogger("dlrn-gitrepo-driver")
logger.setLevel(logging.INFO)

version_match = re.compile('\W*set upstream_version\D+([\w.]+).*')
wrong_match = re.compile('\W*set upstream_version\D+\(\).*')

class GitRepoDriver(PkgInfoDriver):
    def __init__(self, *args, **kwargs):
        super(GitRepoDriver, self).__init__(*args, **kwargs)

    def getpackages(self, **kwargs):
        repo = self.config_options.gitrepo_repo
        path = self.config_options.gitrepo_dir.strip('/')
        distro_branch = self.config_options.distro
        datadir = self.config_options.datadir
        skip_dirs = self.config_options.skip_dirs
        dev_mode = kwargs.get('dev_mode')
        packages = []

        gitpath = os.path.join(datadir, 'package_info')
        if not os.path.exists(gitpath):
            sh.git.clone(repo, gitpath)

        if not dev_mode:
            git = sh.git.bake(_cwd=gitpath, _tty_out=False, _timeout=3600)
            git.fetch("origin")
            # Use the the configured distro branch, fall back to master
            # if it fails
            try:
                git.reset("--hard", "origin/%s" % distro_branch)
            except Exception:
                logger.info("Falling back to master")
                git.reset("--hard", "origin/master")

        packagepath = os.path.join(gitpath, path)

        for package in os.listdir(packagepath):
            if (os.path.isdir(os.path.join(packagepath, package)) and
               package not in skip_dirs):
                pkg_hash = {}
                pkg_hash['name'] = package
                pkg_hash['upstream'] = ('https://github.com/openstack/' +
                                        module2upstream(package))
                pkg_hash['maintainers'] = 'test@example.com'
                pkg_hash['master-distgit'] = (repo + '/' + path + '/' +
                                              package)
                if self.config_options.use_version_from_spec is True:
                    version = None
                    # Try to deduce version from spec template
                    pkgdir = os.path.join(packagepath, package)
                    for pkgfile in os.listdir(pkgdir):
                        if pkgfile.endswith('.j2'):
                            with open(os.path.join(pkgdir, pkgfile)) as fp:
                                j2content = fp.readlines()
                            for line in j2content:
                                # Make sure we are not matching the wrong
                                # version of the upstream_version() macro
                                m = wrong_match.match(line)
                                if m is not None:
                                    break   # So do not store a version
                                # Check if template defines upstream_version
                                m = version_match.match(line)
                                if m is not None:
                                    version = m.group(1)
                                    break
                                # Otherwise, we're using a direct version
                                if line.startswith('Version:'):
                                    version = line.split(':')[1].strip().\
                                        replace('~', '')
                                    break

                    if version is not None:
                        pkg_hash['source-branch'] = version
                packages.append(pkg_hash)
        return packages

    def getinfo(self, **kwargs):
        project = kwargs.get('project')
        package = kwargs.get('package')
        since = kwargs.get('since')
        local = kwargs.get('local')
        dev_mode = kwargs.get('dev_mode')
        datadir = self.config_options.datadir
        repo = package['upstream']

        distro_dir = self.distgit_dir(package['name'])
        source_branch = getsourcebranch(package)

        if dev_mode is True:
            distro_hash = "dev"
            dt_distro = 0  # Doesn't get used in dev mode
        else:
            # Get distro_hash from last commit in distgit directory
            git = sh.git.bake(_cwd=distro_dir, _tty_out=False)
            repoinfo = str(git.log("--pretty=format:%H %ct", "-1", ".")
                           ).strip().split(" ")
            distro_hash = repoinfo[0]
            dt_distro = repoinfo[1]

        # repo is usually a string, but if it contains more then one entry we
        # git clone into a project subdirectory
        repos = [repo]
        if isinstance(repo, list):
            repos = repo
        project_toprocess = []
        for repo in repos:
            repo_dir = os.path.join(datadir, project)
            if len(repos) > 1:
                repo_dir = os.path.join(repo_dir, os.path.split(repo)[1])
            source_branch, _, _ = refreshrepo(repo, repo_dir, source_branch,
                                              local=local)

            git = sh.git.bake(_cwd=repo_dir, _tty_out=False)
            # Git gives us commits already sorted in the right order
            lines = git.log("--pretty=format:'%ct %H'", since,
                            "--first-parent", "--reverse")

            for line in lines:
                dt, commit_hash = str(line).strip().strip("'").split(" ")
                commit = Commit(dt_commit=float(dt), project_name=project,
                                commit_hash=commit_hash, repo_dir=repo_dir,
                                distro_hash=distro_hash, dt_distro=dt_distro,
                                distgit_dir=distro_dir,
                                commit_branch=source_branch)
                project_toprocess.append(commit)

        return project_toprocess

    def preprocess(self, **kwargs):
        package_name = kwargs.get('package_name')
        distgit_dir = self.distgit_dir(package_name)
        preprocess = sh.renderspec.bake(_cwd=distgit_dir,
                                        _tty_out=False, _timeout=3600)
        preprocess('--spec-style', 'fedora', '--epoch',
                   '../../epoch/fedora.yaml')
        # Replace %{version} with %{upstream_version} in spec
        # This is required by rpm-packaging specs
        for specfile in os.listdir(distgit_dir):
            if specfile.endswith(".spec"):
                filename = os.path.join(distgit_dir, specfile)
                with open(filename, 'r+') as fp:
                    spec = fp.read()
                    spec = re.sub('-%{version}', '-%{upstream_version}', spec)
                    fp.seek(0)
                    fp.write(spec)

    def distgit_dir(self, package_name):
        datadir = self.config_options.datadir
        path = self.config_options.gitrepo_dir.strip('/')
        return os.path.join(datadir, 'package_info', path, package_name)
<commit_msg>Allow several base URLs for projects in the gitrepo driver<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# The GitRepoDriver provides the following:
#
# 1- A getpackages function based on finding all directories inside a specific
#    git repo, where each directory represents a package
#
# 2- A getinfo function based on a single-distgit repo paradigm
#
# 3- A preprocess function using renderspec on any *.spec.j2 file found in the
#    distgit.

import logging
import os
import re
import sh
import urllib2

from dlrn.db import Commit
from dlrn.drivers.pkginfo import PkgInfoDriver
from dlrn.repositories import getsourcebranch
from dlrn.repositories import refreshrepo
from pymod2pkg import module2upstream

logging.basicConfig(level=logging.ERROR)
logger = logging.getLogger("dlrn-gitrepo-driver")
logger.setLevel(logging.INFO)

version_match = re.compile('\W*set upstream_version\D+([\w.]+).*')
wrong_match = re.compile('\W*set upstream_version\D+\(\).*')

base_urls = ['https://github.com/openstack',
             'https://github.com/openstack-dev']

def check_url(url):
    logger.info("Checking url %s" % url)
    try:
        urllib2.urlopen(url)
        # URL found
        return True
    except (urllib2.HTTPError, urllib2.URLError):
        # Trouble finding URL
        return False

class GitRepoDriver(PkgInfoDriver):
    def __init__(self, *args, **kwargs):
        super(GitRepoDriver, self).__init__(*args, **kwargs)

    def getpackages(self, **kwargs):
        repo = self.config_options.gitrepo_repo
        path = self.config_options.gitrepo_dir.strip('/')
        distro_branch = self.config_options.distro
        datadir = self.config_options.datadir
        skip_dirs = self.config_options.skip_dirs
        dev_mode = kwargs.get('dev_mode')
        packages = []

        gitpath = os.path.join(datadir, 'package_info')
        if not os.path.exists(gitpath):
            sh.git.clone(repo, gitpath)

        if not dev_mode:
            git = sh.git.bake(_cwd=gitpath, _tty_out=False, _timeout=3600)
            git.fetch("origin")
            # Use the the configured distro branch, fall back to master
            # if it fails
            try:
                git.reset("--hard", "origin/%s" % distro_branch)
            except Exception:
                logger.info("Falling back to master")
                git.reset("--hard", "origin/master")

        packagepath = os.path.join(gitpath, path)

        for package in os.listdir(packagepath):
            if (os.path.isdir(os.path.join(packagepath, package)) and
               package not in skip_dirs):
                pkg_hash = {}
                pkg_hash['name'] = package
                for url in base_urls:
                    if check_url("%s/%s" % (url, module2upstream(package))):
                        pkg_hash['upstream'] = ("%s/%s" %
                                                (url,
                                                 module2upstream(package)))
                        break
                else:
                    logger.error("Could not find upstream URL for project %s" %
                                 package)

                pkg_hash['maintainers'] = 'test@example.com'
                pkg_hash['master-distgit'] = (repo + '/' + path + '/' +
                                              package)
                if self.config_options.use_version_from_spec is True:
                    version = None
                    # Try to deduce version from spec template
                    pkgdir = os.path.join(packagepath, package)
                    for pkgfile in os.listdir(pkgdir):
                        if pkgfile.endswith('.j2'):
                            with open(os.path.join(pkgdir, pkgfile)) as fp:
                                j2content = fp.readlines()
                            for line in j2content:
                                # Make sure we are not matching the wrong
                                # version of the upstream_version() macro
                                m = wrong_match.match(line)
                                if m is not None:
                                    break   # So do not store a version
                                # Check if template defines upstream_version
                                m = version_match.match(line)
                                if m is not None:
                                    version = m.group(1)
                                    break
                                # Otherwise, we're using a direct version
                                if line.startswith('Version:'):
                                    version = line.split(':')[1].strip().\
                                        replace('~', '')
                                    break

                    if version is not None:
                        pkg_hash['source-branch'] = version
                packages.append(pkg_hash)
        return packages

    def getinfo(self, **kwargs):
        project = kwargs.get('project')
        package = kwargs.get('package')
        since = kwargs.get('since')
        local = kwargs.get('local')
        dev_mode = kwargs.get('dev_mode')
        datadir = self.config_options.datadir
        repo = package['upstream']

        distro_dir = self.distgit_dir(package['name'])
        source_branch = getsourcebranch(package)

        if dev_mode is True:
            distro_hash = "dev"
            dt_distro = 0  # Doesn't get used in dev mode
        else:
            # Get distro_hash from last commit in distgit directory
            git = sh.git.bake(_cwd=distro_dir, _tty_out=False)
            repoinfo = str(git.log("--pretty=format:%H %ct", "-1", ".")
                           ).strip().split(" ")
            distro_hash = repoinfo[0]
            dt_distro = repoinfo[1]

        # repo is usually a string, but if it contains more then one entry we
        # git clone into a project subdirectory
        repos = [repo]
        if isinstance(repo, list):
            repos = repo
        project_toprocess = []
        for repo in repos:
            repo_dir = os.path.join(datadir, project)
            if len(repos) > 1:
                repo_dir = os.path.join(repo_dir, os.path.split(repo)[1])
            source_branch, _, _ = refreshrepo(repo, repo_dir, source_branch,
                                              local=local)

            git = sh.git.bake(_cwd=repo_dir, _tty_out=False)
            # Git gives us commits already sorted in the right order
            lines = git.log("--pretty=format:'%ct %H'", since,
                            "--first-parent", "--reverse")

            for line in lines:
                dt, commit_hash = str(line).strip().strip("'").split(" ")
                commit = Commit(dt_commit=float(dt), project_name=project,
                                commit_hash=commit_hash, repo_dir=repo_dir,
                                distro_hash=distro_hash, dt_distro=dt_distro,
                                distgit_dir=distro_dir,
                                commit_branch=source_branch)
                project_toprocess.append(commit)

        return project_toprocess

    def preprocess(self, **kwargs):
        package_name = kwargs.get('package_name')
        distgit_dir = self.distgit_dir(package_name)
        preprocess = sh.renderspec.bake(_cwd=distgit_dir,
                                        _tty_out=False, _timeout=3600)
        preprocess('--spec-style', 'fedora', '--epoch',
                   '../../epoch/fedora.yaml')
        # Replace %{version} with %{upstream_version} in spec
        # This is required by rpm-packaging specs
        for specfile in os.listdir(distgit_dir):
            if specfile.endswith(".spec"):
                filename = os.path.join(distgit_dir, specfile)
                with open(filename, 'r+') as fp:
                    spec = fp.read()
                    spec = re.sub('-%{version}', '-%{upstream_version}', spec)
                    fp.seek(0)
                    fp.write(spec)

    def distgit_dir(self, package_name):
        datadir = self.config_options.datadir
        path = self.config_options.gitrepo_dir.strip('/')
        return os.path.join(datadir, 'package_info', path, package_name)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Thu Sep 10 11:35:26 2015

@author: salimeh
"""

from dlstats.fetchers._commons import Fetcher, Categories, Series, Datasets, Providers, CodeDict, ElasticIndex
#from make_elastic_index import ElasticIndex
import urllib
import xlrd
import csv
import codecs
from datetime import datetime
import pandas
import pprint
from collections import OrderedDict
from re import match
from time import sleep
import zipfile
import io

class BEA(Fetcher):
    def __init__(self):
        super().__init__(provider_name='BEA')
        self.provider_name = 'BEA'
        self.provider = Provider(name=self.provider_name,website='www.bea.gov/')
        self.url = 'http://www.bea.gov//national/nipaweb/GetCSV.asp?GetWhat=SS_Data/SectionAll_xls.zip&Section=11'
    def upsert_nipa(self):

        response = urllib.request.urlopen(self.url)
        zipfile_ = zipfile.ZipFile(io.BytesIO(response.read()))
        excel_filenames = iter(zipfile_.namelist())
        fname = next(excel_filenames)

        for section in zipfile_.namelist():
            excel_book = xlrd.open_workbook(file_contents = zipfile_.read(section))
            for sheet_name in excel_book.sheet_names():
                sheet = excel_book.sheet_by_name(sheet_name)
                if  sheet_name != 'Contents':
                    datasetCode = sheet_name
                    self.upsert_dataset(datasetCode, sheet)

    def upsert_dataset(self, datasetCode, sheet):

        dataset = Dataset(self.provider_name,datasetCode)
        bea_data = BeaData(dataset,self.url, sheet)
        dataset.name = datasetCode
        dataset.doc_href = 'http://www.bea.gov/newsreleases/national/gdp/gdpnewsrelease.htm'
        dataset.last_update = bea_data.release_date
        dataset.series.data_iterator = bea_data
        dataset.update_database()

    def upsert_categories(self):
        document = Category(provider = self.provider_name,
                            name = 'BEA' ,
                            categoryCode ='BEA',
                            children = [None] )
        return document.update_database()

class BeaData():
    def __init__(self,dataset,url, sheet):
        self.sheet = sheet
        self.provider_name = dataset.provider_name
        self.dataset_code = dataset.dataset_code
        self.dimension_list = dataset.dimension_list
        self.attribute_list = dataset.attribute_list

        str = sheet.cell_value(2,0)
        info = []
        if 'Quartely' in str :
            self.frequency = 'Q'
        else :
            self.frequency = 'A'
        years = [int(s) for s in str.split() if s.isdigit()]
        self.start_date = pandas.Period(years[0],freq = self.frequency).ordinal
        self.end_date = pandas.Period(years[1],freq = self.frequency).ordinal
        self.release_date = datetime.strptime(sheet.cell_value(5,0)[13:].strip(), "%m/%d/%Y %H:%M:%S %p")
        self.dimensions = {}

        row_start = sheet.col_values(0).index(1)
        self.row_range = iter(range(row_start, sheet.nrows))
        if '' in sheet.col_values(1)[row_start:] :
            row_info = sheet.col_values(1).index('',row_start,sheet.nrows)+1
            if sheet.col_values(0)[row_info]:
                for row_no in range(row_info, sheet.nrows) :
                    info.append(sheet.cell_value(row_no,0))

    def __next__(self):
        row = self.sheet.row(next(self.row_range))
        if row is None:
            raise StopIteration()
        series = self.build_series(row)
        if series is None:
            raise StopIteration()
        return(series)

    def build_series(self,row):
        dimensions = {}
        series = {}
        series_value = []

        series_name = row[1].value + self.frequency
        series_key = 'BEA.' + self.sheet.col(0)[0].value + '; ' + row[1].value
        dimensions['concept'] = self.dimension_list.update_entry('concept',row[2].value,row[1].value)
        dimensions['line'] = self.dimension_list.update_entry('line',str(row[0].value),str(row[0].value))
        for r in range(3, len(row)):
            series_value.append(str(row[r].value))
        release_dates = [self.release_date for v in series_value]
        series['values'] = series_value
        series['provider'] = self.provider_name
        series['datasetCode'] = self.dataset_code
        series['name'] = series_name
        series['key'] = series_key
        series['startDate'] = self.start_date
        series['endDate'] = self.end_date
        series['releaseDates'] = release_dates
        series['dimensions'] = dimensions
        series['frequency'] = self.frequency
        series['attributes'] = {}
        return(series)

if __name__ == "__main__":
    w = BEA()
    w.provider.update_database()
    w.upsert_categories()
    w.upsert_nipa()

<commit_msg>Fix issues 15<commit_after># -*- coding: utf-8 -*-
"""
Created on Thu Sep 10 11:35:26 2015

@author: salimeh
"""

from dlstats.fetchers._commons import Fetcher, Categories, Series, Datasets, Providers, CodeDict, ElasticIndex
from dlstats import constants
import urllib
import xlrd
import csv
import codecs
from datetime import datetime
import pandas
import pprint
from collections import OrderedDict
from re import match
from time import sleep
import zipfile
import io

class BEA(Fetcher):
    def __init__(self, db=None, es_client=None):
        super().__init__(provider_name='BEA',  db=db, es_client=es_client)
        self.provider_name = 'BEA'
        self.provider = Provider(name = 'BEA' ,website='www.bea.gov/',fetcher=self)
        self.url = 'http://www.bea.gov//national/nipaweb/GetCSV.asp?GetWhat=SS_Data/SectionAll_xls.zip&Section=11'
    def upsert_nipa(self):

        response = urllib.request.urlopen(self.url)
        zipfile_ = zipfile.ZipFile(io.BytesIO(response.read()))
        excel_filenames = iter(zipfile_.namelist())
        fname = next(excel_filenames)

        for section in zipfile_.namelist():
            excel_book = xlrd.open_workbook(file_contents = zipfile_.read(section))
            for sheet_name in excel_book.sheet_names():
                sheet = excel_book.sheet_by_name(sheet_name)
                if  sheet_name != 'Contents':
                    datasetCode = sheet_name
                    self.upsert_dataset(datasetCode, sheet)

    def upsert_dataset(self, datasetCode, sheet):

        dataset = Dataset(self.provider_name,datasetCode)
        bea_data = BeaData(dataset,self.url, sheet)
        dataset.name = datasetCode
        dataset.doc_href = 'http://www.bea.gov/newsreleases/national/gdp/gdpnewsrelease.htm'
        dataset.last_update = bea_data.release_date
        dataset.series.data_iterator = bea_data
        dataset.update_database()
        self.update_metas(datasetCode)

    def upsert_categories(self):
        document = Category(provider = self.provider_name,
                            name = 'BEA' ,
                            categoryCode ='BEA',
                            children = [None] )
        return document.update_database()

class BeaData():
    def __init__(self,dataset,url, sheet):
        self.sheet = sheet
        self.provider_name = dataset.provider_name
        self.dataset_code = dataset.dataset_code
        self.dimension_list = dataset.dimension_list
        self.attribute_list = dataset.attribute_list

        str = sheet.cell_value(2,0)
        info = []
        if 'Quartely' in str :
            self.frequency = 'Q'
        else :
            self.frequency = 'A'
        years = [int(s) for s in str.split() if s.isdigit()]
        self.start_date = pandas.Period(years[0],freq = self.frequency).ordinal
        self.end_date = pandas.Period(years[1],freq = self.frequency).ordinal
        self.release_date = datetime.strptime(sheet.cell_value(5,0)[13:].strip(), "%m/%d/%Y %H:%M:%S %p")
        self.dimensions = {}

        row_start = sheet.col_values(0).index(1)
        self.row_range = iter(range(row_start, sheet.nrows))
        if '' in sheet.col_values(1)[row_start:] :
            row_info = sheet.col_values(1).index('',row_start,sheet.nrows)+1
            if sheet.col_values(0)[row_info]:
                for row_no in range(row_info, sheet.nrows) :
                    info.append(sheet.cell_value(row_no,0))

    def __next__(self):
        row = self.sheet.row(next(self.row_range))
        if row is None:
            raise StopIteration()
        series = self.build_series(row)
        if series is None:
            raise StopIteration()
        return(series)

    def build_series(self,row):
        dimensions = {}
        series = {}
        series_value = []

        series_name = row[1].value + self.frequency
        series_key = 'BEA.' + self.sheet.col(0)[0].value + '; ' + row[1].value
        dimensions['concept'] = self.dimension_list.update_entry('concept',row[2].value,row[1].value)
        dimensions['line'] = self.dimension_list.update_entry('line',str(row[0].value),str(row[0].value))
        for r in range(3, len(row)):
            series_value.append(str(row[r].value))
        release_dates = [self.release_date for v in series_value]
        series['values'] = series_value
        series['provider'] = self.provider_name
        series['datasetCode'] = self.dataset_code
        series['name'] = series_name
        series['key'] = series_key
        series['startDate'] = self.start_date
        series['endDate'] = self.end_date
        series['releaseDates'] = release_dates
        series['dimensions'] = dimensions
        series['frequency'] = self.frequency
        series['attributes'] = {}
        return(series)

if __name__ == "__main__":
    w = BEA()
    w.provider.update_database()
    w.upsert_categories()
    w.upsert_nipa()

<|endoftext|>


<commit_before>import datetime
import pytz
import factory

from . import models

class AccountFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Account

    username = factory.Sequence(lambda n: 'user%d' % n)
    url = factory.LazyAttribute(lambda obj: 'https://pinboard.com/%s' %
                                                                obj.username)
    api_token = factory.LazyAttribute(lambda obj: '%s:123ABC' % obj.username)

class BookmarkFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Bookmark

    # DittoItem properties:
    title = factory.Sequence(lambda n: 'A Title %d' % n)
    is_private = False

    # Bookmark properties:
    account = factory.SubFactory(AccountFactory)
    url = factory.Sequence(lambda n: 'http://www.example.com/%d' % n)

    post_time = factory.LazyAttribute(lambda o:
                        datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
                        - datetime.timedelta(hours=1)
                    )
    description = factory.Sequence(lambda n: 'A description of %d' % n)
    to_read = False

<commit_msg>Add comment<commit_after>import datetime
import pytz
import factory

from . import models

class AccountFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Account

    username = factory.Sequence(lambda n: 'user%d' % n)
    url = factory.LazyAttribute(lambda obj: 'https://pinboard.com/%s' %
                                                                obj.username)
    api_token = factory.LazyAttribute(lambda obj: '%s:123ABC' % obj.username)

class BookmarkFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Bookmark

    # DittoItem properties:
    title = factory.Sequence(lambda n: 'A Title %d' % n)
    is_private = False

    # Bookmark properties:
    account = factory.SubFactory(AccountFactory)
    url = factory.Sequence(lambda n: 'http://www.example.com/%d' % n)

    post_time = factory.LazyAttribute(lambda o:
                        datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
                        - datetime.timedelta(hours=1)
                    )
    description = factory.Sequence(lambda n: 'A description of %d' % n)
    to_read = False

    # Leave tags empty by default.

<|endoftext|>


<commit_before># These URLs are normally mapped to /admin/urls.py. This URLs file is
# provided as a convenience to those who want to deploy these URLs elsewhere.
# This file is also used to provide a reliable view deployment for test purposes.

from django.conf.urls.defaults import *

urlpatterns = patterns('',
    ('^logout/$', 'django.contrib.auth.views.logout'),
    ('^password_change/$', 'django.contrib.auth.views.password_change'),
    ('^password_change/done/$', 'django.contrib.auth.views.password_change_done'),
    ('^password_reset/$', 'django.contrib.auth.views.password_reset'),
    ('^password_reset/done/$', 'django.contrib.auth.views.password_reset_done'),
    ('^reset/(?P<uidb36>[0-9A-Za-z]+)-(?P<token>.+)/$', 'django.contrib.auth.views.password_reset_confirm'),
    ('^reset/done/$', 'django.contrib.auth.views.password_reset_complete'),
)

<commit_msg>Changed some URL conf regexes to rawstrings for consistency<commit_after># These URLs are normally mapped to /admin/urls.py. This URLs file is
# provided as a convenience to those who want to deploy these URLs elsewhere.
# This file is also used to provide a reliable view deployment for test purposes.

from django.conf.urls.defaults import *

urlpatterns = patterns('',
    (r'^logout/$', 'django.contrib.auth.views.logout'),
    (r'^password_change/$', 'django.contrib.auth.views.password_change'),
    (r'^password_change/done/$', 'django.contrib.auth.views.password_change_done'),
    (r'^password_reset/$', 'django.contrib.auth.views.password_reset'),
    (r'^password_reset/done/$', 'django.contrib.auth.views.password_reset_done'),
    (r'^reset/(?P<uidb36>[0-9A-Za-z]+)-(?P<token>.+)/$', 'django.contrib.auth.views.password_reset_confirm'),
    (r'^reset/done/$', 'django.contrib.auth.views.password_reset_complete'),
)

<|endoftext|>


<commit_before>from django.contrib.auth.models import User
from django.db import models

class BaseModel(models.Model):
    """
    Defines an abstract model built off of Django's Model class that
    provides some common fields that are useful on multiple models
    across multiple projects.
    """

    time_created = models.DateTimeField(auto_now_add=True)
    time_modified = models.DateTimeField(auto_now=True)
    last_modified_by = models.ForeignKey(
        User,
        related_name='%(app_label)s_%(class)s_related',
        null=True,
        blank=True
    )

    class Meta:
        abstract = True
<commit_msg>Added defaults to time_created and time_modified fields.<commit_after>import datetime

from django.contrib.auth.models import User
from django.db import models

class BaseModel(models.Model):
    """
    Defines an abstract model built off of Django's Model class that
    provides some common fields that are useful on multiple models
    across multiple projects.
    """

    time_created = models.DateTimeField(
        auto_now_add=True,
        default=datetime.datetime.now()
    )
    time_modified = models.DateTimeField(
        auto_now=True,
        default=datetime.datetime.now()
    )
    last_modified_by = models.ForeignKey(
        User,
        related_name='%(app_label)s_%(class)s_related',
        null=True,
        blank=True
    )

    class Meta:
        abstract = True
<|endoftext|>


<commit_before>"""
Created on 04/sep/2016

@author: Marco Pompili
"""

from socket import error as SocketError
from lxml import html
import requests
from requests.exceptions import ConnectionError
import json

SCRIPT_JSON_PREFIX = 18
SCRIPT_JSON_DATA_INDEX = 21

def instagram_scrap_profile(username):
    """
    Scrap an instagram profile page
    :param username:
    :return:
    """
    try:
        url = "https://www.instagram.com/{}/".format(username)
        page = requests.get(url)
        return html.fromstring(page.content)
    except (ConnectionError, SocketError) as e:
        return ''

def instagram_profile_js(username):
    """
    Retrieve the script tags from the parsed page.
    :param username:
    :return:
    """
    try:
        tree = instagram_scrap_profile(username)
        return tree.xpath('//script')
    except AttributeError:
        return ''

def instagram_profile_json(username):
    """
    Get the JSON data string from the scripts.
    :param username:
    :return:
    """
    scripts = instagram_profile_js(username)
    source = None

    for script in scripts:
        if script.text:
            if script.text[0:SCRIPT_JSON_PREFIX] == "window._sharedData":
                source = script.text[SCRIPT_JSON_DATA_INDEX:-1]

    return source

def instagram_profile_obj(username):
    """
    Retrieve the JSON from the page and parse it to a python dict.
    :param username:
    :return:
    """
    json_data = instagram_profile_json(username)
    return json.loads(json_data)
<commit_msg>Fixes in instagram_profile_json <commit_after>"""
Created on 04/sep/2016

@author: Marco Pompili
"""

from socket import error as SocketError
from lxml import html
import requests
from requests.exceptions import ConnectionError
import json

SCRIPT_JSON_PREFIX = 18
SCRIPT_JSON_DATA_INDEX = 21

def instagram_scrap_profile(username):
    """
    Scrap an instagram profile page
    :param username:
    :return:
    """
    try:
        url = "https://www.instagram.com/{}/".format(username)
        page = requests.get(url)
        return html.fromstring(page.content)
    except (ConnectionError, SocketError) as e:
        return ''

def instagram_profile_js(username):
    """
    Retrieve the script tags from the parsed page.
    :param username:
    :return:
    """
    try:
        tree = instagram_scrap_profile(username)
        return tree.xpath('//script')
    except AttributeError:
        return ''

def instagram_profile_json(username):
    """
    Get the JSON data string from the scripts.
    :param username:
    :return:
    """
    scripts = instagram_profile_js(username)
    source = None
    if scripts != '':
        for script in scripts:
            if script.text:
                if script.text[0:SCRIPT_JSON_PREFIX] == "window._sharedData":
                    source = script.text[SCRIPT_JSON_DATA_INDEX:-1]

    return source

def instagram_profile_obj(username):
    """
    Retrieve the JSON from the page and parse it to a python dict.
    :param username:
    :return:
    """
    json_data = instagram_profile_json(username)
    if json_data:
        return json.loads(json_data)

<|endoftext|>


<commit_before>from django.conf import settings
from django.utils.html import strip_tags
from django.utils.text import unescape_entities
from django.utils.encoding import force_unicode

ROA_DJANGO_ERRORS = getattr(settings, 'ROA_DJANGO_ERRORS', False)

class ROAException(Exception):
    def __init__(self, exception):
        if ROA_DJANGO_ERRORS and 'message' in exception \
                             and 'status_code' in exception:
            self.msg = force_unicode(exception.message)
            self.status_code = exception.status_code
        else:
            self.msg = force_unicode(exception)
            self.status_code = 'XXX'

    def __str__(self):
        if ROA_DJANGO_ERRORS and '' in self.msg:
            return self.parse_django_error()
        return self.msg

    def parse_django_error(self):
        """Extract the summary part of a Django HTML error."""
        summary = self.msg.split(u'\n
\n  ', 1)[1]\
                          .split(u'Python Executable:	', 1)[0]
        traceback = self.msg.split(u'\n\nTraceback:', 1)[1]\
                            .split(u'</textarea>', 1)[0]
        result = []
        title = None
        for line in strip_tags(summary).split('\n'):
            line_content = unescape_entities(line.strip())
            if line_content:
                if line_content.endswith(':'):
                    title = line_content
                elif title is None:
                    title = "%s:" % line_content
                else:
                    result.append("%s %s\n" % (title, line_content))
        result.append("Status code: %s" % self.status_code)
        indent, indent2 = u'  ', u'    '
        return u"%(summary)s %(traceback)s".strip() % {
            'summary': indent.join(force_unicode(line) for line in result),
            'traceback': indent2.join(force_unicode(line+"\n") \
                                        for line in traceback.split('\n')),
        }

class ROANotImplementedYetException(Exception):
    pass
<commit_msg>More robust way to parse Django's errors<commit_after>from django.conf import settings
from django.utils.html import strip_tags
from django.utils.text import unescape_entities
from django.utils.encoding import force_unicode

ROA_DJANGO_ERRORS = getattr(settings, 'ROA_DJANGO_ERRORS', False)

class ROAException(Exception):
    def __init__(self, exception):
        if ROA_DJANGO_ERRORS and 'message' in exception \
                             and 'status_code' in exception:
            self.msg = force_unicode(exception.message)
            self.status_code = exception.status_code
        else:
            self.msg = force_unicode(exception)
            self.status_code = 'XXX'

    def __str__(self):
        if ROA_DJANGO_ERRORS and '' in self.msg:
            return self.parse_django_error()
        return self.msg

    def parse_django_error(self):
        """Extract the summary part of a Django HTML error."""
        try:
            summary = self.msg.split(u'\n
\n  ', 1)[1]\
                              .split(u'Python Executable:	', 1)[0]
            traceback = self.msg.split(u'\n\nTraceback:', 1)[1]\
                                .split(u'</textarea>', 1)[0]
        except IndexError:
            return self.msg
        result = []
        title = None
        for line in strip_tags(summary).split('\n'):
            line_content = unescape_entities(line.strip())
            if line_content:
                if line_content.endswith(':'):
                    title = line_content
                elif title is None:
                    title = "%s:" % line_content
                else:
                    result.append("%s %s\n" % (title, line_content))
        result.append("Status code: %s" % self.status_code)
        indent, indent2 = u'  ', u'    '
        return u"%(summary)s %(traceback)s".strip() % {
            'summary': indent.join(force_unicode(line) for line in result),
            'traceback': indent2.join(force_unicode(line+"\n") \
                                        for line in traceback.split('\n')),
        }

class ROANotImplementedYetException(Exception):
    pass
<|endoftext|>


<commit_before>import operator
import storymarket
from django import forms
from django.core.cache import cache
from django.conf import settings
from .models import SyncedObject

# Timeout for choices cached from Storymarket. 5 minutes.
CHOICE_CACHE_TIMEOUT = 600

class StorymarketAPIChoiceField(forms.ChoiceField):
    """
    Helper base class for ChoiceFields that offer choices by calling a
    Storymarket API.

    For speed, the list of allowable items are stored in the cache.
    """

    def __init__(self, *args, **kwargs):
        super(StorymarketAPIChoiceField, self).__init__(choices=self._get_choices(), *args, **kwargs)

    def _get_choices(self):
        # Return a list of choices sorted by name, along with an empty choice.
        cache_key = 'storymarket_choice_cache:%s' % self.__class__.__name__
        choices = cache.get(cache_key)
        if choices is None:
            objs = sorted(self._call_api(), key=operator.attrgetter('name'))

            # If there's only a single object, just select it -- don't offer
            # an empty choice. Otherwise, offer an empty.
            if len(objs) == 1:
                empty_choice = []
            else:
                empty_choice = [(u'', u'---------')]
            choices = empty_choice + [(o.id, o.name) for o in objs]
            cache.set(cache_key, choices, CHOICE_CACHE_TIMEOUT)
        return choices

class OrgChoiceField(StorymarketAPIChoiceField):
    def _call_api(self):
        api = storymarket.Storymarket(settings.STORYMARKET_API_KEY)
        return api.orgs.all()

class CategoryChoiceField(StorymarketAPIChoiceField):
    def _call_api(self):
        api = storymarket.Storymarket(settings.STORYMARKET_API_KEY)
        return api.subcategories.all()

class PricingChoiceField(StorymarketAPIChoiceField):
    def _call_api(self):
        api = storymarket.Storymarket(settings.STORYMARKET_API_KEY)
        return api.pricing.all()

class RightsChoiceField(StorymarketAPIChoiceField):
    def _call_api(self):
        api = storymarket.Storymarket(settings.STORYMARKET_API_KEY)
        return api.rights.all()

class StorymarketSyncForm(forms.Form):
    """
    A form allowing the choice of sync options for a given model instance.
    """
    org = OrgChoiceField()
    category = CategoryChoiceField()
    pricing = PricingChoiceField()
    rights = RightsChoiceField()
    tags = forms.CharField()

    def __init__(self, *args, **kwargs):
        api = storymarket.Storymarket(settings.STORYMARKET_API_KEY)
        if 'type' in kwargs:
            self.storymarket_type = kwargs.pop('type')
        else:
            raise TypeError("__init__() missing required keyword argument 'type'")
        initial = kwargs.pop('initial', {})
        instance = kwargs.pop('instance', None)

        # If we've been given an object instance then look up initial data
        # from Storymarket.
        if instance:
            try:
                sync_info = SyncedObject.objects.for_model(instance).get()
            except SyncedObject.DoesNotExist:
                pass
            else:
                initial.update({
                    'org': sync_info.org_id,
                    'category': sync_info.category_id,
                    'pricing': sync_info.pricing_id,
                    'rights': sync_info.rights_id,
                    'tags': sync_info_tags,
                })

        super(StorymarketSyncForm, self).__init__(initial=initial, *args, **kwargs)<commit_msg>Fixed the way storymarket choice fields generate choices.<commit_after>import operator
import storymarket
from django import forms
from django.core.cache import cache
from django.conf import settings
from .models import SyncedObject

# Timeout for choices cached from Storymarket. 5 minutes.
CHOICE_CACHE_TIMEOUT = 600

class StorymarketAPIChoiceField(forms.ChoiceField):
    """
    Helper base class for ChoiceFields that offer choices by calling a
    Storymarket API.

    For speed, the list of allowable items are stored in the cache.
    """

    def __init__(self, *args, **kwargs):
        # Skip ChoiceField's initializer since we're handling choices
        # via the property below.
        super(forms.ChoiceField, self).__init__(*args, **kwargs)

    @property
    def choices(self):
        # Return a list of choices sorted by name, along with an empty choice.
        cache_key = 'storymarket_choice_cache:%s' % self.__class__.__name__
        choices = cache.get(cache_key)
        if choices is None:
            objs = sorted(self._call_api(), key=operator.attrgetter('name'))

            # If there's only a single object, just select it -- don't offer
            # an empty choice. Otherwise, offer an empty.
            if len(objs) == 1:
                empty_choice = []
            else:
                empty_choice = [(u'', u'---------')]
            choices = empty_choice + [(o.id, o.name) for o in objs]
            cache.set(cache_key, choices, CHOICE_CACHE_TIMEOUT)
        return choices

class OrgChoiceField(StorymarketAPIChoiceField):
    def _call_api(self):
        api = storymarket.Storymarket(settings.STORYMARKET_API_KEY)
        return api.orgs.all()

class CategoryChoiceField(StorymarketAPIChoiceField):
    def _call_api(self):
        api = storymarket.Storymarket(settings.STORYMARKET_API_KEY)
        return api.subcategories.all()

class PricingChoiceField(StorymarketAPIChoiceField):
    def _call_api(self):
        api = storymarket.Storymarket(settings.STORYMARKET_API_KEY)
        return api.pricing.all()

class RightsChoiceField(StorymarketAPIChoiceField):
    def _call_api(self):
        api = storymarket.Storymarket(settings.STORYMARKET_API_KEY)
        return api.rights.all()

class StorymarketSyncForm(forms.Form):
    """
    A form allowing the choice of sync options for a given model instance.
    """
    org = OrgChoiceField()
    category = CategoryChoiceField()
    pricing = PricingChoiceField()
    rights = RightsChoiceField()
    tags = forms.CharField()

    def __init__(self, *args, **kwargs):
        api = storymarket.Storymarket(settings.STORYMARKET_API_KEY)
        if 'type' in kwargs:
            self.storymarket_type = kwargs.pop('type')
        else:
            raise TypeError("__init__() missing required keyword argument 'type'")
        initial = kwargs.pop('initial', {})
        instance = kwargs.pop('instance', None)

        # If we've been given an object instance then look up initial data
        # from Storymarket.
        if instance:
            try:
                sync_info = SyncedObject.objects.for_model(instance).get()
            except SyncedObject.DoesNotExist:
                pass
            else:
                initial.update({
                    'org': sync_info.org_id,
                    'category': sync_info.category_id,
                    'pricing': sync_info.pricing_id,
                    'rights': sync_info.rights_id,
                    'tags': sync_info_tags,
                })

        super(StorymarketSyncForm, self).__init__(initial=initial, *args, **kwargs)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os.path

from datetime import datetime

DEBUG = False
DEBUG = True

ADMINS = (
    ('', ''),
)
MANAGERS = ADMINS

DATABASES = {
    'default': {
        'HOST': '127.0.0.1',
        'PORT': '',
        'NAME': '',
        'ENGINE': 'django.db.backends.mysql',
        'USER': '',
        'PASSWORD': '',
        'OPTIONS': {
           'init_command': 'SET storage_engine=MyISAM',
        }
    },
}

ALLOWED_HOSTS =  []
TIME_ZONE = 'America/Chicago'
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
USE_I18N = True
DEFAULT_CHARSET = 'utf-8'
FILE_CHARSET = 'utf-8'
ROOT_URLCONF = 'djforms.core.urls'
MEDIA_ROOT = '/data2/django_projects/djforms/assets/'
MEDIA_URL = '/static/forms/assets/'
UPLOADS_DIR = 'files'
ROOT_DIR = os.path.dirname(__file__)
STATIC_URL = '/djmedia/'
ROOT_URL = '/forms/'
SERVER_URL = ''
API_URL = '{}/{}'.format(SERVER_URL, 'api')
LIVEWHALE_API_URL = 'https://{}'.format(SERVER_URL)
API_KEY = ''
API_PEOPLE_URL=''
AUTH_PROFILE_MODULE = 'core.UserProfile'
SECRET_KEY = ''
HONEYPOT_FIELD_NAME=''
HONEYPOT_VALUE=''
# Character Quest Window
CHARACTER_QUEST_END_MONTH = 5
CHARACTER_QUEST_END_DAY = 1
CHARACTER_QUEST_START_MONTH = 3
CHARACTER_QUEST_START_DAY = 1
# Celebration of Scholars Window
COS_END_MONTH = 3
COS_END_DAY = 31
COS_START_MONTH = 2
COS_START_DAY = 2
# giving constants
GIVING_DEFAULT_CONTACT_FORM = 'GivingDay'
GIVING_DEFAULT_CAMPAIGN = 'giving-day'
GIVING_DAY_START_DATE = datetime.strptime(
    'Jan 1 2017 6:00PM','%b %d %Y %I:%M%p'
)
MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'honeypot.middleware.HoneypotMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)
TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(os.path.dirname(__file__), 'templates'),
            '/data2/django_templates/djkorra/',
            '/data2/django_templates/djcher/',
            '/data2/django_templates/',
        ],
        'OPTIONS': {
            'debug':DEBUG,
            'context_processors': [
                'djtools.context_processors.sitevars',
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.media',
                'django.template.context_processors.request',
                'django.template.context_processors.static',
                'django.contrib.messages.context_processors.messages',
            ],
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader'
            ]
        },
    },
]
INSTALLED_APPS = (
    # django apps
    'django.contrib.admin',
    'django.contrib.admindocs',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.humanize',
    'django.contrib.sessions',
    'django.contrib.sites',
    # third party projects
    #'authority',
    'bootstrapform',
    'django_countries',
    'honeypot',
    'imagekit',
    #'oembed',
    'taggit',
    'userprofile',
    # djforms stuff
    'djforms.admissions',
    'djforms.admissions.admitted',
    'djforms.admissions.visitdays',
    'djforms.alumni.classnotes',
    'djforms.alumni.memory',
    'djforms.athletics.soccer',
    'djforms.biology.genomics',
    'djforms.catering',
    'djforms.characterquest',
    'djforms.communications.metamorphosis',
    'djforms.communications.printrequest',
    'djforms.core',
    'djforms.giving',
    'djforms.languages',
    'djforms.lis.copyprint',
    'djforms.lis.conferences.mathematica',
    'djforms.music.ensembles.choral',
    'djforms.music.theatre.summer_camp',
    'djforms.polisci.iea.proposal',
    'djforms.polisci.iea.registration',
    'djforms.prehealth.committee_letter',
    'djforms.processors',
    'djforms.scholars',
    'djforms.security',
    'djforms.sustainability.green',
    'djforms.writingcurriculum',
    # other inhouse django
    'djtools',
)
COUNTRIES_FIRST = {
    'US': ('United States'),
}
COUNTRIES_FLAG_URL = '/static/forms/img/flags/{code}.gif'
# LDAP Constants
LDAP_SERVER = ''
LDAP_PORT = ''
LDAP_PROTOCOL = ''
LDAP_BASE = ''
LDAP_USER = ''
LDAP_PASS = ''
LDAP_EMAIL_DOMAIN = ''
LDAP_OBJECT_CLASS = ''
LDAP_OBJECT_CLASS_LIST = []
LDAP_GROUPS = {}
LDAP_RETURN = []
LDAP_ID_ATTR=''
LDAP_AUTH_USER_PK = False
# auth backends
AUTHENTICATION_BACKENDS = (
    'djauth.ldapBackend.LDAPBackend',
    'django.contrib.auth.backends.ModelBackend',
)
LOGIN_URL = '{}accounts/login/'.format(ROOT_URL)
LOGOUT_URL = '{}accounts/logout/'.format(ROOT_URL)
LOGIN_REDIRECT_URL = ROOT_URL
USE_X_FORWARDED_HOST = True
SESSION_COOKIE_DOMAIN=''
SESSION_EXPIRE_AT_BROWSER_CLOSE = True
SESSION_COOKIE_NAME =''
SESSION_COOKIE_AGE = 86400
COS_FACULTY = [{}]
# logggin stuff
LOG_FILEPATH = os.path.join(os.path.dirname(__file__), 'logs/')
LOG_FILENAME = LOG_FILEPATH + 'debug.log'
LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {
        'standard': {
            'format' : '[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s',
            'datefmt' : '%Y/%b/%d %H:%M:%S'
        },
        'verbose': {
            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s',
            'datefmt' : '%Y/%b/%d %H:%M:%S'
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'null': {
            'level':'DEBUG',
            'class':'django.utils.log.NullHandler',
        },
        'logfile': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_FILENAME,
            'maxBytes': 50000,
            'backupCount': 2,
            'formatter': 'standard',
        },
        'console':{
            'level':'INFO',
            'class':'logging.StreamHandler',
            'formatter': 'standard'
        },
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'include_html': True,
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'djforms': {
            'handlers':['logfile'],
            'propagate': True,
            'level':'DEBUG',
        },
        'django': {
            'handlers':['console'],
            'propagate': True,
            'level':'WARN',
        },
        'django.db.backends': {
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}
# SMTP settings
EMAIL_HOST = ''
EMAIL_HOST_USER = ''
EMAIL_HOST_PASSWORD = ''
EMAIL_USE_TLS = True
EMAIL_PORT = 587
EMAIL_FAIL_SILENTLY = True
DEFAULT_FROM_EMAIL = ''
SERVER_EMAIL = ''
SERVER_MAIL=''

# email addresses
SUSTAINABILITY_EMAIL_LIST = []
SOCCER_CAMP_TO_LIST = []
MAINTENANCE_MANAGER = ''
CHARACTER_QUEST_TO_LIST = []
COPYPRINT_CARD_REQUEST_EMAIL = ''
COS_DEFAULT_NAME = ''
COS_DEFAULT_EMAIL = ''
ALUMNI_OFFICE_EMAIL = ''
ALUMNI_MEMORY_EMAIL = []
GIVING_DONATIONS_BCC = [
    SERVER_EMAIL,
]
COMMUNICATIONS_METAMORPHOSIS_TO_LIST = []
COMMUNICATIONS_PRINT_REQUEST_EMAIL = ''
# TLE applications
MODERN_LANGUAGES_TLE_APPLICATIONS = []
STUDY_ABROAD_EMAIL = ''
BIOLOGY_GENOMICS = []
POLITICAL_SCIENCE_IEA_EMAIL = []
POLISCI_MODEL_UN_TO_LIST = []
CONTINUING_STUDIES_ENRICHMENT_REGISTRATION_EMAIL = ''
ALUMNI_CLASSNOTES_EMAILS = []
ADMISSIONS_CHINA_EMAIL_LIST = []
ADMISSIONS_ADMITTED_EMAIL_LIST = []
ADMISSIONS_EMAIL = ''
CATERING_TO_LIST = []
LIS_MATHEMATICA_REGISTRATION_EMAIL = []
LIS_PRINT_REQUEST_EMAIL = ''
CHORAL_TRYOUTS_FROM = ''
PREHEALTH_DO = ''
PREHEALTH_MD = ''
WAC_EMAIL = []
SECURITY_PARKING_TICKET_APPEAL_EMAIL = ''
# Authorize.net
GATEWAY_API_LOGIN = ''
GATEWAY_TRANS_KEY = ''
GATEWAY_USE_TEST_MODE = True
GATEWAY_USE_TEST_URL = True
GATEWAY_AUTHORIZE_ONLY = True
GATEWAY_NAME = ''
# TrustCommerce
TC_LOGIN = ''
TC_PASSWORD = ''
#TC_LIVE = True
TC_LIVE = False
TC_AVS = 'n'
TC_AUTH_TYPE = 'sale'
TC_CYCLE = '1m'
TC_OPERATOR = 'DJForms'
# caching
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        #'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
        #'LOCATION': '127.0.0.1:11211',
        #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
        #'LOCATION': '/var/tmp/django_directory_cache',
        #'TIMEOUT': 60*60*24*365,
        #'KEY_PREFIX': 'DJFORMS_',
        #'OPTIONS': {
        #    'MAX_ENTRIES': 100000,
        #    'CULL_FREQUENCY': 4,
        #}
    }
}
<commit_msg>merging production settings constants into defaults<commit_after># -*- coding: utf-8 -*-
import os.path

from datetime import datetime

from djzbar.settings import INFORMIX_EARL as INFORMIX_EARL

DEBUG = False
INFORMIX_DEBUG = ''

ADMINS = (
    ('', ''),
)
MANAGERS = ADMINS

DATABASES = {
    'default': {
        'HOST': '127.0.0.1',
        'PORT': '',
        'NAME': '',
        'ENGINE': 'django.db.backends.mysql',
        'USER': '',
        'PASSWORD': '',
        'OPTIONS': {
           'init_command': 'SET storage_engine=MyISAM',
        }
    },
}

ALLOWED_HOSTS =  []
TIME_ZONE = 'America/Chicago'
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
USE_I18N = True
DEFAULT_CHARSET = 'utf-8'
FILE_CHARSET = 'utf-8'
ROOT_URLCONF = 'djforms.core.urls'
MEDIA_ROOT = '/data2/django_projects/djforms/assets/'
MEDIA_URL = '/static/forms/assets/'
UPLOADS_DIR = 'files'
ROOT_DIR = os.path.dirname(__file__)
STATIC_URL = '/djmedia/'
ROOT_URL = '/forms/'
SERVER_URL = ''
API_URL = '{}/{}'.format(SERVER_URL, 'api')
LIVEWHALE_API_URL = 'https://{}'.format(SERVER_URL)
API_KEY = ''
API_PEOPLE_URL=''
AUTH_PROFILE_MODULE = 'core.UserProfile'
SECRET_KEY = ''
HONEYPOT_FIELD_NAME=''
HONEYPOT_VALUE=''
# Character Quest Window
CHARACTER_QUEST_END_MONTH = 5
CHARACTER_QUEST_END_DAY = 1
CHARACTER_QUEST_START_MONTH = 3
CHARACTER_QUEST_START_DAY = 1
# Celebration of Scholars Window
COS_END_MONTH = 3
COS_END_DAY = 31
COS_START_MONTH = 2
COS_START_DAY = 2
# giving constants
GIVING_DEFAULT_CONTACT_FORM = 'GivingDay'
GIVING_DEFAULT_CAMPAIGN = 'giving-day'
GIVING_DAY_START_DATE = datetime.strptime(
    'Jan 1 2017 6:00PM','%b %d %Y %I:%M%p'
)
MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'honeypot.middleware.HoneypotMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)
TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(os.path.dirname(__file__), 'templates'),
            '/data2/django_templates/djkorra/',
            '/data2/django_templates/djcher/',
            '/data2/django_templates/',
        ],
        'OPTIONS': {
            'debug':DEBUG,
            'context_processors': [
                'djtools.context_processors.sitevars',
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.media',
                'django.template.context_processors.request',
                'django.template.context_processors.static',
                'django.contrib.messages.context_processors.messages',
            ],
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader'
            ]
        },
    },
]
INSTALLED_APPS = (
    # django apps
    'django.contrib.admin',
    'django.contrib.admindocs',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.humanize',
    'django.contrib.sessions',
    'django.contrib.sites',
    # third party projects
    #'authority',
    'bootstrapform',
    'django_countries',
    'honeypot',
    'imagekit',
    #'oembed',
    'taggit',
    'userprofile',
    # djforms stuff
    'djforms.admissions',
    'djforms.admissions.admitted',
    'djforms.admissions.visitdays',
    'djforms.alumni.classnotes',
    'djforms.alumni.memory',
    'djforms.athletics.soccer',
    'djforms.biology.genomics',
    'djforms.catering',
    'djforms.characterquest',
    'djforms.communications.metamorphosis',
    'djforms.communications.printrequest',
    'djforms.core',
    'djforms.giving',
    'djforms.languages',
    #'djforms.lis.conferences.mathematica',
    'djforms.lis.copyprint',
    'djforms.music.ensembles.choral',
    'djforms.music.theatre.summer_camp',
    'djforms.polisci.iea.proposal',
    'djforms.polisci.iea.registration',
    'djforms.prehealth.committee_letter',
    'djforms.president.honorary_degree',
    'djforms.processors',
    'djforms.scholars',
    'djforms.security',
    'djforms.sustainability.green',
    'djforms.writingcurriculum',
    # other inhouse django
    'djtools',
)
# django-countries settings
COUNTRIES_FIRST = {
    'US': ('United States'),
}
COUNTRIES_FLAG_URL = '/static/forms/img/flags/{code}.gif'
# LDAP Constants
LDAP_SERVER = ''
LDAP_PORT = ''
LDAP_PROTOCOL = ''
LDAP_BASE = ''
LDAP_USER = ''
LDAP_PASS = ''
LDAP_EMAIL_DOMAIN = ''
LDAP_OBJECT_CLASS = ''
LDAP_OBJECT_CLASS_LIST = []
LDAP_GROUPS = {}
LDAP_RETURN = []
LDAP_ID_ATTR=''
LDAP_AUTH_USER_PK = False
# auth backends
AUTHENTICATION_BACKENDS = (
    'djauth.ldapBackend.LDAPBackend',
    'django.contrib.auth.backends.ModelBackend',
)
LOGIN_URL = '{}accounts/login/'.format(ROOT_URL)
LOGOUT_URL = '{}accounts/logout/'.format(ROOT_URL)
LOGIN_REDIRECT_URL = ROOT_URL
USE_X_FORWARDED_HOST = True
SESSION_COOKIE_DOMAIN=''
SESSION_EXPIRE_AT_BROWSER_CLOSE = True
SESSION_COOKIE_NAME =''
SESSION_COOKIE_AGE = 86400
COS_FACULTY = [{}]
# logggin stuff
LOG_FILEPATH = os.path.join(os.path.dirname(__file__), 'logs/')
LOG_FILENAME = LOG_FILEPATH + 'debug.log'
LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {
        'standard': {
            'format' : '[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s',
            'datefmt' : '%Y/%b/%d %H:%M:%S'
        },
        'verbose': {
            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s',
            'datefmt' : '%Y/%b/%d %H:%M:%S'
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'null': {
            'level':'DEBUG',
            'class':'django.utils.log.NullHandler',
        },
        'logfile': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_FILENAME,
            'maxBytes': 50000,
            'backupCount': 2,
            'formatter': 'standard',
        },
        'console':{
            'level':'INFO',
            'class':'logging.StreamHandler',
            'formatter': 'standard'
        },
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'include_html': True,
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'djforms': {
            'handlers':['logfile'],
            'propagate': True,
            'level':'DEBUG',
        },
        'djauth': {
            'handlers':['logfile'],
            'propagate': True,
            'level':'DEBUG',
        },
        'django': {
            'handlers':['console'],
            'propagate': True,
            'level':'WARN',
        },
        'django.db.backends': {
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': False,
        },
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}
# SMTP settings
EMAIL_HOST = ''
EMAIL_HOST_USER = ''
EMAIL_HOST_PASSWORD = ''
EMAIL_USE_TLS = True
EMAIL_PORT = 587
EMAIL_FAIL_SILENTLY = True
DEFAULT_FROM_EMAIL = ''
SERVER_EMAIL = ''
SERVER_MAIL=''

# email addresses
SUSTAINABILITY_EMAIL_LIST = []
SOCCER_CAMP_TO_LIST = []
MAINTENANCE_MANAGER = ''
CHARACTER_QUEST_TO_LIST = []
COPYPRINT_CARD_REQUEST_EMAIL = ''
COS_DEFAULT_NAME = ''
COS_DEFAULT_EMAIL = ''
ALUMNI_OFFICE_EMAIL = ''
ALUMNI_MEMORY_EMAIL = []
GIVING_DONATIONS_BCC = [
    SERVER_EMAIL,
]
HONORARY_DEGREE_NOMINATION_EMAIL = ''
COMMUNICATIONS_METAMORPHOSIS_TO_LIST = []
COMMUNICATIONS_PRINT_REQUEST_EMAIL = ''
# TLE applications
MODERN_LANGUAGES_TLE_APPLICATIONS = []
STUDY_ABROAD_EMAIL = ''
BIOLOGY_GENOMICS = []
POLITICAL_SCIENCE_IEA_EMAIL = []
POLISCI_MODEL_UN_TO_LIST = []
CONTINUING_STUDIES_ENRICHMENT_REGISTRATION_EMAIL = ''
ALUMNI_CLASSNOTES_EMAILS = []
ADMISSIONS_CHINA_EMAIL_LIST = []
ADMISSIONS_ADMITTED_EMAIL_LIST = []
ADMISSIONS_EMAIL = ''
CATERING_TO_LIST = []
LIS_MATHEMATICA_REGISTRATION_EMAIL = []
LIS_PRINT_REQUEST_EMAIL = ''
CHORAL_TRYOUTS_FROM = ''
PREHEALTH_DO = ''
PREHEALTH_MD = ''
WAC_EMAIL = []
SECURITY_PARKING_TICKET_APPEAL_EMAIL = ''
# Authorize.net
GATEWAY_API_LOGIN = ''
GATEWAY_TRANS_KEY = ''
GATEWAY_USE_TEST_MODE = True
GATEWAY_USE_TEST_URL = True
GATEWAY_AUTHORIZE_ONLY = True
GATEWAY_NAME = ''
# TrustCommerce
TC_LOGIN = ''
TC_PASSWORD = ''
#TC_LIVE = True
TC_LIVE = False
TC_AVS = 'n'
TC_AUTH_TYPE = 'sale'
TC_CYCLE = '1m'
TC_OPERATOR = 'DJForms'
# caching
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        #'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
        #'LOCATION': '127.0.0.1:11211',
        #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
        #'LOCATION': '/var/tmp/django_directory_cache',
        #'TIMEOUT': 60*60*24*365,
        #'KEY_PREFIX': 'DJFORMS_',
        #'OPTIONS': {
        #    'MAX_ENTRIES': 100000,
        #    'CULL_FREQUENCY': 4,
        #}
    }
}
<|endoftext|>


<commit_before>#!/usr/bin/env python

import logging, sys, argparse

logging.basicConfig(level=0, format="%(asctime)-15s %(name)s %(message)s")

# from https://github.com/kvpradap/dmagellan/blob/chtc/notebooks/chtc/example.ipynb

import pandas as pd

# imports
from dmagellan.blocker.attrequivalence.attr_equiv_blocker import AttrEquivalenceBlocker
from dmagellan.blocker.overlap.overlapblocker import OverlapBlocker
from dmagellan.blocker.blackbox.blackbox_blocker import BlackBoxBlocker
from dmagellan.blocker.rulebased.rule_based_blocker import RuleBasedBlocker
from dmagellan.feature.autofeaturegen import get_features_for_blocking
from dmagellan.feature.extractfeatures import extract_feature_vecs
from dmagellan.feature.autofeaturegen import get_features_for_matching
from dmagellan.matcher.dtmatcher import DTMatcher
from dmagellan.utils.py_utils.utils import concat_df
from dask import delayed
from dask.threaded import get
from distributed import Client
import py_entitymatching as em

parser = argparse.ArgumentParser(description="dmagellan test")
parser.add_argument('client_type', choices=['local', 'sched'],
                    help='use LocalCluster or a scheduler on 127.0.0.1:8786')
parser.add_argument('--block-tables-chunks', metavar='N', type=int, default=4,
                    help='number of chunks to use in ob.block_tables')
parser.add_argument('--extract-feature-vecs-chunks', metavar='N', type=int, default=4,
                    help='number of chunks to use in extract_feature_vecs')
parser.add_argument('--usecols', action='store_const', const=['id', 'title'], default=None,
                    help='load a subset of the columns from the datafile')
args = parser.parse_args()

if args.client_type == 'local':
    logging.info("Using LocalCluster")
    client = Client()
elif args.client_type == 'sched':
    logging.info("Connecting to scheduler at 127.0.0.1:8786")
    client = Client('127.0.0.1:8786')
else:
    assert False, 'should have been caught'

logging.info("Parameters:")
logging.info("block_tables_chunks: %s" % args.block_tables_chunks)
logging.info("extract_feature_vecs_chunks: %s" % args.extract_feature_vecs_chunks)
logging.info("usecols: %s" % args.usecols)

orig_A = pd.read_csv('./data/citeseer_nonans.csv', usecols=args.usecols)
orig_B = pd.read_csv('./data/dblp_nonans.csv', usecols=args.usecols)
logging.debug('loaded full data')

# sample datasets
A = pd.read_csv('./data/sample_citeseer.csv')
B = pd.read_csv('./data/sample_dblp.csv')
logging.debug('loaded sample data')

# blocking
ob = OverlapBlocker()
logging.debug("starting ob.block_tables()")
C = ob.block_tables(orig_A, orig_B, 'id', 'id', 'title', 'title',
                    # increasing the chunks bloats the memory usage of the client
                    # and also how long it takes before it creates tasks
                    # but it's not a problem if I set usecols when loading orig_A
                    # and orig_B, or if I use lz4 compression
                    overlap_size=6, nltable_chunks=args.block_tables_chunks, nrtable_chunks=args.block_tables_chunks,
                    # I set compute to True to see which part of the workflow was
                    # taking a long time
                    scheduler=client.get, compute=True,
                    rem_stop_words=True
                   )
logging.debug('finished ob.block_tables()')
logging.debug('len(C) = %d' % len(C))

L = pd.read_csv('./data/sample_labeled_data.csv')
logging.debug('loaded sample labeled data')

F = em.get_features_for_matching(orig_A, orig_B)
logging.debug('ran em.get_features_for_matching()')

# Convert L into feature vectors using updated F
# must use orig_A and orig_B; I get a KeyError when I try to use the sample data
# requires workers with > 1GB of memory, otherwise (sometimes) does not finish
logging.debug("starting H=extract_feature_vecs()")
H = extract_feature_vecs(L, orig_A, orig_B,
                         '_id', 'l_id', 'r_id', 'id', 'id',
                          feature_table=F,
                    # increasing the chunks bloats the memory usage of the client
                    # and also how long it takes before it creates tasks
                          attrs_after='label', nchunks=args.extract_feature_vecs_chunks,
                          show_progress=False,
                          # we have to compute here else mlmatcher will
                          # complain that "Input table is not of type DataFrame"
                          compute=True,
                         scheduler=client.get)
logging.debug('finished H=extract_feature_vecs()')
logging.debug("len(H) = %d" % len(H))

sys.exit(0)  # EXIT ------------------------

# Instantiate the matcher to evaluate.
dt = DTMatcher(name='DecisionTree', random_state=0)

dt.fit(table=H,
       exclude_attrs=['_id', 'l_id', 'r_id', 'label'],
       target_attr='label')
logging.debug('ran dt.fit()')

# Convert J into a set of feature vectors using F
I = extract_feature_vecs(C, A, B,
                         '_id', 'l_id', 'r_id', 'id', 'id',
                            nchunks=args.extract_feature_vecs_chunks,
                            feature_table=F,
                            show_progress=False,
                            compute=False)
logging.debug('ran I=extract_feature_vecs()')

predictions = dt.predict(table=I, exclude_attrs=['_id', 'l_id', 'r_id'],
              append=True, target_attr='predicted', inplace=False,
                        nchunks=4, scheduler=client.get, compute=False)
logging.debug('ran dt.predict()')

# Can't visualize - no graphviz
#predictions.visualize()

p = predictions.compute(get=client.get)
logging.debug('ran predictions.compute()')
print(p)
logging.debug('done')

<commit_msg>Remove early exit<commit_after>#!/usr/bin/env python

import logging, sys, argparse

logging.basicConfig(level=0, format="%(asctime)-15s %(name)s %(message)s")

# from https://github.com/kvpradap/dmagellan/blob/chtc/notebooks/chtc/example.ipynb

import pandas as pd

# imports
from dmagellan.blocker.attrequivalence.attr_equiv_blocker import AttrEquivalenceBlocker
from dmagellan.blocker.overlap.overlapblocker import OverlapBlocker
from dmagellan.blocker.blackbox.blackbox_blocker import BlackBoxBlocker
from dmagellan.blocker.rulebased.rule_based_blocker import RuleBasedBlocker
from dmagellan.feature.autofeaturegen import get_features_for_blocking
from dmagellan.feature.extractfeatures import extract_feature_vecs
from dmagellan.feature.autofeaturegen import get_features_for_matching
from dmagellan.matcher.dtmatcher import DTMatcher
from dmagellan.utils.py_utils.utils import concat_df
from dask import delayed
from dask.threaded import get
from distributed import Client
import py_entitymatching as em

parser = argparse.ArgumentParser(description="dmagellan test")
parser.add_argument('client_type', choices=['local', 'sched'],
                    help='use LocalCluster or a scheduler on 127.0.0.1:8786')
parser.add_argument('--block-tables-chunks', metavar='N', type=int, default=4,
                    help='number of chunks to use in ob.block_tables')
parser.add_argument('--extract-feature-vecs-chunks', metavar='N', type=int, default=4,
                    help='number of chunks to use in extract_feature_vecs')
parser.add_argument('--usecols', action='store_const', const=['id', 'title'], default=None,
                    help='load a subset of the columns from the datafile')
args = parser.parse_args()

if args.client_type == 'local':
    logging.info("Using LocalCluster")
    client = Client()
elif args.client_type == 'sched':
    logging.info("Connecting to scheduler at 127.0.0.1:8786")
    client = Client('127.0.0.1:8786')
else:
    assert False, 'should have been caught'

logging.info("Parameters:")
logging.info("block_tables_chunks: %s" % args.block_tables_chunks)
logging.info("extract_feature_vecs_chunks: %s" % args.extract_feature_vecs_chunks)
logging.info("usecols: %s" % args.usecols)

orig_A = pd.read_csv('./data/citeseer_nonans.csv', usecols=args.usecols)
orig_B = pd.read_csv('./data/dblp_nonans.csv', usecols=args.usecols)
logging.debug('loaded full data')

# sample datasets
A = pd.read_csv('./data/sample_citeseer.csv')
B = pd.read_csv('./data/sample_dblp.csv')
logging.debug('loaded sample data')

# blocking
ob = OverlapBlocker()
logging.debug("starting ob.block_tables()")
C = ob.block_tables(orig_A, orig_B, 'id', 'id', 'title', 'title',
                    # increasing the chunks bloats the memory usage of the client
                    # and also how long it takes before it creates tasks
                    # but it's not a problem if I set usecols when loading orig_A
                    # and orig_B, or if I use lz4 compression
                    overlap_size=6, nltable_chunks=args.block_tables_chunks, nrtable_chunks=args.block_tables_chunks,
                    # I set compute to True to see which part of the workflow was
                    # taking a long time
                    scheduler=client.get, compute=True,
                    rem_stop_words=True
                   )
logging.debug('finished ob.block_tables()')
logging.debug('len(C) = %d' % len(C))

L = pd.read_csv('./data/sample_labeled_data.csv')
logging.debug('loaded sample labeled data')

F = em.get_features_for_matching(orig_A, orig_B)
logging.debug('ran em.get_features_for_matching()')

# Convert L into feature vectors using updated F
# must use orig_A and orig_B; I get a KeyError when I try to use the sample data
# requires workers with > 1GB of memory, otherwise (sometimes) does not finish
logging.debug("starting H=extract_feature_vecs()")
H = extract_feature_vecs(L, orig_A, orig_B,
                         '_id', 'l_id', 'r_id', 'id', 'id',
                          feature_table=F,
                    # increasing the chunks bloats the memory usage of the client
                    # and also how long it takes before it creates tasks
                          attrs_after='label', nchunks=args.extract_feature_vecs_chunks,
                          show_progress=False,
                          # we have to compute here else mlmatcher will
                          # complain that "Input table is not of type DataFrame"
                          compute=True,
                         scheduler=client.get)
logging.debug('finished H=extract_feature_vecs()')
logging.debug("len(H) = %d" % len(H))

# Instantiate the matcher to evaluate.
dt = DTMatcher(name='DecisionTree', random_state=0)

dt.fit(table=H,
       exclude_attrs=['_id', 'l_id', 'r_id', 'label'],
       target_attr='label')
logging.debug('ran dt.fit()')

# Convert J into a set of feature vectors using F
I = extract_feature_vecs(C, A, B,
                         '_id', 'l_id', 'r_id', 'id', 'id',
                            nchunks=args.extract_feature_vecs_chunks,
                            feature_table=F,
                            show_progress=False,
                            compute=False)
logging.debug('ran I=extract_feature_vecs()')

predictions = dt.predict(table=I, exclude_attrs=['_id', 'l_id', 'r_id'],
              append=True, target_attr='predicted', inplace=False,
                        nchunks=4, scheduler=client.get, compute=False)
logging.debug('ran dt.predict()')

# Can't visualize - no graphviz
#predictions.visualize()

p = predictions.compute(get=client.get)
logging.debug('ran predictions.compute()')
print(p)
logging.debug('done')

<|endoftext|>


<commit_before>#!/usr/bin/python3 python3

# This is a heavily modified version of the jupyterhub cull-idle script, retrieved October 31st 2016 from
# https://github.com/jupyterhub/jupyterhub/blob/master/examples/cull-idle/cull_idle_servers.py
# Due to environment variable bugs we have changed the mechanism by which we attain the jupyterhub
# API key.
# Author: Eli <eli@zagaran.com>

import datetime
import json
import os
import sys
import boto3
import logging

sys.path.insert(1, '/etc/jupyterhub')
from models import Server

from dateutil.parser import parse as parse_date

from botocore.exceptions import ClientError, WaiterError
from concurrent.futures import ThreadPoolExecutor
from tornado.gen import coroutine, sleep
from tornado.log import app_log
from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPError
from tornado.ioloop import IOLoop, PeriodicCallback
from tornado.options import define, options, parse_command_line

with open("/etc/jupyterhub/server_config.json", "r") as f:
    SERVER_PARAMS = json.load(f) # load local server parameters

app_log.setLevel(logging.INFO)
logging.getLogger('boto3').setLevel(logging.ERROR)
logging.getLogger('botocore').setLevel(logging.ERROR)

thread_pool = ThreadPoolExecutor(10)

@coroutine
def retry(function, *args, **kwargs):
    """ Retries a function up to max_retries, waiting `timeout` seconds between tries.
        This function is designed to retry both boto3 and fabric calls.  In the
        case of boto3, it is necessary because sometimes aws calls return too
        early and a resource needed by the next call is not yet available. """
    max_retries = kwargs.pop("max_retries", 20)
    timeout = kwargs.pop("timeout", 0.25)
    for attempt in range(max_retries):
        try:
            ret = yield thread_pool.submit(function, *args, **kwargs)
            return ret
        except (ClientError, WaiterError) as e:
            app_log.warn("encountered %s, waiting for %s seconds before retrying..." % (type(e), timeout) )
            yield sleep(timeout)
     else:
         print("Failure in %s with args %s and kwargs %s" % (function.__name__, args, kwargs))
         #raise e

@coroutine
def manually_kill_server(user_name):
    # Get our AWS server db's instance for the user
    try:
        server = Server.get_server(user_name)
        app_log.debug("Checking server for %s manually..." % user_name)
    except Server.DoesNotExist:
        # it is not necessarily the case that a server will exist, we return early if that is the case.
        app_log.warn("There is no matching, allocated server for user %s" % user_name)
        return
    # get server instance information
    resource = yield retry(boto3.resource, "ec2", region_name=SERVER_PARAMS["REGION"])
    instance = yield retry(resource.Instance, server.server_id)
    # instance object is lazy, run this to get full info...
    yield retry(instance.load)

    #stop server if state is running (possible states are stopped, stopping, pending, shutting-down, terminated, and running)
    if instance.state["Name"] == "running":
        retry(instance.stop)
        app_log.info("manually killed server for user %s" % user_name)
    else:
        app_log.debug("server state for user %s is %s, no action taken" % (user_name, instance.state["Name"]))

@coroutine
def cull_idle(url, api_token, timeout):
    #last valid activity timestame
    cull_limit = datetime.datetime.utcnow() - datetime.timedelta(seconds=timeout)

    #get user list
    hub_api_authorization_header = { 'Authorization': 'token %s' % api_token}
    users_request = HTTPRequest(url=url + '/users', headers=hub_api_authorization_header )

    #run request tornado-asynchronously, extract user list (contains more information)
    resp = yield AsyncHTTPClient().fetch(users_request)
    all_users = json.loads(resp.body.decode('utf8', 'replace'))

    #build a bunch of (asynchronous) HTTP request futures...
    stop_notebook_futures = []
    servers_to_check = []
    dont_cull_these = set()
    for user in all_users:

        #extract last activity time, determine cullability of the server.
        last_activity = parse_date(user['last_activity'])
        should_cull = last_activity.replace(tzinfo=None)  < cull_limit.replace(tzinfo=None)
        user_name = user['name']
        app_log.debug("checking %s, last activity: %s, server: %s" % (user_name, last_activity, user['server']) )

        if not should_cull:
            dont_cull_these.add(user_name)

        #server should be culled:
        if user['server'] and should_cull:
            app_log.info("Culling %s (inactive since %s)", user_name, last_activity)
            stop_user_request = HTTPRequest(url=url + '/users/%s/server' % user_name,
                                            method='DELETE',
                                            headers=hub_api_authorization_header )
            stop_notebook_futures.append( (user_name, AsyncHTTPClient().fetch(stop_user_request)) )

        #Server status is None, which means actual status needs to be checked.
        if not user['server'] and should_cull:
            servers_to_check.append(user_name)

        #server should not be culled, just a log statement
        if user['server'] and not should_cull:
            app_log.info("Not culling %s (active since %s)", user['name'], last_activity)

    # Cull notebooks using normal API.
    for (user_name, cull_request) in stop_notebook_futures:
        try:
            yield cull_request #this line actually runs the api call to kill a server
        except HTTPError:
            #Due to a bug in Jupyterhub
            app_log.error("Something went wrong culling %s, will be manually killing it.", user_name)
            servers_to_check.append( user_name )
            continue
        app_log.info("Finished culling %s", user_name)

    for user_name in servers_to_check:
        if user_name not in dont_cull_these:
            yield manually_kill_server(user_name)

if __name__ == '__main__':
    define('url', default=os.environ.get('JUPYTERHUB_API_URL'), help="The JupyterHub API URL")
    define('timeout', default=SERVER_PARAMS["JUPYTER_NOTEBOOK_TIMEOUT"], help="The idle timeout (in seconds)")
    define('cull_every', default=300, help="The interval (in seconds) for checking for idle servers to cull")

    parse_command_line()
    if not options.cull_every:
        options.cull_every = options.timeout // 2

    # we were having significant issues with environment variables, we will just read from a file
    # api_token = os.environ['JUPYTERHUB_API_TOKEN']
    with open("/etc/jupyterhub/api_token.txt", 'r') as f:
        # for culler script to work without modification we need to set this value in the machine environment.
        api_token = f.read().strip()

    loop = IOLoop.current()
    cull = lambda: cull_idle(options.url, api_token, options.timeout)
    # run once before scheduling periodic call
    loop.run_sync(cull)
    # schedule periodic cull
    pc = PeriodicCallback(cull, 1e3 * options.cull_every)
    pc.start()
    try:
        loop.start()
    except KeyboardInterrupt:
        pass
<commit_msg>add space tothe cull script, otherwise the script will not work<commit_after>#!/usr/bin/python3 python3

# This is a heavily modified version of the jupyterhub cull-idle script, retrieved October 31st 2016 from
# https://github.com/jupyterhub/jupyterhub/blob/master/examples/cull-idle/cull_idle_servers.py
# Due to environment variable bugs we have changed the mechanism by which we attain the jupyterhub
# API key.
# Author: Eli <eli@zagaran.com>

import datetime
import json
import os
import sys
import boto3
import logging

sys.path.insert(1, '/etc/jupyterhub')
from models import Server

from dateutil.parser import parse as parse_date

from botocore.exceptions import ClientError, WaiterError
from concurrent.futures import ThreadPoolExecutor
from tornado.gen import coroutine, sleep
from tornado.log import app_log
from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPError
from tornado.ioloop import IOLoop, PeriodicCallback
from tornado.options import define, options, parse_command_line

with open("/etc/jupyterhub/server_config.json", "r") as f:
    SERVER_PARAMS = json.load(f) # load local server parameters

app_log.setLevel(logging.INFO)
logging.getLogger('boto3').setLevel(logging.ERROR)
logging.getLogger('botocore').setLevel(logging.ERROR)

thread_pool = ThreadPoolExecutor(10)

@coroutine
def retry(function, *args, **kwargs):
    """ Retries a function up to max_retries, waiting `timeout` seconds between tries.
        This function is designed to retry both boto3 and fabric calls.  In the
        case of boto3, it is necessary because sometimes aws calls return too
        early and a resource needed by the next call is not yet available. """
    max_retries = kwargs.pop("max_retries", 20)
    timeout = kwargs.pop("timeout", 0.25)
    for attempt in range(max_retries):
        try:
            ret = yield thread_pool.submit(function, *args, **kwargs)
            return ret
        except (ClientError, WaiterError) as e:
            app_log.warn("encountered %s, waiting for %s seconds before retrying..." % (type(e), timeout) )
            yield sleep(timeout)
    else:
         print("Failure in %s with args %s and kwargs %s" % (function.__name__, args, kwargs))
         #raise e

@coroutine
def manually_kill_server(user_name):
    # Get our AWS server db's instance for the user
    try:
        server = Server.get_server(user_name)
        app_log.debug("Checking server for %s manually..." % user_name)
    except Server.DoesNotExist:
        # it is not necessarily the case that a server will exist, we return early if that is the case.
        app_log.warn("There is no matching, allocated server for user %s" % user_name)
        return
    # get server instance information
    resource = yield retry(boto3.resource, "ec2", region_name=SERVER_PARAMS["REGION"])
    instance = yield retry(resource.Instance, server.server_id)
    # instance object is lazy, run this to get full info...
    yield retry(instance.load)

    #stop server if state is running (possible states are stopped, stopping, pending, shutting-down, terminated, and running)
    if instance.state["Name"] == "running":
        retry(instance.stop)
        app_log.info("manually killed server for user %s" % user_name)
    else:
        app_log.debug("server state for user %s is %s, no action taken" % (user_name, instance.state["Name"]))

@coroutine
def cull_idle(url, api_token, timeout):
    #last valid activity timestame
    cull_limit = datetime.datetime.utcnow() - datetime.timedelta(seconds=timeout)

    #get user list
    hub_api_authorization_header = { 'Authorization': 'token %s' % api_token}
    users_request = HTTPRequest(url=url + '/users', headers=hub_api_authorization_header )

    #run request tornado-asynchronously, extract user list (contains more information)
    resp = yield AsyncHTTPClient().fetch(users_request)
    all_users = json.loads(resp.body.decode('utf8', 'replace'))

    #build a bunch of (asynchronous) HTTP request futures...
    stop_notebook_futures = []
    servers_to_check = []
    dont_cull_these = set()
    for user in all_users:

        #extract last activity time, determine cullability of the server.
        last_activity = parse_date(user['last_activity'])
        should_cull = last_activity.replace(tzinfo=None)  < cull_limit.replace(tzinfo=None)
        user_name = user['name']
        app_log.debug("checking %s, last activity: %s, server: %s" % (user_name, last_activity, user['server']) )

        if not should_cull:
            dont_cull_these.add(user_name)

        #server should be culled:
        if user['server'] and should_cull:
            app_log.info("Culling %s (inactive since %s)", user_name, last_activity)
            stop_user_request = HTTPRequest(url=url + '/users/%s/server' % user_name,
                                            method='DELETE',
                                            headers=hub_api_authorization_header )
            stop_notebook_futures.append( (user_name, AsyncHTTPClient().fetch(stop_user_request)) )

        #Server status is None, which means actual status needs to be checked.
        if not user['server'] and should_cull:
            servers_to_check.append(user_name)

        #server should not be culled, just a log statement
        if user['server'] and not should_cull:
            app_log.info("Not culling %s (active since %s)", user['name'], last_activity)

    # Cull notebooks using normal API.
    for (user_name, cull_request) in stop_notebook_futures:
        try:
            yield cull_request #this line actually runs the api call to kill a server
        except HTTPError:
            #Due to a bug in Jupyterhub
            app_log.error("Something went wrong culling %s, will be manually killing it.", user_name)
            servers_to_check.append( user_name )
            continue
        app_log.info("Finished culling %s", user_name)

    for user_name in servers_to_check:
        if user_name not in dont_cull_these:
            yield manually_kill_server(user_name)

if __name__ == '__main__':
    define('url', default=os.environ.get('JUPYTERHUB_API_URL'), help="The JupyterHub API URL")
    define('timeout', default=SERVER_PARAMS["JUPYTER_NOTEBOOK_TIMEOUT"], help="The idle timeout (in seconds)")
    define('cull_every', default=300, help="The interval (in seconds) for checking for idle servers to cull")

    parse_command_line()
    if not options.cull_every:
        options.cull_every = options.timeout // 2

    # we were having significant issues with environment variables, we will just read from a file
    # api_token = os.environ['JUPYTERHUB_API_TOKEN']
    with open("/etc/jupyterhub/api_token.txt", 'r') as f:
        # for culler script to work without modification we need to set this value in the machine environment.
        api_token = f.read().strip()

    loop = IOLoop.current()
    cull = lambda: cull_idle(options.url, api_token, options.timeout)
    # run once before scheduling periodic call
    loop.run_sync(cull)
    # schedule periodic cull
    pc = PeriodicCallback(cull, 1e3 * options.cull_every)
    pc.start()
    try:
        loop.start()
    except KeyboardInterrupt:
        pass
<|endoftext|>