<commit_before>import json
import logging
from .forms import StaffingForm
from .models import FireStation, Staffing, FireDepartment
from firecares.weather.models import DepartmentWarnings
from django.core.serializers.json import DjangoJSONEncoder
from django.contrib.gis import geos
from tastypie import fields
from tastypie.authentication import SessionAuthentication, ApiKeyAuthentication, MultiAuthentication, Authentication
from tastypie.authorization import DjangoAuthorization
from tastypie.cache import SimpleCache
from tastypie.constants import ALL
from tastypie.contrib.gis.resources import ModelResource
from tastypie.exceptions import Unauthorized, TastypieError
from tastypie.serializers import Serializer
from tastypie.validation import FormValidation
from guardian.core import ObjectPermissionChecker
from django.utils import timezone

logger = logging.getLogger(__name__)

# Note this is from one of my other projects, not sure it is actually needed or not.
class SessionAuth(SessionAuthentication):
    """
    This is a hack to fix a bug which returns occasional TypeErrors returned from SessionAuthentication.

    About:
    Every now and then the super class' get_identifier returns a TypeError (getattr(): attribute name must be string).
    It seems that the logic that returns the string used for the username sometimes returns None.

    """
    def get_identifier(self, request):
        """
        Provides a unique string identifier for the requestor.

        This implementation returns the user's username.
        """
        try:
            return super(SessionAuth, self).get_identifier(request)
        except TypeError:
            return getattr(request.user, 'username')

class PrettyJSONSerializer(Serializer):
    json_indent = 2

    def to_json(self, data, options=None):
        options = options or {}
        data = self.to_simple(data, options)
        return json.dumps(data, cls=DjangoJSONEncoder,
                          sort_keys=True, ensure_ascii=False, indent=self.json_indent)

class JSONDefaultModelResourceMixin(object):
    def determine_format(self, request):
        return 'application/json' if not request.GET.get('format') else super(JSONDefaultModelResourceMixin, self).determine_format(request)

class PropertyResolutionFailedException(TastypieError):
    def __init__(self, property_path, class_name, *args, **kwargs):
        msg = '"{}" does not resolve for class type {}'.format(property_path, class_name)
        super(PropertyResolutionFailedException, self).__init__(msg, *args, **kwargs)

# Pulled/modified from https://gist.github.com/7wonders/6557760
class GuardianAuthorization(DjangoAuthorization):
    """
    :create_permission_code:
        the permission code that signifies the user can create one of these objects
    :view_permission_code:
        the permission code that signifies the user can view the detail
    :update_permission_code:
        the permission code that signifies the user can update one of these objects
    :remove_permission_code:
        the permission code that signifies the user can remove one of these objects
    :kwargs:
        other permission codes
        class Something(models.Model):
            name = models.CharField()
        class SomethingResource(ModelResource):
            class Meta:
                queryset = Something.objects.all()
                authorization = GuardianAuthorization(
                    delegate_to_property = None,  # property path (dot notation) to delegate authorization to
                    view_permission_code = 'can_view',  # empty view_permission_code allows anonymous users to view
                    create_permission_code = 'can_create',
                    update_permission_code = 'can_update',
                    delete_permission_code = 'can_delete'
                    )
    """

    def __init__(self, *args, **kwargs):
        # Allow for authorization to be delegated through a property on the object
        self.parent_property = kwargs.pop('delegate_to_property', None)
        self.view_permission_code = kwargs.pop("view_permission_code", 'can_view')
        self.create_permission_code = kwargs.pop("create_permission_code", 'can_create')
        self.update_permission_code = kwargs.pop("update_permission_code", 'can_update')
        self.delete_permission_code = kwargs.pop("delete_permission_code", 'can_delete')
        super(GuardianAuthorization, self).__init__(**kwargs)

    def resolve_property(self, obj, property_path):
        path = property_path.split('.')
        ret = obj
        for p in path:
            if hasattr(ret, p):
                ret = getattr(ret, p)
            else:
                raise PropertyResolutionFailedException(property_path, obj.__class__)
        return ret

    def generic_base_check(self, object_list, bundle):
        """
            Returns False if either:
                a) if the `object_list.model` doesn't have a `_meta` attribute
                b) the `bundle.request` object doesn have a `user` attribute
        """
        klass = self.base_checks(bundle.request, object_list.model)
        if klass is False:
            raise Unauthorized("You are not allowed to access that resource.")
        return True

    def generic_item_check(self, object_list, bundle, permission):
        if bundle.request.user.is_superuser:
            return True

        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")

        checker = ObjectPermissionChecker(bundle.request.user)
        if self.parent_property:
            obj = self.resolve_property(bundle.obj, self.parent_property)
        else:
            obj = bundle.obj

        # When the property resolves to None, prevent access by default
        if obj is None or not checker.has_perm(permission, obj):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

    def generic_list_check(self, object_list, bundle, permission):
        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")
        user = bundle.request.user
        return [i for i in object_list if permission is None or user.has_perm(permission, i) or user.is_superuser]

    def generic_post_check(self, object_list, bundle, permission):
        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")
        user = bundle.request.user
        if not user.has_perm(permission):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

    # List Checks
    def create_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.create_permission_code)

    def read_list(self, object_list, bundle):
        if not self.view_permission_code:
            return object_list

        return self.generic_list_check(object_list, bundle,
                                       self.view_permission_code)

    def update_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.update_permission_code)

    def delete_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.delete_permission_code)

    # Item Checks
    def create_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.create_permission_code)

    def read_detail(self, object_list, bundle):
        if not self.view_permission_code:
            return True
        return self.generic_item_check(object_list, bundle,
                                       self.view_permission_code)

    def update_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.update_permission_code)

    def delete_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.delete_permission_code)

class FireDepartmentResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Fire Department API.
    """

    class Meta:
        readonly_fields = ['owned_tracts_geom']
        resource_name = 'fire-departments'
        queryset = FireDepartment.objects.defer('owned_tracts_geom').filter(archived=False)
        authorization = GuardianAuthorization(view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='admin_firedepartment')
        authentication = MultiAuthentication(SessionAuthentication(), ApiKeyAuthentication())
        cache = SimpleCache()
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get', 'put']
        filtering = {'state': ALL, 'featured': ALL}
        serializer = PrettyJSONSerializer()
        limit = 120
        max_limit = 2000

    def __init__(self, **kwargs):
        super(FireDepartmentResource, self).__init__(**kwargs)
        for f in getattr(self.Meta, 'readonly_fields', []):
            self.fields[f].readonly = True

    def hydrate_geom(self, bundle):
        geom = bundle.data.get('geom')
        boundary = geos.GEOSGeometry(json.dumps(geom))
        if type(boundary) is geos.Polygon:
            bundle.data['geom'] = json.loads(geos.MultiPolygon(boundary).json)
        return bundle

    def dehydrate(self, bundle):
        """
        remove fields not requested

        note: this is called for every object, so works for both detail and
              list requests
        """
        only_fields = bundle.request.GET.get('fields')
        debug_fields = bundle.request.GET.get('fields_debug', False)
        if only_fields:
            only_fields = only_fields.split(',')
            fields_to_remove = [field for field in bundle.data.keys()
                                if field not in only_fields]
            for field in fields_to_remove:
                del bundle.data[field]
            if debug_fields:
                bundle.data['_fields_selected'] = [field for field in only_fields
                                                   if field in bundle.data.keys()]
                bundle.data['_fields_removed'] = fields_to_remove
        return bundle

class FireStationResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Fire Station API.
    """

    department = fields.ForeignKey(FireDepartmentResource, 'department', null=True)

    class Meta:
        resource_name = 'firestations'

        queryset = FireStation.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='admin_firedepartment')
        authentication = MultiAuthentication(Authentication(), SessionAuthentication(), ApiKeyAuthentication())
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get', 'put']
        filtering = {'department': ('exact',), 'state': ('exact',), 'id': ('exact',)}
        excludes = ['addressbuildingname', 'complex_id', 'data_security', 'distribution_policy', 'fcode', 'foot_id',
                    'ftype', 'globalid', 'gnis_id', 'islandmark', 'loaddate', 'objectid', 'permanent_identifier',
                    'pointlocationtype', 'source_datadesc', 'source_datasetid', 'source_featureid',
                    'source_originator', 'admintype'
                    ]
        serializer = PrettyJSONSerializer()
        limit = 120

class StaffingResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The ResponseCapability API.
    """

    firestation = fields.ForeignKey(FireStationResource, 'firestation')

    class Meta:
        resource_name = 'staffing'
        queryset = Staffing.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='firestation.department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='change_firedepartment')
        authentication = MultiAuthentication(SessionAuthentication(), ApiKeyAuthentication())
        filtering = {'firestation': ALL}
        validation = FormValidation(form_class=StaffingForm)
        list_allowed_methods = ['get', 'post']
        detail_allowed_methods = ['get', 'put', 'delete']
        serializer = PrettyJSONSerializer()
        always_return_data = True

class WeatherWarningResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Weather API mege with department id.
    """
    department = fields.ForeignKey(FireDepartmentResource, 'department', null=True)

    class Meta:
        resource_name = 'weather-warning'
        expired = timezone.now()
        # production when Task is running
        queryset = DepartmentWarnings.objects.filter(expiredate__gte=expired)
        # only show all warnings #good for testing Warnings
        # queryset = DepartmentWarnings.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='change_firedepartment')
        filtering = {'department': ('exact',), 'state': ('exact',), 'id': ('exact',)}
        list_allowed_methods = ['get', 'post']
        detail_allowed_methods = ['get']
        serializer = PrettyJSONSerializer()
        always_return_data = True
<commit_msg>limit database cache for department warning request<commit_after>import json
import logging
from .forms import StaffingForm
from .models import FireStation, Staffing, FireDepartment
from firecares.weather.models import DepartmentWarnings
from django.core.serializers.json import DjangoJSONEncoder
from django.contrib.gis import geos
from tastypie import fields
from tastypie.authentication import SessionAuthentication, ApiKeyAuthentication, MultiAuthentication, Authentication
from tastypie.authorization import DjangoAuthorization
from tastypie.cache import SimpleCache
from tastypie.constants import ALL
from tastypie.contrib.gis.resources import ModelResource
from tastypie.exceptions import Unauthorized, TastypieError
from tastypie.serializers import Serializer
from tastypie.validation import FormValidation
from guardian.core import ObjectPermissionChecker
from django.utils import timezone

logger = logging.getLogger(__name__)

# Note this is from one of my other projects, not sure it is actually needed or not.
class SessionAuth(SessionAuthentication):
    """
    This is a hack to fix a bug which returns occasional TypeErrors returned from SessionAuthentication.

    About:
    Every now and then the super class' get_identifier returns a TypeError (getattr(): attribute name must be string).
    It seems that the logic that returns the string used for the username sometimes returns None.

    """
    def get_identifier(self, request):
        """
        Provides a unique string identifier for the requestor.

        This implementation returns the user's username.
        """
        try:
            return super(SessionAuth, self).get_identifier(request)
        except TypeError:
            return getattr(request.user, 'username')

class PrettyJSONSerializer(Serializer):
    json_indent = 2

    def to_json(self, data, options=None):
        options = options or {}
        data = self.to_simple(data, options)
        return json.dumps(data, cls=DjangoJSONEncoder,
                          sort_keys=True, ensure_ascii=False, indent=self.json_indent)

class JSONDefaultModelResourceMixin(object):
    def determine_format(self, request):
        return 'application/json' if not request.GET.get('format') else super(JSONDefaultModelResourceMixin, self).determine_format(request)

class PropertyResolutionFailedException(TastypieError):
    def __init__(self, property_path, class_name, *args, **kwargs):
        msg = '"{}" does not resolve for class type {}'.format(property_path, class_name)
        super(PropertyResolutionFailedException, self).__init__(msg, *args, **kwargs)

# Pulled/modified from https://gist.github.com/7wonders/6557760
class GuardianAuthorization(DjangoAuthorization):
    """
    :create_permission_code:
        the permission code that signifies the user can create one of these objects
    :view_permission_code:
        the permission code that signifies the user can view the detail
    :update_permission_code:
        the permission code that signifies the user can update one of these objects
    :remove_permission_code:
        the permission code that signifies the user can remove one of these objects
    :kwargs:
        other permission codes
        class Something(models.Model):
            name = models.CharField()
        class SomethingResource(ModelResource):
            class Meta:
                queryset = Something.objects.all()
                authorization = GuardianAuthorization(
                    delegate_to_property = None,  # property path (dot notation) to delegate authorization to
                    view_permission_code = 'can_view',  # empty view_permission_code allows anonymous users to view
                    create_permission_code = 'can_create',
                    update_permission_code = 'can_update',
                    delete_permission_code = 'can_delete'
                    )
    """

    def __init__(self, *args, **kwargs):
        # Allow for authorization to be delegated through a property on the object
        self.parent_property = kwargs.pop('delegate_to_property', None)
        self.view_permission_code = kwargs.pop("view_permission_code", 'can_view')
        self.create_permission_code = kwargs.pop("create_permission_code", 'can_create')
        self.update_permission_code = kwargs.pop("update_permission_code", 'can_update')
        self.delete_permission_code = kwargs.pop("delete_permission_code", 'can_delete')
        super(GuardianAuthorization, self).__init__(**kwargs)

    def resolve_property(self, obj, property_path):
        path = property_path.split('.')
        ret = obj
        for p in path:
            if hasattr(ret, p):
                ret = getattr(ret, p)
            else:
                raise PropertyResolutionFailedException(property_path, obj.__class__)
        return ret

    def generic_base_check(self, object_list, bundle):
        """
            Returns False if either:
                a) if the `object_list.model` doesn't have a `_meta` attribute
                b) the `bundle.request` object doesn have a `user` attribute
        """
        klass = self.base_checks(bundle.request, object_list.model)
        if klass is False:
            raise Unauthorized("You are not allowed to access that resource.")
        return True

    def generic_item_check(self, object_list, bundle, permission):
        if bundle.request.user.is_superuser:
            return True

        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")

        checker = ObjectPermissionChecker(bundle.request.user)
        if self.parent_property:
            obj = self.resolve_property(bundle.obj, self.parent_property)
        else:
            obj = bundle.obj

        # When the property resolves to None, prevent access by default
        if obj is None or not checker.has_perm(permission, obj):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

    def generic_list_check(self, object_list, bundle, permission):
        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")
        user = bundle.request.user
        return [i for i in object_list if permission is None or user.has_perm(permission, i) or user.is_superuser]

    def generic_post_check(self, object_list, bundle, permission):
        if not self.generic_base_check(object_list, bundle):
            raise Unauthorized("You are not allowed to access that resource.")
        user = bundle.request.user
        if not user.has_perm(permission):
            raise Unauthorized("You are not allowed to access that resource.")

        return True

    # List Checks
    def create_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.create_permission_code)

    def read_list(self, object_list, bundle):
        if not self.view_permission_code:
            return object_list

        return self.generic_list_check(object_list, bundle,
                                       self.view_permission_code)

    def update_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.update_permission_code)

    def delete_list(self, object_list, bundle):
        return self.generic_list_check(object_list, bundle,
                                       self.delete_permission_code)

    # Item Checks
    def create_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.create_permission_code)

    def read_detail(self, object_list, bundle):
        if not self.view_permission_code:
            return True
        return self.generic_item_check(object_list, bundle,
                                       self.view_permission_code)

    def update_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.update_permission_code)

    def delete_detail(self, object_list, bundle):
        return self.generic_item_check(object_list, bundle,
                                       self.delete_permission_code)

class FireDepartmentResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Fire Department API.
    """

    class Meta:
        readonly_fields = ['owned_tracts_geom']
        resource_name = 'fire-departments'
        queryset = FireDepartment.objects.defer('owned_tracts_geom').filter(archived=False)
        authorization = GuardianAuthorization(view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='admin_firedepartment')
        authentication = MultiAuthentication(SessionAuthentication(), ApiKeyAuthentication())
        cache = SimpleCache()
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get', 'put']
        filtering = {'state': ALL, 'featured': ALL}
        serializer = PrettyJSONSerializer()
        limit = 120
        max_limit = 2000

    def __init__(self, **kwargs):
        super(FireDepartmentResource, self).__init__(**kwargs)
        for f in getattr(self.Meta, 'readonly_fields', []):
            self.fields[f].readonly = True

    def hydrate_geom(self, bundle):
        geom = bundle.data.get('geom')
        boundary = geos.GEOSGeometry(json.dumps(geom))
        if type(boundary) is geos.Polygon:
            bundle.data['geom'] = json.loads(geos.MultiPolygon(boundary).json)
        return bundle

    def dehydrate(self, bundle):
        """
        remove fields not requested

        note: this is called for every object, so works for both detail and
              list requests
        """
        only_fields = bundle.request.GET.get('fields')
        debug_fields = bundle.request.GET.get('fields_debug', False)
        if only_fields:
            only_fields = only_fields.split(',')
            fields_to_remove = [field for field in bundle.data.keys()
                                if field not in only_fields]
            for field in fields_to_remove:
                del bundle.data[field]
            if debug_fields:
                bundle.data['_fields_selected'] = [field for field in only_fields
                                                   if field in bundle.data.keys()]
                bundle.data['_fields_removed'] = fields_to_remove
        return bundle

class FireStationResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Fire Station API.
    """

    department = fields.ForeignKey(FireDepartmentResource, 'department', null=True)

    class Meta:
        resource_name = 'firestations'

        queryset = FireStation.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='admin_firedepartment')
        authentication = MultiAuthentication(Authentication(), SessionAuthentication(), ApiKeyAuthentication())
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get', 'put']
        filtering = {'department': ('exact',), 'state': ('exact',), 'id': ('exact',)}
        excludes = ['addressbuildingname', 'complex_id', 'data_security', 'distribution_policy', 'fcode', 'foot_id',
                    'ftype', 'globalid', 'gnis_id', 'islandmark', 'loaddate', 'objectid', 'permanent_identifier',
                    'pointlocationtype', 'source_datadesc', 'source_datasetid', 'source_featureid',
                    'source_originator', 'admintype'
                    ]
        serializer = PrettyJSONSerializer()
        limit = 120

class StaffingResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The ResponseCapability API.
    """

    firestation = fields.ForeignKey(FireStationResource, 'firestation')

    class Meta:
        resource_name = 'staffing'
        queryset = Staffing.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='firestation.department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='change_firedepartment')
        authentication = MultiAuthentication(SessionAuthentication(), ApiKeyAuthentication())
        filtering = {'firestation': ALL}
        validation = FormValidation(form_class=StaffingForm)
        list_allowed_methods = ['get', 'post']
        detail_allowed_methods = ['get', 'put', 'delete']
        serializer = PrettyJSONSerializer()
        always_return_data = True

class WeatherWarningResource(JSONDefaultModelResourceMixin, ModelResource):
    """
    The Weather API mege with department id.
    """
    department = fields.ForeignKey(FireDepartmentResource, 'department', null=True)

    class Meta:
        resource_name = 'weather-warning'
        expired = timezone.now()
        # production when Task is running
        queryset = DepartmentWarnings.objects.filter(expiredate__gte=expired)
        # only show all warnings #good for testing Warnings
        # queryset = DepartmentWarnings.objects.all()
        authorization = GuardianAuthorization(delegate_to_property='department',
                                              view_permission_code=None,
                                              update_permission_code='change_firedepartment',
                                              create_permission_code='change_firedepartment',
                                              delete_permission_code='change_firedepartment')
        filtering = {'department': ('exact',), 'state': ('exact',), 'id': ('exact',)}
        list_allowed_methods = ['get', 'post']
        detail_allowed_methods = ['get']
        cache = SimpleCache(timeout=120)
        serializer = PrettyJSONSerializer()
        always_return_data = True
<|endoftext|>


<commit_before>import helpers
import json
import loader
import logging
import praw
import sqlite3
import sys
import time

_, login_conf, subs_conf, db_file = sys.argv

with open(login_conf) as f:
    config = json.load(f)

r = praw.Reddit(**config)
db = sqlite3.connect(db_file)
cursor = db.cursor()

loader = loader.YAMLLoader(db, cursor, r)
browsers = loader.load(subs_conf)
db.commit()

print("Loaded")

counter = 0
while True:
    try:
        for browser in browsers:
            browser.run()
        if counter == 20:
            for browser in browsers:
                try:
                    helpers.update_sr_tables(cursor, browser.subreddit)
                except Exception as e:
                    logging.error(e)
                    db.rollback()
                else:
                    db.commit()
            counter = 0
        else:
            counter += 1
        time.sleep(30)
    except KeyboardInterrupt:
        print("Keyboard interrupt; shutting down...")
        break
<commit_msg>Add user agent to bot.py; provide safer loading of PRAW config<commit_after>import helpers
import json
import loader
import logging
import praw
import sqlite3
import sys
import time

_, login_conf, subs_conf, db_file = sys.argv

with open(login_conf) as f:
    config_file_data = json.load(f)

reddit_config = {key: config_file_data[key]
                 for key in ["client_id", "client_secret", "username",
                             "password"]}
reddit_config["user_agent"] = \
  "python:/r/Philosophy reporter:v2.1 (by levimroth@gmail.com)"

r = praw.Reddit(**reddit_config)
db = sqlite3.connect(db_file)
cursor = db.cursor()

loader = loader.YAMLLoader(db, cursor, r)
browsers = loader.load(subs_conf)
db.commit()

print("Loaded")

counter = 0
while True:
    try:
        for browser in browsers:
            browser.run()
        if counter == 20:
            for browser in browsers:
                try:
                    helpers.update_sr_tables(cursor, browser.subreddit)
                except Exception as e:
                    logging.error(e)
                    db.rollback()
                else:
                    db.commit()
            counter = 0
        else:
            counter += 1
        time.sleep(30)
    except KeyboardInterrupt:
        print("Keyboard interrupt; shutting down...")
        break
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from collections import defaultdict

from ols_client import OlsClient
from pybel.constants import CITATION_TYPE_URL, belns_encodings, IS_A
from pybel.parser.language import rev_abundance_labels
from pybel.utils import ensure_quotes
from .definition_utils import write_namespace
from .document_utils import write_boilerplate
from .resources import (
    get_today_arty_namespace,
    deploy_namespace,
    get_latest_arty_namespace,
    get_today_arty_knowledge,
    deploy_knowledge,
)

function_to_encoding = defaultdict(list)
for encoding, functions in belns_encodings.items():
    for fn in functions:
        function_to_encoding[fn].append(encoding)

class OlsNamespaceOntology:
    """Wraps the functions needed to use the OLS to generate and deploy BEL namespaces"""

    def __init__(self, ontology, namespace_domain, bel_function, ols_base=None, auth=None):
        """
        :param str ontology: The name of the ontology. Ex: ``uberon``, ``go``, etc.
        :param str namespace_domain: One of: :data:`pybel.constants.NAMESPACE_DOMAIN_BIOPROCESS`,
                            :data:`pybel.constants.NAMESPACE_DOMAIN_CHEMICAL`,
                            :data:`pybel.constants.NAMESPACE_DOMAIN_GENE`, or
                            :data:`pybel.constants.NAMESPACE_DOMAIN_OTHER`
        :param str bel_function: The BEL function of elements of this ontology
        :param str ols_base: An optional, custom OLS base url
        :param tuple[str,str] auth: A pair of (str username, str password) to give to the auth keyword of the
                                    constructor of :class:`artifactory.ArtifactoryPath`. Defaults to the result of
                                    :func:`pybel_tools.resources.get_arty_auth`.
        """
        self.ontology = ontology
        self.namespace_domain = namespace_domain
        self.function = rev_abundance_labels[bel_function]
        self.encodings = function_to_encoding[bel_function]

        self.ols_client = OlsClient(ols_base=ols_base)
        self.auth = auth

        self.metadata = self.ols_client.get_ontology(self.ontology)

    @property
    def title(self):
        """The ontology's full name"""
        return self.metadata['config']['title']

    @property
    def preferred_prefix(self):
        """The preferred prefix, usually uppercase"""
        return self.metadata['config']['preferredPrefix']

    @property
    def description(self):
        """A description of the ontology"""
        return self.metadata['config']['description']

    @property
    def version(self):
        """The version of the ontology"""
        return self.metadata['config']['version']

    @property
    def version_iri(self):
        """The IRI of this version of the ontoloy"""
        return self.metadata['config']['versionIri']

    def write_namespace(self, file=None):
        """Serializes a BEL namespace.

        :param file file: A write-enable file or file-like
        """
        values = self.ols_client.iter_labels(self.ontology)

        write_namespace(
            namespace_name=self.title,
            namespace_keyword=self.preferred_prefix,
            namespace_domain=self.namespace_domain,
            namespace_description=self.description,
            namespace_version=self.version,
            citation_name=self.title,
            citation_url=self.version_iri,
            author_name='Charles Tapley Hoyt',
            author_contact='charles.hoyt@scai.fraunhofer.de',
            author_copyright='Creative Commons by 4.0',
            values=values,
            functions=self.encodings,
            file=file
        )

    def _write_hierarchy_header(self, file=None):
        write_boilerplate(
            document_name=self.title,
            description=self.description,
            authors='Charles Tapley Hoyt',
            contact='charles.hoyt@scai.fraunhofer.de',
            version=self.version,
            namespace_dict={self.preferred_prefix: get_latest_arty_namespace(self.ontology)},
            annotations_dict={},
            annotations_patterns={},
            namespace_patterns={},
            file=file
        )

    def _write_hierarchy_body(self, file=None):
        print('SET Citation = {{"{}","{}"}}'.format(CITATION_TYPE_URL, self.version_iri), file=file)
        print('SET Evidence = "Automatically generated hierarchy from {}"\n'.format(self.version_iri), file=file)

        for parent, child in self.ols_client.iter_hierarchy(self.ontology):
            print(
                '{fn}({keyword}:{child}) {relation} {fn}({keyword}:{parent})'.format(
                    fn=self.function,
                    keyword=self.preferred_prefix,
                    relation=IS_A,
                    parent=ensure_quotes(parent),
                    child=ensure_quotes(child)
                ),
                file=file
            )

    def write_hierarchy(self, file=None):
        """Serializes the hierarchy

        :param file file: A write-enable file or file-like
        """
        self._write_hierarchy_header(file=file)
        self._write_hierarchy_body(file=file)

    def deploy_namespace(self, hash_check=True):
        """Gets the data and writes BEL namespace file to Artifactory

        :param bool hash_check: Ensure the hash is unique before deploying
        :return: The path, if it was deployed successfully, else none.
        :rtype: str or None
        """
        file_name = get_today_arty_namespace(self.ontology)

        with open(file_name, 'w') as file:
            self.write_namespace(file)

        return deploy_namespace(file_name, self.ontology, hash_check=hash_check, auth=self.auth)

    def deploy_hierarchy(self):
        """Gets the data and writes BEL hierarchy file to Artifactory

        :return: The path, if it was deployed successfully, else none.
        :rtype: str or None
        """
        file_name = get_today_arty_knowledge(self.ontology)

        with open(file_name, 'w') as file:
            self.write_hierarchy(file)

        return deploy_knowledge(file_name, self.ontology, auth=self.auth)

    def deploy(self, hash_check=True):
        """Gets the data and writes BEL namespace file to Artifactory

        :param bool hash_check: Ensure the hash is unique before deploying
        """
        namespace_url = self.deploy_namespace(hash_check=hash_check)

        if not namespace_url:
            self.deploy_hierarchy()
<commit_msg>Add constrained ontology iteration<commit_after># -*- coding: utf-8 -*-

from collections import defaultdict

from ols_client import OlsClient
from pybel.constants import CITATION_TYPE_URL, IS_A, belns_encodings
from pybel.parser.language import rev_abundance_labels
from pybel.utils import ensure_quotes
from pybel_tools.definition_utils import write_annotation, write_namespace
from pybel_tools.document_utils import write_boilerplate
from pybel_tools.resources import (
    deploy_annotation, deploy_knowledge, deploy_namespace, get_latest_arty_namespace,
    get_today_arty_annotation, get_today_arty_knowledge, get_today_arty_namespace,
)

function_to_encoding = defaultdict(list)
for encoding, functions in belns_encodings.items():
    for fn in functions:
        function_to_encoding[fn].append(encoding)

class OlsNamespaceOntology:
    """Wraps the functions needed to use the OLS to generate and deploy BEL namespaces"""

    def __init__(self, ontology, namespace_domain, bel_function, ols_base=None, auth=None):
        """
        :param str ontology: The name of the ontology. Ex: ``uberon``, ``go``, etc.
        :param str namespace_domain: One of: :data:`pybel.constants.NAMESPACE_DOMAIN_BIOPROCESS`,
                            :data:`pybel.constants.NAMESPACE_DOMAIN_CHEMICAL`,
                            :data:`pybel.constants.NAMESPACE_DOMAIN_GENE`, or
                            :data:`pybel.constants.NAMESPACE_DOMAIN_OTHER`
        :param str bel_function: The BEL function of elements of this ontology. One of
                                :data:`pybel.constants.ABUNDANCE`, etc.
        :param str ols_base: An optional, custom OLS base url
        :param tuple[str,str] auth: A pair of (str username, str password) to give to the auth keyword of the
                                    constructor of :class:`artifactory.ArtifactoryPath`. Defaults to the result of
                                    :func:`pybel_tools.resources.get_arty_auth`.
        """
        self.ontology = ontology
        self.namespace_domain = namespace_domain
        self._bel_function = bel_function
        self.ols_client = OlsClient(ols_base=ols_base)
        self.auth = auth

        self.metadata = self.ols_client.get_ontology(self.ontology)

    @property
    def bel_function(self):
        return rev_abundance_labels[self._bel_function]

    @property
    def encodings(self):
        return function_to_encoding[self._bel_function]

    @property
    def title(self):
        """The ontology's full name"""
        return self.metadata['config']['title']

    @property
    def preferred_prefix(self):
        """The preferred prefix, usually uppercase"""
        return self.metadata['config']['preferredPrefix']

    @property
    def description(self):
        """A description of the ontology"""
        return self.metadata['config']['description']

    @property
    def version(self):
        """The version of the ontology"""
        return self.metadata['config']['version']

    @property
    def version_iri(self):
        """The IRI of this version of the ontoloy"""
        return self.metadata['config']['versionIri']

    def _get_values(self):
        return self.ols_client.iter_labels(self.ontology)

    def write_namespace(self, file=None):
        """Serializes a BEL namespace.

        :param file file: A write-enable file or file-like
        """
        values = self._get_values()

        write_namespace(
            namespace_name=self.title,
            namespace_keyword=self.preferred_prefix,
            namespace_domain=self.namespace_domain,
            namespace_description=self.description,
            namespace_version=self.version,
            citation_name=self.title,
            citation_url=self.version_iri,
            author_name='Charles Tapley Hoyt',
            author_contact='charles.hoyt@scai.fraunhofer.de',
            author_copyright='Creative Commons by 4.0',
            values=values,
            functions=self.encodings,
            file=file
        )

    def write_annotation(self, file=None):
        """Serializes a BEL annotation

        :param file file: A write-enable file or file-like
        """
        write_annotation(
            keyword=self.preferred_prefix,
            values={x: '' for x in self._get_values()},
            citation_name=self.title,
            description=self.description,
            version=self.version,
            author_name='Charles Tapley Hoyt',
            author_contact='charles.hoyt@scai.fraunhofer.de',
            author_copyright='Creative Commons by 4.0',
            file=file
        )

    def _write_hierarchy_header(self, file=None):
        write_boilerplate(
            document_name=self.title,
            description=self.description,
            authors='Charles Tapley Hoyt',
            contact='charles.hoyt@scai.fraunhofer.de',
            version=self.version,
            namespace_dict={self.preferred_prefix: get_latest_arty_namespace(self.ontology)},
            annotations_dict={},
            annotations_patterns={},
            namespace_patterns={},
            file=file
        )

    def _get_hierarchy(self):
        return self.ols_client.iter_hierarchy(self.ontology)

    def _write_hierarchy_body(self, file=None):
        print('SET Citation = {{"{}","{}"}}'.format(CITATION_TYPE_URL, self.version_iri), file=file)
        print('SET Evidence = "Automatically generated hierarchy from {}"\n'.format(self.version_iri), file=file)

        for parent, child in self._get_hierarchy():
            print(
                '{fn}({keyword}:{child}) {relation} {fn}({keyword}:{parent})'.format(
                    fn=self.bel_function,
                    keyword=self.preferred_prefix,
                    relation=IS_A,
                    parent=ensure_quotes(parent),
                    child=ensure_quotes(child)
                ),
                file=file
            )

    def write_hierarchy(self, file=None):
        """Serializes the hierarchy

        :param file file: A write-enable file or file-like
        """
        self._write_hierarchy_header(file=file)
        self._write_hierarchy_body(file=file)

    def deploy_namespace(self, hash_check=True):
        """Gets the data and writes BEL namespace file to Artifactory

        :param bool hash_check: Ensure the hash is unique before deploying
        :return: The path, if it was deployed successfully, else none.
        :rtype: str or None
        """
        file_name = get_today_arty_namespace(self.ontology)

        with open(file_name, 'w') as file:
            self.write_namespace(file)

        return deploy_namespace(file_name, self.ontology, hash_check=hash_check, auth=self.auth)

    def deploy_annotation(self, hash_check=True):
        """Gets the data and writes BEL annotation file to Artifactory

       :param bool hash_check: Ensure the hash is unique before deploying
       :return: The path, if it was deployed successfully, else none.
       :rtype: str or None
       """
        file_name = get_today_arty_annotation(self.ontology)

        with open(file_name, 'w') as file:
            self.write_annotation(file)

        return deploy_annotation(file_name, self.ontology, hash_check=hash_check, auth=self.auth)

    def deploy_hierarchy(self):
        """Gets the data and writes BEL hierarchy file to Artifactory

        :return: The path, if it was deployed successfully, else none.
        :rtype: str or None
        """
        file_name = get_today_arty_knowledge(self.ontology)

        with open(file_name, 'w') as file:
            self.write_hierarchy(file)

        return deploy_knowledge(file_name, self.ontology, auth=self.auth)

    def deploy(self, hash_check=True):
        """Gets the data and writes BEL namespace file to Artifactory

        :param bool hash_check: Ensure the hash is unique before deploying
        """
        namespace_url = self.deploy_namespace(hash_check=hash_check)

        if not namespace_url:
            self.deploy_hierarchy()

class OlsConstrainedNamespaceOntology(OlsNamespaceOntology):
    """Specifies that only the hierarchy under a certain term should be followed"""

    def __init__(self, ontology, namespace_domain, bel_function, base_term_iri, ols_base=None, auth=None):
        """
        :param str ontology: The name of the ontology. Ex: ``uberon``, ``go``, etc.
        :param str namespace_domain: One of: :data:`pybel.constants.NAMESPACE_DOMAIN_BIOPROCESS`,
                            :data:`pybel.constants.NAMESPACE_DOMAIN_CHEMICAL`,
                            :data:`pybel.constants.NAMESPACE_DOMAIN_GENE`, or
                            :data:`pybel.constants.NAMESPACE_DOMAIN_OTHER`
        :param str bel_function: The BEL function of elements of this ontology
        :param str ols_base: An optional, custom OLS base url
        :param tuple[str,str] auth: A pair of (str username, str password) to give to the auth keyword of the
                                    constructor of :class:`artifactory.ArtifactoryPath`. Defaults to the result of
                                    :func:`pybel_tools.resources.get_arty_auth`.
        """
        super(OlsConstrainedNamespaceOntology, self).__init__(
            ontology=ontology,
            namespace_domain=namespace_domain,
            bel_function=bel_function,
            ols_base=ols_base,
            auth=auth
        )
        self.base_term_iri = base_term_iri

        self._labels = None
        self._hierarchy = None

    def _get_values(self):
        return self.ols_client.iter_descendants_labels(self.ontology, self.base_term_iri)

    def _get_hierarchy(self):
        raise NotImplementedError
<|endoftext|>


<commit_before>'''
bgdataframe.py
This file contains the functions to get the OpenAPS data from the devicestatus.json files.

Main Function:
        get_bg_data(id_str, start_train_str, end_train_str, start_valid_str, end_valid_str, start_test_str, end_test_str)

Input:
        id_str                          ID number as a string
        start_train_str                 Start train date (%Y-%m-%d) as a string or datetime
        end_train_str                   End train date (%Y-%m-%d) as a string or datetime
        start_valid_str                 Start validation date (%Y-%m-%d) as a string or datetime
        end_valid_str                   End validation date (%Y-%m-%d) as a string or datetime
        start_test_str                  Start test date (%Y-%m-%d) as a string or datetime
        end_test_str                    End test date (%Y-%m-%d) as a string or datetime

Output:
        bg_df                           Pandas dataframe of all of the data from /data/[id_str]/devicestatus.json
        start_train_index               Index of the start of training
        end_train_index                 Index of the end of training (inclusive)
        start_valid_index               Index of the start of validation
        end_valid_index                 Index of the end of validation (inclusive)
        start_test_index                Index of the start of testing
        end_test_index                  Index of the end of testing (inclusive)

USAGE:
        bg_df, start_train_index, end_train_index, start_valid_index, end_valid_index, start_test_index, end_test_index = get_bg_data(id_str, start_train_str, end_train_str,
                                                                                start_valid_str, end_valid_str, start_test_str, end_test_str)

Trevor Tsue
2017-7-24
'''

import pandas as pd
import numpy as np

#Function to read in the start and end date according to year-month-day format
def _get_date(start_str, end_str):
    start_date = pd.to_datetime(start_str, format='%Y-%m-%d')
    end_date = pd.to_datetime(end_str, format='%Y-%m-%d')

    return start_date, end_date

#Function to find the start and stop indices for both the train and test dates
#The start_index is a higher number than the end_index because the higher indices are the earlier dates while the lower indices are the later dates
def _find_index(bg_df, start_date, end_date, make_col):
    if (make_col): bg_df['date'] = bg_df['created_at'].apply(lambda x: x.date()) #create column with just the date only once

    start_index = bg_df[bg_df['date'] == start_date.date()].index.max()
    end_index = bg_df[bg_df['date'] == end_date.date()].index.min()

    #Raises exception if invalid dates
    if np.isnan(start_index): raise Exception("Invalid start date: " + str(start_date.date()))
    if np.isnan(end_index): raise Exception("Invalid end date: " + str(end_date.date()))

    return start_index, end_index

#Function to get the bg data
def get_bg_data(id_str, start_train_str, end_train_str, start_valid_str, end_valid_str, start_test_str, end_test_str):
    """
    Function to convert the json file to a pandas dataframe.

    It takes in the string of the id and looks for the devicestatus.json file.
    All data should be stored such that in the directory where main.py lies,
    there is a directory called "data". Inside this directory,
    there is another directory with just the ID Number. Inside this data folder lies the

    """

    try:
        file_location = "data/" + id_str + "/devicestatus.json"
        bg_df = pd.read_json(file_location) #Opens the data file and reads in the data into a dataFrame
    except:
        raise IOError(file_location + " is not a valid file.")

    print
    print("{} total entries.".format(len(bg_df)))

    return bg_df

#Function to find the indices for the given start and end date strings
def get_bg_index(bg_df, start_str, end_str, set_str, make_col_bool):
    start_date, end_date = _get_date(start_str, end_str)
    start_index, end_index = _find_index(bg_df, start_date, end_date, make_col_bool)

    print("{} Set: {} total entries from {} to {}".format(set_str, start_index - end_index + 1, start_date, end_date))
    print("{} Set Start Index = {} and {} Set End Index = {}".format(set_str, start_index, set_str, end_index))
    print

    return start_index, end_index
<commit_msg>removed old files<commit_after><|endoftext|>


<commit_before># coding=utf-8
from __future__ import unicode_literals
import os
import sys

try:
    import ConfigParser as configparser
except ImportError:
    import configparser

__all__ = ('DMHY_URL', 'BGMI_PATH', 'DB_PATH', 'BGMI_SAVE_PATH',
           'BGMI_LX_PATH', 'DOWNLOAD_DELEGATE', 'CONFIG_FILE_PATH',
           'DETAIL_URL', 'FETCH_URL', 'IS_PYTHON3', 'MAX_PAGE',
           'BGMI_TMP_PATH', 'ARIA2_PATH', 'ARIA2_RPC_URL')

__readonly__ = ('BGMI_PATH', 'DB_PATH', 'CONFIG_FILE_PATH',
                'IS_PYTHON3', 'DETAIL_URL', 'FETCH_URL')

__writeable__ = tuple([i for i in __all__ if i not in __readonly__])

# --------- Immutable ---------- #
BGMI_PATH = os.path.join(os.environ.get('HOME', '/tmp'), '.bgmi')
DB_PATH = os.path.join(BGMI_PATH, 'bangumi.db')
CONFIG_FILE_PATH = os.path.join(BGMI_PATH, 'bgmi.cfg')

def read_config():
    c = configparser.ConfigParser()
    if not os.path.exists(CONFIG_FILE_PATH):
        write_default_config()
        return
    c.read(CONFIG_FILE_PATH)

    for i in __writeable__:
        if c.has_option('bgmi', i):
            globals().update({i: c.get('bgmi', i)})

def print_config():
    c = configparser.ConfigParser()
    if not os.path.exists(CONFIG_FILE_PATH):
        return
    c.read(CONFIG_FILE_PATH)
    for i in __writeable__:
        print('{0}={1}'.format(i, c.get('bgmi', i)))

def write_default_config():
    c = configparser.ConfigParser()
    if not c.has_section('bgmi'):
        c.add_section('bgmi')

    for i in __writeable__:
        v = globals().get(i, None)
        c.set('bgmi', i, v)

    try:
        c.write(open(CONFIG_FILE_PATH, 'w'))
    except IOError:
        print('[-] Error writing to config file and ignored')

def write_config(config=None, value=None):
    c = configparser.ConfigParser()
    c.read(CONFIG_FILE_PATH)

    if not c.has_section('bgmi'):
        c.add_section('bgmi')

    if config is not None and config not in __writeable__:
        print('{0} does not exist or not writeable'.format(config))
        exit(1)

    for i in __writeable__:
        v = globals().get(i, None)
        c.set('bgmi', i, v)

    try:
        if config is None and value is None:
            print_config()
            return
        elif value is None:
            print('{0}={1}'.format(config, c.get('bgmi', config)))
            return
        else:
            if config in __writeable__:
                c.set('bgmi', config, value)
                with open(CONFIG_FILE_PATH, 'w') as f:
                    c.write(f)
            print_config()

    except (configparser.NoOptionError):
        write_default_config()

# --------- Writeable ---------- #
# Setting dmhy url
DMHY_URL = 'http://dmhy.ricterz.me'

# BGmi user path
BGMI_SAVE_PATH = os.path.join(BGMI_PATH, 'bangumi')

# Xunlei offline download
BGMI_LX_PATH = os.path.join(BGMI_PATH, 'bgmi-lx')

# aria2
ARIA2_PATH = '/usr/bin/aria2c'
ARIA2_RPC_URL = 'http://localhost:6800/rpc'

# temp path
BGMI_TMP_PATH = os.path.join(BGMI_PATH, 'tmp')

# Download delegate
DOWNLOAD_DELEGATE = 'xunlei'

# max page number of fetch bangumi info
MAX_PAGE = '2'

# ------------------------------ #
# !!! Read config from file and write to globals() !!!
read_config()
# ------------------------------ #

# --------- Read-Only ---------- #
# Python version
IS_PYTHON3 = sys.version_info > (3, 0)

# Detail URL
FETCH_URL = '{0}/cms/page/name/programme.html'.format(DMHY_URL)
DETAIL_URL = '{0}/topics/list/page/[PAGE]?keyword='.format(DMHY_URL)

DANMAKU_API_URL = ''
<commit_msg>add DANMAKU_API_URL to __all__<commit_after># coding=utf-8
from __future__ import unicode_literals
import os
import sys

try:
    import ConfigParser as configparser
except ImportError:
    import configparser

__all__ = ('DMHY_URL', 'BGMI_PATH', 'DB_PATH', 'BGMI_SAVE_PATH',
           'BGMI_LX_PATH', 'DOWNLOAD_DELEGATE', 'CONFIG_FILE_PATH',
           'DETAIL_URL', 'FETCH_URL', 'IS_PYTHON3', 'MAX_PAGE',
           'BGMI_TMP_PATH', 'ARIA2_PATH', 'ARIA2_RPC_URL',
           'DANMAKU_API_URL', )

__readonly__ = ('BGMI_PATH', 'DB_PATH', 'CONFIG_FILE_PATH',
                'IS_PYTHON3', 'DETAIL_URL', 'FETCH_URL')

__writeable__ = tuple([i for i in __all__ if i not in __readonly__])

# --------- Immutable ---------- #
BGMI_PATH = os.path.join(os.environ.get('HOME', '/tmp'), '.bgmi')
DB_PATH = os.path.join(BGMI_PATH, 'bangumi.db')
CONFIG_FILE_PATH = os.path.join(BGMI_PATH, 'bgmi.cfg')

def read_config():
    c = configparser.ConfigParser()
    if not os.path.exists(CONFIG_FILE_PATH):
        write_default_config()
        return
    c.read(CONFIG_FILE_PATH)

    for i in __writeable__:
        if c.has_option('bgmi', i):
            globals().update({i: c.get('bgmi', i)})

def print_config():
    c = configparser.ConfigParser()
    if not os.path.exists(CONFIG_FILE_PATH):
        return
    c.read(CONFIG_FILE_PATH)
    for i in __writeable__:
        print('{0}={1}'.format(i, c.get('bgmi', i)))

def write_default_config():
    c = configparser.ConfigParser()
    if not c.has_section('bgmi'):
        c.add_section('bgmi')

    for i in __writeable__:
        v = globals().get(i, None)
        c.set('bgmi', i, v)

    try:
        c.write(open(CONFIG_FILE_PATH, 'w'))
    except IOError:
        print('[-] Error writing to config file and ignored')

def write_config(config=None, value=None):
    c = configparser.ConfigParser()
    c.read(CONFIG_FILE_PATH)

    if not c.has_section('bgmi'):
        c.add_section('bgmi')

    if config is not None and config not in __writeable__:
        print('{0} does not exist or not writeable'.format(config))
        exit(1)

    for i in __writeable__:
        v = globals().get(i, None)
        c.set('bgmi', i, v)

    try:
        if config is None and value is None:
            print_config()
            return
        elif value is None:
            print('{0}={1}'.format(config, c.get('bgmi', config)))
            return
        else:
            if config in __writeable__:
                c.set('bgmi', config, value)
                with open(CONFIG_FILE_PATH, 'w') as f:
                    c.write(f)
            print_config()

    except (configparser.NoOptionError):
        write_default_config()

# --------- Writeable ---------- #
# Setting dmhy url
DMHY_URL = 'http://dmhy.ricterz.me'

# BGmi user path
BGMI_SAVE_PATH = os.path.join(BGMI_PATH, 'bangumi')

# Xunlei offline download
BGMI_LX_PATH = os.path.join(BGMI_PATH, 'bgmi-lx')

# aria2
ARIA2_PATH = '/usr/bin/aria2c'
ARIA2_RPC_URL = 'http://localhost:6800/rpc'

# temp path
BGMI_TMP_PATH = os.path.join(BGMI_PATH, 'tmp')

# Download delegate
DOWNLOAD_DELEGATE = 'xunlei'

# max page number of fetch bangumi info
MAX_PAGE = '2'

# ------------------------------ #
# !!! Read config from file and write to globals() !!!
read_config()
# ------------------------------ #

# --------- Read-Only ---------- #
# Python version
IS_PYTHON3 = sys.version_info > (3, 0)

# Detail URL
FETCH_URL = '{0}/cms/page/name/programme.html'.format(DMHY_URL)
DETAIL_URL = '{0}/topics/list/page/[PAGE]?keyword='.format(DMHY_URL)

DANMAKU_API_URL = ''
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2013 DNAnexus, Inc.
#
# This file is part of dx-toolkit (DNAnexus platform client libraries).
#
#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
#   use this file except in compliance with the License. You may obtain a copy
#   of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.

import os, sys, unittest, json, tempfile, subprocess, csv, shutil, re, time

from dxpy_testutil import DXTestCase
import dxpy_testutil as testutil

import dxpy

@unittest.skipUnless(testutil.TEST_FUSE,
                     'skipping tests that would mount FUSE filesystems')
class TestDXFS(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        if 'DXTEST_FUSE' not in os.environ:
            return
        proj_name = u"dxclient_test_pröject"
        cls.project_id = subprocess.check_output(u"dx new project '{p}' --brief".format(p=proj_name), shell=True).strip()
        os.environ["DX_PROJECT_CONTEXT_ID"] = cls.project_id
        os.environ["DX_CLI_WD"] = '/'
        cls.project = dxpy.DXProject(cls.project_id)
        dxpy._initialize(suppress_warning=True)

        subprocess.check_call(['dx', 'mkdir', 'foo'])
        subprocess.check_call(['dx', 'mkdir', 'bar'])
        dxpy.upload_local_file(__file__, wait_on_close=True)

        cls.mountpoint = tempfile.mkdtemp()
        args = ['dx-mount', cls.mountpoint, '--foreground']
        args.append('--debug')
        cls.fuse_driver = subprocess.Popen(args)
        time.sleep(1)
        assert(cls.fuse_driver.poll() == None)

    @classmethod
    def tearDownClass(cls):
        try:
            cls.fuse_driver.terminate()
            subprocess.check_call(u"dx rmproject --yes {p}".format(p=cls.project_id), shell=True)
        except:
            pass

    def test_dxfs_operations(self):
        # FIXME: Make the mount live or add command to refresh it with remote changes
        #subprocess.check_call(['dx', 'mkdir', 'foo'])
        #subprocess.check_call(['dx', 'mkdir', 'bar'])
        #subprocess.check_call(['dx', 'mkdir', '-p', '/bar/baz'])

        self.assertEqual(set(os.listdir(self.mountpoint)), set(['foo', 'bar', os.path.basename(__file__)]))

        # Reading
        self.assertEqual(open(__file__).read(), open(os.path.join(self.mountpoint, __file__)).read())

        # Moving
        shutil.move(os.path.join(self.mountpoint, __file__), os.path.join(self.mountpoint, __file__+"2"))
        self.assertEqual(set(os.listdir(self.mountpoint)), set(['foo', 'bar', os.path.basename(__file__+"2")]))
        shutil.move(os.path.join(self.mountpoint, __file__+"2"), os.path.join(self.mountpoint, "foo"))
        self.assertEqual(set(os.listdir(os.path.join(self.mountpoint, 'foo'))), set([os.path.basename(__file__+"2")]))
        folder_listing = self.project.list_folder('/foo')
        self.assertEqual(len(folder_listing['folders']), 0)
        self.assertEqual(len(folder_listing['objects']), 1)
        self.assertEqual(dxpy.get_handler(folder_listing['objects'][0]['id']).name, os.path.basename(__file__+"2"))
        self.assertEqual(open(__file__).read(), open(os.path.join(self.mountpoint, 'foo', __file__+"2")).read())

    def test_dxfs_mkdir(self):
        os.mkdir(os.path.join(self.mountpoint, 'xyz'))
        self.assertIn('/xyz', self.project.list_folder('/')['folders'])
        os.mkdir(os.path.join(self.mountpoint, 'xyz', 'abc'))
        self.assertIn('/xyz/abc', self.project.list_folder('/xyz')['folders'])
        os.rmdir(os.path.join(self.mountpoint, 'xyz', 'abc'))
        self.assertNotIn('/xyz/abc', self.project.list_folder('/xyz')['folders'])
        os.rmdir(os.path.join(self.mountpoint, 'xyz'))
        self.assertNotIn('/xyz', self.project.list_folder('/')['folders'])

    def test_dxfs_write(self):
        filename1 = os.path.join(self.mountpoint, 'foo', 'f1')
        with open(filename1, 'w') as fh:
            fh.write('0123456789ABCDEF'*256)
        subprocess.check_call(['dxfs', 'close', filename1, '--wait'])
        with open(filename1) as fh:
            d = fh.read()
            print len(d)
            self.assertEqual(d, '0123456789ABCDEF'*256, "File readback failed")

if __name__ == '__main__':
    unittest.main()
<commit_msg>Remove unused imports<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2013 DNAnexus, Inc.
#
# This file is part of dx-toolkit (DNAnexus platform client libraries).
#
#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
#   use this file except in compliance with the License. You may obtain a copy
#   of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.

import os, unittest, tempfile, subprocess, shutil, time

import test.dxpy_testutil as testutil

import dxpy

@unittest.skipUnless(testutil.TEST_FUSE,
                     'skipping tests that would mount FUSE filesystems')
class TestDXFS(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        if 'DXTEST_FUSE' not in os.environ:
            return
        proj_name = u"dxclient_test_pröject"
        cls.project_id = subprocess.check_output(u"dx new project '{p}' --brief".format(p=proj_name), shell=True).strip()
        os.environ["DX_PROJECT_CONTEXT_ID"] = cls.project_id
        os.environ["DX_CLI_WD"] = '/'
        cls.project = dxpy.DXProject(cls.project_id)
        dxpy._initialize(suppress_warning=True)

        subprocess.check_call(['dx', 'mkdir', 'foo'])
        subprocess.check_call(['dx', 'mkdir', 'bar'])
        dxpy.upload_local_file(__file__, wait_on_close=True)

        cls.mountpoint = tempfile.mkdtemp()
        args = ['dx-mount', cls.mountpoint, '--foreground']
        args.append('--debug')
        cls.fuse_driver = subprocess.Popen(args)
        time.sleep(1)
        assert(cls.fuse_driver.poll() == None)

    @classmethod
    def tearDownClass(cls):
        try:
            cls.fuse_driver.terminate()
            subprocess.check_call(u"dx rmproject --yes {p}".format(p=cls.project_id), shell=True)
        except:
            pass

    def test_dxfs_operations(self):
        # FIXME: Make the mount live or add command to refresh it with remote changes
        #subprocess.check_call(['dx', 'mkdir', 'foo'])
        #subprocess.check_call(['dx', 'mkdir', 'bar'])
        #subprocess.check_call(['dx', 'mkdir', '-p', '/bar/baz'])

        self.assertEqual(set(os.listdir(self.mountpoint)), set(['foo', 'bar', os.path.basename(__file__)]))

        # Reading
        self.assertEqual(open(__file__).read(), open(os.path.join(self.mountpoint, __file__)).read())

        # Moving
        shutil.move(os.path.join(self.mountpoint, __file__), os.path.join(self.mountpoint, __file__+"2"))
        self.assertEqual(set(os.listdir(self.mountpoint)), set(['foo', 'bar', os.path.basename(__file__+"2")]))
        shutil.move(os.path.join(self.mountpoint, __file__+"2"), os.path.join(self.mountpoint, "foo"))
        self.assertEqual(set(os.listdir(os.path.join(self.mountpoint, 'foo'))), set([os.path.basename(__file__+"2")]))
        folder_listing = self.project.list_folder('/foo')
        self.assertEqual(len(folder_listing['folders']), 0)
        self.assertEqual(len(folder_listing['objects']), 1)
        self.assertEqual(dxpy.get_handler(folder_listing['objects'][0]['id']).name, os.path.basename(__file__+"2"))
        self.assertEqual(open(__file__).read(), open(os.path.join(self.mountpoint, 'foo', __file__+"2")).read())

    def test_dxfs_mkdir(self):
        os.mkdir(os.path.join(self.mountpoint, 'xyz'))
        self.assertIn('/xyz', self.project.list_folder('/')['folders'])
        os.mkdir(os.path.join(self.mountpoint, 'xyz', 'abc'))
        self.assertIn('/xyz/abc', self.project.list_folder('/xyz')['folders'])
        os.rmdir(os.path.join(self.mountpoint, 'xyz', 'abc'))
        self.assertNotIn('/xyz/abc', self.project.list_folder('/xyz')['folders'])
        os.rmdir(os.path.join(self.mountpoint, 'xyz'))
        self.assertNotIn('/xyz', self.project.list_folder('/')['folders'])

    def test_dxfs_write(self):
        filename1 = os.path.join(self.mountpoint, 'foo', 'f1')
        with open(filename1, 'w') as fh:
            fh.write('0123456789ABCDEF'*256)
        subprocess.check_call(['dxfs', 'close', filename1, '--wait'])
        with open(filename1) as fh:
            d = fh.read()
            print len(d)
            self.assertEqual(d, '0123456789ABCDEF'*256, "File readback failed")

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# macf.py - Waqas Bhatti (wbhatti@astro.princeton.edu) - Oct 2017

'''
This contains the ACF period-finding algorithm from McQuillan+ 2013a and
McQuillan+ 2014.

'''

#############
## LOGGING ##
#############

import logging
from astrobase import log_sub, log_fmt, log_date_fmt

DEBUG = False
if DEBUG:
    level = logging.DEBUG
else:
    level = logging.INFO
LOGGER = logging.getLogger(__name__)
logging.basicConfig(
    level=level,
    style=log_sub,
    format=log_fmt,
    datefmt=log_date_fmt,
)

LOGDEBUG = LOGGER.debug
LOGINFO = LOGGER.info
LOGWARNING = LOGGER.warning
LOGERROR = LOGGER.error
LOGEXCEPTION = LOGGER.exception

#############
## IMPORTS ##
#############

from multiprocessing import Pool, cpu_count

import numpy as np

# import these to avoid lookup overhead
from numpy import nan as npnan, sum as npsum, abs as npabs, \
    roll as nproll, isfinite as npisfinite, std as npstd, \
    sign as npsign, sqrt as npsqrt, median as npmedian, \
    array as nparray, percentile as nppercentile, \
    polyfit as nppolyfit, var as npvar, max as npmax, min as npmin, \
    log10 as nplog10, arange as nparange, pi as MPI, floor as npfloor, \
    argsort as npargsort, cos as npcos, sin as npsin, tan as nptan, \
    where as npwhere, linspace as nplinspace, \
    zeros_like as npzeros_like, full_like as npfull_like, \
    arctan as nparctan, nanargmax as npnanargmax, nanargmin as npnanargmin, \
    empty as npempty, ceil as npceil, mean as npmean, \
    digitize as npdigitize, unique as npunique, \
    argmax as npargmax, argmin as npargmin

from scipy.signal import argrelmax, argrelmin, savgol_filter
from astropy.convolution import convolve, Gaussian1DKernel

###################
## LOCAL IMPORTS ##
###################

from ..lcmath import phase_magseries, sigclip_magseries, time_bin_magseries, \
    phase_bin_magseries, fill_magseries_gaps

from ..varbase.autocorr import autocorr_magseries

############
## CONFIG ##
############

NCPUS = cpu_count()

######################
## HELPER FUNCTIONS ##
######################

def _smooth_acf(acf, windowfwhm=7, windowsize=21):
    '''
    This returns a smoothed version of the ACF.

    Convolves the ACF with a Gaussian of given windowsize and windowfwhm

    '''

    convkernel = Gaussian1DKernel(windowfwhm, x_size=windowsize)
    smoothed = convolve(acf, convkernel, boundary='extend')

    return smoothed

def _smooth_acf_savgol(acf, windowsize=21, polyorder=2):
    '''
    This returns a smoothed version of the ACF.

    This version uses the Savitsky-Golay smoothing filter

    '''

    smoothed = savgol_filter(acf, windowsize, polyorder)

    return smoothed

def _get_acf_peakheights(lags, acf, npeaks=20, searchinterval=1):
    '''This calculates the relative peak heights for first npeaks in ACF.

    Usually, the first peak or the second peak (if its peak height > first peak)
    corresponds to the correct lag. When we know the correct lag, the period is
    then:

    bestperiod = time[lags == bestlag] - time[0]

    '''

    maxinds = argrelmax(acf, order=searchinterval)[0]
    maxacfs = acf[maxinds]
    maxlags = lags[maxinds]
    mininds = argrelmin(acf, order=searchinterval)[0]
    minacfs = acf[mininds]
    minlags = lags[mininds]

    relpeakheights = np.zeros(npeaks)
    relpeaklags = np.zeros(npeaks,dtype=np.int64)
    peakindices = np.zeros(npeaks,dtype=np.int64)

    for peakind, mxi in enumerate(maxinds[:npeaks]):

        # check if there are no mins to the left
        # throw away this peak because it's probably spurious
        # (FIXME: is this OK?)
        if np.all(mxi < mininds):
            continue

        leftminind = mininds[mininds < mxi][-1]  # the last index to the left
        rightminind = mininds[mininds > mxi][0]  # the first index to the right
        relpeakheights[peakind] = (
            acf[mxi] - (acf[leftminind] + acf[rightminind])/2.0
        )
        relpeaklags[peakind] = lags[mxi]
        peakindices[peakind] = peakind

    # figure out the bestperiod if possible
    if relpeakheights[0] > relpeakheights[1]:
        bestlag = relpeaklags[0]
        bestpeakheight = relpeakheights[0]
        bestpeakindex = peakindices[0]
    else:
        bestlag = relpeaklags[1]
        bestpeakheight = relpeakheights[1]
        bestpeakindex = peakindices[1]

    return {'maxinds':maxinds,
            'maxacfs':maxacfs,
            'maxlags':maxlags,
            'mininds':mininds,
            'minacfs':minacfs,
            'minlags':minlags,
            'relpeakheights':relpeakheights,
            'relpeaklags':relpeaklags,
            'peakindices':peakindices,
            'bestlag':bestlag,
            'bestpeakheight':bestpeakheight,
            'bestpeakindex':bestpeakindex}

def plot_acf_results(acfp, outfile, maxlags=5000, yrange=[-0.4,0.4]):
    '''
    This plots the unsmoothed/smoothed ACF vs lag.

    acfp is the resultdict from macf_period_find below.

    '''

    import matplotlib.pyplot as plt

    lags = acfp['acfresults']['lags'][:maxlags]
    smoothedacf = acfp['acf'][:maxlags]
    unsmoothedacf = acfp['acfresults']['acf'][:maxlags]

    acfparams = acfp['kwargs']['smoothfunckwargs'].copy()
    acfparams.update({'peakinterval': int(acfp['kwargs']['smoothacf']/2.0)})

    # plot the ACFs
    fig, ax1 = plt.subplots()

    # this is lags vs acf
    ax1.plot(lags, unsmoothedacf, label='unsmoothed ACF',color='#1f77b4')
    ax1.plot(lags, smoothedacf, label='smoothed ACF', color='#ff7f0e')

    ax1.set_xlim((0,maxlags))

    ax1.set_xlabel('lags')

    # overplot the identified peaks
    acfmaxinds = acfp['acfpeaks']['maxinds']

    for i, maxind in enumerate(acfmaxinds):
        if i == 0:
            ax1.axvline(maxind,
                        linewidth=2.0,
                        color='red',
                        ymin=0.2, ymax=0.3,
                        label='identified ACF peaks')
        else:
            ax1.axvline(maxind,
                        linewidth=2.0,
                        color='red',
                        ymin=0.2, ymax=0.3)

    plt.ylabel('ACF')
    plt.ylim(yrange)
    ax1.legend()
    plt.title('%s' % repr(acfparams))
    plt.tight_layout()
    plt.savefig(outfile)
    plt.close('all')

    return outfile

############################
## PERIOD FINDER FUNCTION ##
############################

def macf_period_find(
        times,
        mags,
        errs,
        fillgaps=0.0,
        filterwindow=11,
        forcetimebin=None,
        maxlags=None,
        maxacfpeaks=10,
        smoothacf=21,  # set for Kepler-type LCs, see details below
        smoothfunc=_smooth_acf_savgol,
        smoothfunckwargs={},
        magsarefluxes=False,
        sigclip=3.0,
        verbose=True,
        periodepsilon=0.1,  # doesn't do anything, for consistent external API
        nworkers=None,      # doesn't do anything, for consistent external API
        startp=None,        # doesn't do anything, for consistent external API
        endp=None,          # doesn't do anything, for consistent external API
        autofreq=None,      # doesn't do anything, for consistent external API
        stepsize=None,      # doesn't do anything, for consistent external API
):
    '''This finds periods using the McQuillan+ (2013a, 2014) method.

    Args
    ----

    times, mags, errs are np.arrays.

    fillgaps is what to use to fill in gaps in the time series. If this is
    'noiselevel', will smooth the light curve using a point window size of
    filterwindow (this should be an odd integer), subtract the smoothed LC from
    the actual LC and estimate the RMS. This RMS will be used to fill in the
    gaps. Other useful values here are 0.0, and np.nan.

    forcetimebin is used to force a particular cadence in the light curve other
    than the automatically determined cadence. This effectively rebins the light
    curve to this cadence.

    maxlags is maximum number of lags to calculate. If None, will calculate all
    lags.

    maxacfpeaks is the maximum number of ACF peaks to use when finding the
    highest peak and obtaining a fit period.

    smoothacf is the number of points to use as the window size when smoothing
    the acf with the smoothfunc. This should be an odd integer value. If this is
    None, will not smooth the ACF, but this will probably lead to finding
    spurious peaks in a generally noisy ACF.

    For Kepler, a value between 21 and 51 seems to work fine. For ground based
    data, much larger values may be necessary: between 1001 and 2001 seem to
    work best for the HAT surveys. This is dependent on cadence, RMS of the
    light curve, the periods of the objects you're looking for, and finally, any
    correlated noise in the light curve. Make a plot of the smoothed/unsmoothed
    ACF vs. lag using the result dict of this function and the plot_acf_results
    function above to see the identified ACF peaks and what kind of smoothing
    might be needed.

    The value of smoothacf will also be used to figure out the interval to use
    when searching for local peaks in the ACF: this interval is 1/2 of the
    smoothacf value.

    smoothfunc is a function to use when smoothing the ACF. This should take at
    least one kwarg: 'windowsize'. Other kwargs can be passed in using a dict
    provided in smoothfunckwargs. By default, this uses a Savitsky-Golay filter,
    a Gaussian filter is also provided but not used. Another good option would
    be an actual low-pass filter (generated using scipy.signal?) to remove all
    high frequency noise from the ACF.

    magarefluxes is True if the measurements provided in mags are actually
    fluxes, False otherwise.

    sigclip is the sigma to use when sigma-clipping the magnitude time series.

    Returns
    -------

    Returns a dictionary with results. dict['bestperiod'] is the estimated best
    period and dict['fitperiodrms'] is its estimated error. Other interesting
    things in the output include:

    - dict['acfresults']: all results from calculating the ACF. in particular,
      the unsmoothed ACF might be of interest: dict['acfresults']['acf'] and
      dict['acfresults']['lags'].

    - dict['lags'] and dict['acf'] contain the ACF after smoothing was applied.

    - dict['periods'] and dict['lspvals'] can be used to construct a
      pseudo-periodogram.

    - dict['naivebestperiod'] is obtained by multiplying the lag at the highest
      ACF peak with the cadence. This is usually close to the fit period
      (dict['fitbestperiod']), which is calculated by doing a fit to the lags
      vs. peak index relation as in McQuillan+ 2014.

    '''

    # get the ACF
    acfres = autocorr_magseries(
        times,
        mags,
        errs,
        maxlags=maxlags,
        fillgaps=fillgaps,
        forcetimebin=forcetimebin,
        sigclip=sigclip,
        magsarefluxes=magsarefluxes,
        filterwindow=filterwindow,
        verbose=verbose
    )

    xlags = acfres['lags']

    # smooth the ACF if requested
    if smoothacf and isinstance(smoothacf, int) and smoothacf > 0:

        sfkwargs = smoothfunckwargs.copy()

        sfkwargs.update({'windowsize':smoothacf})
        xacf = smoothfunc(acfres['acf'], **sfkwargs)

    else:

        xacf = acfres['acf']

    # get the relative peak heights and fit best lag
    peakres = _get_acf_peakheights(xlags, xacf, npeaks=maxacfpeaks,
                                   searchinterval=int(smoothacf/2))

    # this is the best period's best ACF peak height
    bestlspval = peakres['bestpeakheight']

    try:

        # get the fit best lag from a linear fit to the peak index vs time(peak
        # lag) function as in McQillian+ (2014)
        fity = np.concatenate((
            [0.0, peakres['bestlag']],
            peakres['relpeaklags'][peakres['relpeaklags'] > peakres['bestlag']]
        ))
        fity = fity*acfres['cadence']
        fitx = np.arange(fity.size)

        fitcoeffs, fitcovar = np.polyfit(fitx, fity, 1, cov=True)

        # fit best period is the gradient of fit
        fitbestperiod = fitcoeffs[0]
        bestperiodrms = np.sqrt(fitcovar[0,0])  # from the covariance matrix

    except Exception as e:

        LOGWARNING('linear fit to time at each peak lag '
                   'value vs. peak number failed, '
                   'naively calculated ACF period may not be accurate')
        fitcoeffs = np.array([np.nan, np.nan])
        fitcovar = np.array([[np.nan, np.nan], [np.nan, np.nan]])
        fitbestperiod = np.nan
        bestperiodrms = np.nan
        raise

    # calculate the naive best period using delta_tau = lag * cadence
    naivebestperiod = peakres['bestlag']*acfres['cadence']

    if fitbestperiod < naivebestperiod:
        LOGWARNING('fit bestperiod = %.5f may be an alias, '
                   'naively calculated bestperiod is = %.5f' %
                   (fitbestperiod, naivebestperiod))

    if np.isfinite(fitbestperiod):
        bestperiod = fitbestperiod
    else:
        bestperiod = naivebestperiod

    return {'bestperiod':bestperiod,
            'bestlspval':bestlspval,
            'nbestpeaks':maxacfpeaks,
            # for compliance with the common pfmethod API
            'nbestperiods':np.concatenate([
                [fitbestperiod],
                peakres['relpeaklags'][1:maxacfpeaks]*acfres['cadence']
            ]),
            'nbestlspvals':peakres['maxacfs'][:maxacfpeaks],
            'lspvals':xacf,
            'periods':xlags*acfres['cadence'],
            'acf':xacf,
            'lags':xlags,
            'method':'acf',
            'naivebestperiod':naivebestperiod,
            'fitbestperiod':fitbestperiod,
            'fitperiodrms':bestperiodrms,
            'periodfitcoeffs':fitcoeffs,
            'periodfitcovar':fitcovar,
            'kwargs':{'maxlags':maxlags,
                      'maxacfpeaks':maxacfpeaks,
                      'fillgaps':fillgaps,
                      'filterwindow':filterwindow,
                      'smoothacf':smoothacf,
                      'smoothfunckwargs':sfkwargs,
                      'magsarefluxes':magsarefluxes,
                      'sigclip':sigclip},
            'acfresults':acfres,
            'acfpeaks':peakres}
<commit_msg>macf: clean up imports<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# macf.py - Waqas Bhatti (wbhatti@astro.princeton.edu) - Oct 2017

'''
This contains the ACF period-finding algorithm from McQuillan+ 2013a and
McQuillan+ 2014.

'''

#############
## LOGGING ##
#############

import logging
from astrobase import log_sub, log_fmt, log_date_fmt

DEBUG = False
if DEBUG:
    level = logging.DEBUG
else:
    level = logging.INFO
LOGGER = logging.getLogger(__name__)
logging.basicConfig(
    level=level,
    style=log_sub,
    format=log_fmt,
    datefmt=log_date_fmt,
)

LOGDEBUG = LOGGER.debug
LOGINFO = LOGGER.info
LOGWARNING = LOGGER.warning
LOGERROR = LOGGER.error
LOGEXCEPTION = LOGGER.exception

#############
## IMPORTS ##
#############

from numpy import (
    nan as npnan, arange as nparange, array as nparray,
    isfinite as npisfinite, sqrt as npsqrt, concatenate as npconcatenate,
    zeros as npzeros, int64 as npint64, all as npall, polyfit as nppolyfit
)

from scipy.signal import argrelmax, argrelmin, savgol_filter
from astropy.convolution import convolve, Gaussian1DKernel

###################
## LOCAL IMPORTS ##
###################

from ..varbase.autocorr import autocorr_magseries

######################
## HELPER FUNCTIONS ##
######################

def _smooth_acf(acf, windowfwhm=7, windowsize=21):
    '''
    This returns a smoothed version of the ACF.

    Convolves the ACF with a Gaussian of given windowsize and windowfwhm

    '''

    convkernel = Gaussian1DKernel(windowfwhm, x_size=windowsize)
    smoothed = convolve(acf, convkernel, boundary='extend')

    return smoothed

def _smooth_acf_savgol(acf, windowsize=21, polyorder=2):
    '''
    This returns a smoothed version of the ACF.

    This version uses the Savitsky-Golay smoothing filter

    '''

    smoothed = savgol_filter(acf, windowsize, polyorder)

    return smoothed

def _get_acf_peakheights(lags, acf, npeaks=20, searchinterval=1):
    '''This calculates the relative peak heights for first npeaks in ACF.

    Usually, the first peak or the second peak (if its peak height > first peak)
    corresponds to the correct lag. When we know the correct lag, the period is
    then:

    bestperiod = time[lags == bestlag] - time[0]

    '''

    maxinds = argrelmax(acf, order=searchinterval)[0]
    maxacfs = acf[maxinds]
    maxlags = lags[maxinds]
    mininds = argrelmin(acf, order=searchinterval)[0]
    minacfs = acf[mininds]
    minlags = lags[mininds]

    relpeakheights = npzeros(npeaks)
    relpeaklags = npzeros(npeaks,dtype=npint64)
    peakindices = npzeros(npeaks,dtype=npint64)

    for peakind, mxi in enumerate(maxinds[:npeaks]):

        # check if there are no mins to the left
        # throw away this peak because it's probably spurious
        # (FIXME: is this OK?)
        if npall(mxi < mininds):
            continue

        leftminind = mininds[mininds < mxi][-1]  # the last index to the left
        rightminind = mininds[mininds > mxi][0]  # the first index to the right
        relpeakheights[peakind] = (
            acf[mxi] - (acf[leftminind] + acf[rightminind])/2.0
        )
        relpeaklags[peakind] = lags[mxi]
        peakindices[peakind] = peakind

    # figure out the bestperiod if possible
    if relpeakheights[0] > relpeakheights[1]:
        bestlag = relpeaklags[0]
        bestpeakheight = relpeakheights[0]
        bestpeakindex = peakindices[0]
    else:
        bestlag = relpeaklags[1]
        bestpeakheight = relpeakheights[1]
        bestpeakindex = peakindices[1]

    return {'maxinds':maxinds,
            'maxacfs':maxacfs,
            'maxlags':maxlags,
            'mininds':mininds,
            'minacfs':minacfs,
            'minlags':minlags,
            'relpeakheights':relpeakheights,
            'relpeaklags':relpeaklags,
            'peakindices':peakindices,
            'bestlag':bestlag,
            'bestpeakheight':bestpeakheight,
            'bestpeakindex':bestpeakindex}

def plot_acf_results(acfp, outfile, maxlags=5000, yrange=(-0.4,0.4)):
    '''
    This plots the unsmoothed/smoothed ACF vs lag.

    acfp is the resultdict from macf_period_find below.

    '''

    import matplotlib.pyplot as plt

    lags = acfp['acfresults']['lags'][:maxlags]
    smoothedacf = acfp['acf'][:maxlags]
    unsmoothedacf = acfp['acfresults']['acf'][:maxlags]

    acfparams = acfp['kwargs']['smoothfunckwargs'].copy()
    acfparams.update({'peakinterval': int(acfp['kwargs']['smoothacf']/2.0)})

    # plot the ACFs
    fig, ax1 = plt.subplots()

    # this is lags vs acf
    ax1.plot(lags, unsmoothedacf, label='unsmoothed ACF',color='#1f77b4')
    ax1.plot(lags, smoothedacf, label='smoothed ACF', color='#ff7f0e')

    ax1.set_xlim((0,maxlags))

    ax1.set_xlabel('lags')

    # overplot the identified peaks
    acfmaxinds = acfp['acfpeaks']['maxinds']

    for i, maxind in enumerate(acfmaxinds):
        if i == 0:
            ax1.axvline(maxind,
                        linewidth=2.0,
                        color='red',
                        ymin=0.2, ymax=0.3,
                        label='identified ACF peaks')
        else:
            ax1.axvline(maxind,
                        linewidth=2.0,
                        color='red',
                        ymin=0.2, ymax=0.3)

    plt.ylabel('ACF')
    plt.ylim(yrange)
    ax1.legend()
    plt.title('%s' % repr(acfparams))
    plt.tight_layout()
    plt.savefig(outfile)
    plt.close('all')

    return outfile

############################
## PERIOD FINDER FUNCTION ##
############################

def macf_period_find(
        times,
        mags,
        errs,
        fillgaps=0.0,
        filterwindow=11,
        forcetimebin=None,
        maxlags=None,
        maxacfpeaks=10,
        smoothacf=21,  # set for Kepler-type LCs, see details below
        smoothfunc=_smooth_acf_savgol,
        smoothfunckwargs=None,
        magsarefluxes=False,
        sigclip=3.0,
        verbose=True,
        periodepsilon=0.1,  # doesn't do anything, for consistent external API
        nworkers=None,      # doesn't do anything, for consistent external API
        startp=None,        # doesn't do anything, for consistent external API
        endp=None,          # doesn't do anything, for consistent external API
        autofreq=None,      # doesn't do anything, for consistent external API
        stepsize=None,      # doesn't do anything, for consistent external API
):
    '''This finds periods using the McQuillan+ (2013a, 2014) method.

    Args
    ----

    times, mags, errs are nparrays.

    fillgaps is what to use to fill in gaps in the time series. If this is
    'noiselevel', will smooth the light curve using a point window size of
    filterwindow (this should be an odd integer), subtract the smoothed LC from
    the actual LC and estimate the RMS. This RMS will be used to fill in the
    gaps. Other useful values here are 0.0, and npnan.

    forcetimebin is used to force a particular cadence in the light curve other
    than the automatically determined cadence. This effectively rebins the light
    curve to this cadence.

    maxlags is maximum number of lags to calculate. If None, will calculate all
    lags.

    maxacfpeaks is the maximum number of ACF peaks to use when finding the
    highest peak and obtaining a fit period.

    smoothacf is the number of points to use as the window size when smoothing
    the acf with the smoothfunc. This should be an odd integer value. If this is
    None, will not smooth the ACF, but this will probably lead to finding
    spurious peaks in a generally noisy ACF.

    For Kepler, a value between 21 and 51 seems to work fine. For ground based
    data, much larger values may be necessary: between 1001 and 2001 seem to
    work best for the HAT surveys. This is dependent on cadence, RMS of the
    light curve, the periods of the objects you're looking for, and finally, any
    correlated noise in the light curve. Make a plot of the smoothed/unsmoothed
    ACF vs. lag using the result dict of this function and the plot_acf_results
    function above to see the identified ACF peaks and what kind of smoothing
    might be needed.

    The value of smoothacf will also be used to figure out the interval to use
    when searching for local peaks in the ACF: this interval is 1/2 of the
    smoothacf value.

    smoothfunc is a function to use when smoothing the ACF. This should take at
    least one kwarg: 'windowsize'. Other kwargs can be passed in using a dict
    provided in smoothfunckwargs. By default, this uses a Savitsky-Golay filter,
    a Gaussian filter is also provided but not used. Another good option would
    be an actual low-pass filter (generated using scipy.signal?) to remove all
    high frequency noise from the ACF.

    magarefluxes is True if the measurements provided in mags are actually
    fluxes, False otherwise.

    sigclip is the sigma to use when sigma-clipping the magnitude time series.

    Returns
    -------

    Returns a dictionary with results. dict['bestperiod'] is the estimated best
    period and dict['fitperiodrms'] is its estimated error. Other interesting
    things in the output include:

    - dict['acfresults']: all results from calculating the ACF. in particular,
      the unsmoothed ACF might be of interest: dict['acfresults']['acf'] and
      dict['acfresults']['lags'].

    - dict['lags'] and dict['acf'] contain the ACF after smoothing was applied.

    - dict['periods'] and dict['lspvals'] can be used to construct a
      pseudo-periodogram.

    - dict['naivebestperiod'] is obtained by multiplying the lag at the highest
      ACF peak with the cadence. This is usually close to the fit period
      (dict['fitbestperiod']), which is calculated by doing a fit to the lags
      vs. peak index relation as in McQuillan+ 2014.

    '''

    # get the ACF
    acfres = autocorr_magseries(
        times,
        mags,
        errs,
        maxlags=maxlags,
        fillgaps=fillgaps,
        forcetimebin=forcetimebin,
        sigclip=sigclip,
        magsarefluxes=magsarefluxes,
        filterwindow=filterwindow,
        verbose=verbose
    )

    xlags = acfres['lags']

    # smooth the ACF if requested
    if smoothacf and isinstance(smoothacf, int) and smoothacf > 0:

        if smoothfunckwargs is None:
            sfkwargs = {'windowsize':smoothacf}
        else:
            sfkwargs = smoothfunckwargs.copy()
            sfkwargs.update({'windowsize':smoothacf})

        xacf = smoothfunc(acfres['acf'], **sfkwargs)

    else:

        xacf = acfres['acf']

    # get the relative peak heights and fit best lag
    peakres = _get_acf_peakheights(xlags, xacf, npeaks=maxacfpeaks,
                                   searchinterval=int(smoothacf/2))

    # this is the best period's best ACF peak height
    bestlspval = peakres['bestpeakheight']

    try:

        # get the fit best lag from a linear fit to the peak index vs time(peak
        # lag) function as in McQillian+ (2014)
        fity = npconcatenate((
            [0.0, peakres['bestlag']],
            peakres['relpeaklags'][peakres['relpeaklags'] > peakres['bestlag']]
        ))
        fity = fity*acfres['cadence']
        fitx = nparange(fity.size)

        fitcoeffs, fitcovar = nppolyfit(fitx, fity, 1, cov=True)

        # fit best period is the gradient of fit
        fitbestperiod = fitcoeffs[0]
        bestperiodrms = npsqrt(fitcovar[0,0])  # from the covariance matrix

    except Exception as e:

        LOGWARNING('linear fit to time at each peak lag '
                   'value vs. peak number failed, '
                   'naively calculated ACF period may not be accurate')
        fitcoeffs = nparray([npnan, npnan])
        fitcovar = nparray([[npnan, npnan], [npnan, npnan]])
        fitbestperiod = npnan
        bestperiodrms = npnan
        raise

    # calculate the naive best period using delta_tau = lag * cadence
    naivebestperiod = peakres['bestlag']*acfres['cadence']

    if fitbestperiod < naivebestperiod:
        LOGWARNING('fit bestperiod = %.5f may be an alias, '
                   'naively calculated bestperiod is = %.5f' %
                   (fitbestperiod, naivebestperiod))

    if npisfinite(fitbestperiod):
        bestperiod = fitbestperiod
    else:
        bestperiod = naivebestperiod

    return {'bestperiod':bestperiod,
            'bestlspval':bestlspval,
            'nbestpeaks':maxacfpeaks,
            # for compliance with the common pfmethod API
            'nbestperiods':npconcatenate([
                [fitbestperiod],
                peakres['relpeaklags'][1:maxacfpeaks]*acfres['cadence']
            ]),
            'nbestlspvals':peakres['maxacfs'][:maxacfpeaks],
            'lspvals':xacf,
            'periods':xlags*acfres['cadence'],
            'acf':xacf,
            'lags':xlags,
            'method':'acf',
            'naivebestperiod':naivebestperiod,
            'fitbestperiod':fitbestperiod,
            'fitperiodrms':bestperiodrms,
            'periodfitcoeffs':fitcoeffs,
            'periodfitcovar':fitcovar,
            'kwargs':{'maxlags':maxlags,
                      'maxacfpeaks':maxacfpeaks,
                      'fillgaps':fillgaps,
                      'filterwindow':filterwindow,
                      'smoothacf':smoothacf,
                      'smoothfunckwargs':sfkwargs,
                      'magsarefluxes':magsarefluxes,
                      'sigclip':sigclip},
            'acfresults':acfres,
            'acfpeaks':peakres}
<|endoftext|>