<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    zifeiyu.models
    ~~~~~~~~~~~~~~~~~~~~

    The module provides the models for admin

    :copyright: (c) 2016 by zifeiyu.
    :license: MIT, see LICENSE for more details.
"""
from zifeiyu.constants import ADMIN_ID, EMAIL, PASSWORD_HASH, NICKNAME
from flask_login import UserMixin
from datetime import datetime
from zifeiyu.extensions import db
from zifeiyu.constants import POST_STATUS
import uuid
from zifeiyu.utils.encryption import generate_SHA1
from sqlalchemy.ext.hybrid import hybrid_property

class Admin(UserMixin):

    def __init__(self):
        self.__id = ADMIN_ID
        self.__nickname = NICKNAME
        self.__email = EMAIL
        self.__password_hash = PASSWORD_HASH

    @classmethod
    def authenticate(cls, email, password):
        admin = Admin()
        authenticate = False
        if email == EMAIL and generate_SHA1(password) == PASSWORD_HASH:
            authenticate = True
        return admin, authenticate

    def get_id(self):
        return self.__id

    def __unicode__(self):
        return self.__nickname

tags = db.Table('tags',
                db.Column('tag_id', db.String(50), db.ForeignKey('tag.id')),
                db.Column('post_id', db.String(50), db.ForeignKey('post.id'))
                )

class Tag(db.Model):

    __tablename__='tag'
    id = db.Column(db.String(50), primary_key=True)

    def __init__(self, tag):
        self.id = tag

    @hybrid_property
    def post_num(self):
        return len(self.posts)

class Post(db.Model):

    __tablename__ = 'post'
    id = db.Column(db.String(50), primary_key=True)
    title = db.Column(db.String(100))
    content = db.Column(db.Text)
    abstract = db.Column(db.Text)
    status = db.Column(db.String(10), default=dict(POST_STATUS)['DRAFT'])
    archive_id = db.Column(db.String(50), db.ForeignKey('archive.id'))
    column_id = db.Column(db.String(50), db.ForeignKey('column.id'))
    tags = db.relationship('Tag', secondary=tags, backref=db.backref('posts'))
    created_date = db.Column(db.DateTime)
    updated_date = db.Column(db.DateTime)
    publish_date = db.Column(db.DateTime)
    deleted_date = db.Column(db.DateTime)

    def __init__(self):
        pass

    @hybrid_property
    def local_created_date(self):
        return self.created_date.strftime("%Y-%m-%dT%H:%M:%S Z")

    @hybrid_property
    def local_updated_date(self):
        return self.updated_date.strftime("%Y-%m-%dT%H:%M:%S Z")

    @hybrid_property
    def local_publish_date(self):
        return self.publish_date.strftime("%Y-%m-%dT%H:%M:%S Z")

    def save(self, tag_list):
        self.updated_date = datetime.utcnow()
        # set post status
        statuses = dict(POST_STATUS)
        if self.status == statuses['PUBLISHED']:
            self.publish_date = datetime.utcnow()
        elif self.status == statuses['DELETED']:
            self.deleted_date = datetime.utcnow()

        # insert post
        if self.id is None:
            u = uuid.uuid1()
            self.id = str(uuid.uuid5(u, 'post'))
            self.created_date = datetime.utcnow()
            archive = Archive(datetime.now())
            db.session.merge(archive)
            # archive_id = datetime.now().strftime("%Y-%m")
            # archive = Archive.query.filter(Archive.id == archive_id).first()
            # if archive is None:
            #     # TODO: test
            #     archive = Archive(datetime.now())
            #     db.session.add(archive)
            #     db.session.commit()
            self.archive_id = archive.id
        self.updated_date = datetime.utcnow()
        if len(tag_list) > 0:
            post_tags_id = [t.id for t in self.tags]
            for tag in tag_list:
                if tag.id not in post_tags_id:
                    self.tags.append(tag)
        db.session.add(self)
        db.session.commit()

class Archive(db.Model):

    __tabname__ = 'archive'
    id = db.Column(db.String(50), primary_key=True)
    label = db.Column(db.String(15))
    posts = db.relationship('Post', backref="archive")
    created_date = db.Column(db.DateTime)
    updated_date = db.Column(db.DateTime)
    deleted_date = db.Column(db.DateTime)

    def __init__(self, archive_time):
        self.id = archive_time.strftime("%Y-%m")
        self.label = str(archive_time.year) + u"年" + \
            str(archive_time.month) + u"月"

    @hybrid_property
    def post_num(self):
        return len(self.posts)

    def save(self, label, operation):
        if self.id is None and operation == 'insert':
            self.created_date = datetime.utcnow()
        elif self.id is not None and operation == 'update':
            self.updated_date = datetime.utcnow()
        else:
            return False
        self.label = label
        db.session.add(self)
        db.session.commit()
        return True

class Column(db.Model):

    __tablename__ = 'column'
    id = db.Column(db.String(50), primary_key=True)
    posts = db.relationship('Post', backref="column")
    label = db.Column(db.String(15))
    created_date = db.Column(db.DateTime)
    updated_date = db.Column(db.DateTime)
    deleted_date = db.Column(db.DateTime)

    def __init__(self):
        pass

    @hybrid_property
    def post_num(self):
        return len(self.posts)

    @hybrid_property
    def local_created_date(self):
        return self.created_date.strftime("%Y-%m-%dT%H:%M:%S Z")

    @hybrid_property
    def local_updated_date(self):
        return self.updated_date.strftime("%Y-%m-%dT%H:%M:%S Z")

    @classmethod
    def getColumnList(self):
        choices = [(u'', u'未选择')]
        for k, v in db.session.query(Column.id, Column.label).all():
            choices.append((k, v))
        return choices

    def save(self, operation):
        if self.id is None and operation == 'insert':
            u = uuid.uuid1()
            self.id = str(uuid.uuid5(u, 'column'))
            self.created_date = datetime.utcnow()
        elif self.id is None and operation != 'update':
            return False
        self.updated_date = datetime.utcnow()
        db.session.add(self)
        db.session.commit()
        return True

class Weibo(db.Model):

    __tablename__ = 'weibo'
    id = db.Column(db.String(100), primary_key=True)
    access_token = db.Column(db.String(100))
    screen_name = db.Column(db.String(100))
    expires = db.Column(db.Float)
    authenticate_date = db.Column(db.DateTime)
    profile_image_url = db.Column(db.String(150))
    created_date = db.Column(db.DateTime)
    updated_date = db.Column(db.DateTime)
    messages = db.relationship('Message', backref="weibo")
    message_replies = db.relationship('MessageRelpy', backref="weibo")

    def __init__(self, id, access_token, expires, screen_name, profile_image_url):
        self.id = id
        self.access_token = access_token
        self.expires = expires
        self.screen_name = screen_name
        self.profile_image_url = profile_image_url

    def save(self):
        db.session.add(self)
        db.session.commit()

class Message(db.Model):

    __tablename__ = 'message'
    id = db.Column(db.String(100), primary_key=True)
    created_date = db.Column(db.DateTime)
    content = db.Column(db.String(130))
    replies = db.relationship('MessageRelpy', backref="message")
    weibo_id = db.Column(db.String(100), db.ForeignKey('weibo.id'))
    is_deleted = db.Column(db.Boolean)

    def __init__(self, content, weibo_id=None, message_id=None):
        self.content = content
        self.weibo_id = weibo_id
        self.message_id = message_id

    def save(self):
        u = uuid.uuid1()
        self.id = str(uuid.uuid5(u, 'message'))
        self.created_date = datetime.utcnow()
        db.session.add(self)
        db.session.commit()

    @hybrid_property
    def reply_num(self):
        return len(self.replies)

class MessageRelpy(db.Model):

    __tablename__ = 'message_reply'
    id = db.Column(db.String(100), primary_key=True)
    created_date = db.Column(db.DateTime)
    content = db.Column(db.String(130))
    message_id = db.Column(db.String(100), db.ForeignKey('message.id'))
    reply_id = db.Column(db.String(100), db.ForeignKey('message_reply.id'))
    weibo_id = db.Column(db.String(100), db.ForeignKey('weibo.id'))
    is_from_admin = db.Column(db.Boolean)
    is_deleted = db.Column(db.Boolean)

    def __init__(self, content, weibo_id=None, message_id=None):
        self.content = content

    def save(self):
        u = uuid.uuid1()
        self.id = str(uuid.uuid5(u, 'message_reply'))
        self.created_date = datetime.utcnow()
        db.session.add(self)
        db.session.commit()

<commit_msg>weibouser save changed<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    zifeiyu.models
    ~~~~~~~~~~~~~~~~~~~~

    The module provides the models for admin

    :copyright: (c) 2016 by zifeiyu.
    :license: MIT, see LICENSE for more details.
"""
from zifeiyu.constants import ADMIN_ID, EMAIL, PASSWORD_HASH, NICKNAME
from flask_login import UserMixin
from datetime import datetime
from zifeiyu.extensions import db
from zifeiyu.constants import POST_STATUS
import uuid
from zifeiyu.utils.encryption import generate_SHA1
from sqlalchemy.ext.hybrid import hybrid_property

class Admin(UserMixin):

    def __init__(self):
        self.__id = ADMIN_ID
        self.__nickname = NICKNAME
        self.__email = EMAIL
        self.__password_hash = PASSWORD_HASH

    @classmethod
    def authenticate(cls, email, password):
        admin = Admin()
        authenticate = False
        if email == EMAIL and generate_SHA1(password) == PASSWORD_HASH:
            authenticate = True
        return admin, authenticate

    def get_id(self):
        return self.__id

    def __unicode__(self):
        return self.__nickname

tags = db.Table('tags',
                db.Column('tag_id', db.String(50), db.ForeignKey('tag.id')),
                db.Column('post_id', db.String(50), db.ForeignKey('post.id'))
                )

class Tag(db.Model):

    __tablename__='tag'
    id = db.Column(db.String(50), primary_key=True)

    def __init__(self, tag):
        self.id = tag

    @hybrid_property
    def post_num(self):
        return len(self.posts)

class Post(db.Model):

    __tablename__ = 'post'
    id = db.Column(db.String(50), primary_key=True)
    title = db.Column(db.String(100))
    content = db.Column(db.Text)
    abstract = db.Column(db.Text)
    status = db.Column(db.String(10), default=dict(POST_STATUS)['DRAFT'])
    archive_id = db.Column(db.String(50), db.ForeignKey('archive.id'))
    column_id = db.Column(db.String(50), db.ForeignKey('column.id'))
    tags = db.relationship('Tag', secondary=tags, backref=db.backref('posts'))
    created_date = db.Column(db.DateTime)
    updated_date = db.Column(db.DateTime)
    publish_date = db.Column(db.DateTime)
    deleted_date = db.Column(db.DateTime)

    def __init__(self):
        pass

    @hybrid_property
    def local_created_date(self):
        return self.created_date.strftime("%Y-%m-%dT%H:%M:%S Z")

    @hybrid_property
    def local_updated_date(self):
        return self.updated_date.strftime("%Y-%m-%dT%H:%M:%S Z")

    @hybrid_property
    def local_publish_date(self):
        return self.publish_date.strftime("%Y-%m-%dT%H:%M:%S Z")

    def save(self, tag_list):
        self.updated_date = datetime.utcnow()
        # set post status
        statuses = dict(POST_STATUS)
        if self.status == statuses['PUBLISHED']:
            self.publish_date = datetime.utcnow()
        elif self.status == statuses['DELETED']:
            self.deleted_date = datetime.utcnow()

        # insert post
        if self.id is None:
            u = uuid.uuid1()
            self.id = str(uuid.uuid5(u, 'post'))
            self.created_date = datetime.utcnow()
            archive = Archive(datetime.now())
            db.session.merge(archive)
            # archive_id = datetime.now().strftime("%Y-%m")
            # archive = Archive.query.filter(Archive.id == archive_id).first()
            # if archive is None:
            #     # TODO: test
            #     archive = Archive(datetime.now())
            #     db.session.add(archive)
            #     db.session.commit()
            self.archive_id = archive.id
        self.updated_date = datetime.utcnow()
        if len(tag_list) > 0:
            post_tags_id = [t.id for t in self.tags]
            for tag in tag_list:
                if tag.id not in post_tags_id:
                    self.tags.append(tag)
        db.session.add(self)
        db.session.commit()

class Archive(db.Model):

    __tabname__ = 'archive'
    id = db.Column(db.String(50), primary_key=True)
    label = db.Column(db.String(15))
    posts = db.relationship('Post', backref="archive")
    created_date = db.Column(db.DateTime)
    updated_date = db.Column(db.DateTime)
    deleted_date = db.Column(db.DateTime)

    def __init__(self, archive_time):
        self.id = archive_time.strftime("%Y-%m")
        self.label = str(archive_time.year) + u"年" + \
            str(archive_time.month) + u"月"

    @hybrid_property
    def post_num(self):
        return len(self.posts)

    def save(self, label, operation):
        if self.id is None and operation == 'insert':
            self.created_date = datetime.utcnow()
        elif self.id is not None and operation == 'update':
            self.updated_date = datetime.utcnow()
        else:
            return False
        self.label = label
        db.session.add(self)
        db.session.commit()
        return True

class Column(db.Model):

    __tablename__ = 'column'
    id = db.Column(db.String(50), primary_key=True)
    posts = db.relationship('Post', backref="column")
    label = db.Column(db.String(15))
    created_date = db.Column(db.DateTime)
    updated_date = db.Column(db.DateTime)
    deleted_date = db.Column(db.DateTime)

    def __init__(self):
        pass

    @hybrid_property
    def post_num(self):
        return len(self.posts)

    @hybrid_property
    def local_created_date(self):
        return self.created_date.strftime("%Y-%m-%dT%H:%M:%S Z")

    @hybrid_property
    def local_updated_date(self):
        return self.updated_date.strftime("%Y-%m-%dT%H:%M:%S Z")

    @classmethod
    def getColumnList(self):
        choices = [(u'', u'未选择')]
        for k, v in db.session.query(Column.id, Column.label).all():
            choices.append((k, v))
        return choices

    def save(self, operation):
        if self.id is None and operation == 'insert':
            u = uuid.uuid1()
            self.id = str(uuid.uuid5(u, 'column'))
            self.created_date = datetime.utcnow()
        elif self.id is None and operation != 'update':
            return False
        self.updated_date = datetime.utcnow()
        db.session.add(self)
        db.session.commit()
        return True

class Weibo(db.Model):

    __tablename__ = 'weibo'
    id = db.Column(db.String(100), primary_key=True)
    access_token = db.Column(db.String(100))
    screen_name = db.Column(db.String(100))
    expires = db.Column(db.Float)
    authenticate_date = db.Column(db.DateTime)
    profile_image_url = db.Column(db.String(150))
    created_date = db.Column(db.DateTime)
    updated_date = db.Column(db.DateTime)
    messages = db.relationship('Message', backref="weibo")
    message_replies = db.relationship('MessageRelpy', backref="weibo")

    def __init__(self, id, access_token, expires, screen_name, profile_image_url):
        self.id = id
        self.access_token = access_token
        self.expires = expires
        self.screen_name = screen_name
        self.profile_image_url = profile_image_url

    def save(self):
        user = db.session.query(Weibo).filter_by(id == self.id).first()
        if user is not None:
            user.update(self)
        else:
            db.session.add(self)
        db.session.commit()

class Message(db.Model):

    __tablename__ = 'message'
    id = db.Column(db.String(100), primary_key=True)
    created_date = db.Column(db.DateTime)
    content = db.Column(db.String(130))
    replies = db.relationship('MessageRelpy', backref="message")
    weibo_id = db.Column(db.String(100), db.ForeignKey('weibo.id'))
    is_deleted = db.Column(db.Boolean)

    def __init__(self, content, weibo_id=None, message_id=None):
        self.content = content
        self.weibo_id = weibo_id
        self.message_id = message_id

    def save(self):
        u = uuid.uuid1()
        self.id = str(uuid.uuid5(u, 'message'))
        self.created_date = datetime.utcnow()
        db.session.add(self)
        db.session.commit()

    @hybrid_property
    def reply_num(self):
        return len(self.replies)

class MessageRelpy(db.Model):

    __tablename__ = 'message_reply'
    id = db.Column(db.String(100), primary_key=True)
    created_date = db.Column(db.DateTime)
    content = db.Column(db.String(130))
    message_id = db.Column(db.String(100), db.ForeignKey('message.id'))
    reply_id = db.Column(db.String(100), db.ForeignKey('message_reply.id'))
    weibo_id = db.Column(db.String(100), db.ForeignKey('weibo.id'))
    is_from_admin = db.Column(db.Boolean)
    is_deleted = db.Column(db.Boolean)

    def __init__(self, content, weibo_id=None, message_id=None):
        self.content = content

    def save(self):
        u = uuid.uuid1()
        self.id = str(uuid.uuid5(u, 'message_reply'))
        self.created_date = datetime.utcnow()
        db.session.add(self)
        db.session.commit()

<|endoftext|>


<commit_before>import logging
import os

import onedrivesdk.error

from . import update_mtime
from ..od_api_helper import item_request_call

class UploadFileTask(update_mtime.UpdateTimestampTask):

    def __init__(self, repo, task_pool, parent_dir_request, parent_relpath, item_name):
        """
        :param onedrived.od_repo.OneDriveLocalRepository repo:
        :param onedrived.od_task.TaskPool task_pool:
        :param onedrivesdk.request.item_request_builder.ItemRequestBuilder parent_dir_request:
        :param str parent_relpath:
        :param str item_name:
        """
        super().__init__(repo, task_pool, parent_relpath, item_name)
        self.parent_dir_request = parent_dir_request

    def __repr__(self):
        return type(self).__name__ + '(%s)' % self.local_abspath

    def handle(self):
        logging.info('Uploading file "%s" to OneDrive.', self.local_abspath)
        try:
            item_stat = os.stat(self.local_abspath)
            returned_item = item_request_call(
                self.repo, self.parent_dir_request.children[self.item_name].upload, self.local_abspath)
            self.update_timestamp_and_record(returned_item, item_stat)
            logging.info('Finished uploading file "%s".', self.local_abspath)
            return True
        except (onedrivesdk.error.OneDriveError, OSError) as e:
            logging.error('Error uploading file "%s": %s.', self.local_abspath, e)
            return False
<commit_msg>[tasks] Upload large files by session.<commit_after>import logging
import os

import onedrivesdk.error

from . import update_mtime
from ..od_api_helper import item_request_call

class UploadFileTask(update_mtime.UpdateTimestampTask):

    # If file is smaller than this size (in Bytes) use HTTP PUT method to upload. Otherwise upload in chunks
    # using Session API (https://dev.onedrive.com/items/upload_large_files.htm).
    PUT_FILE_SIZE_THRESHOLD_BYTES = 10 << 20

    def __init__(self, repo, task_pool, parent_dir_request, parent_relpath, item_name):
        """
        :param onedrived.od_repo.OneDriveLocalRepository repo:
        :param onedrived.od_task.TaskPool task_pool:
        :param onedrivesdk.request.item_request_builder.ItemRequestBuilder parent_dir_request:
        :param str parent_relpath:
        :param str item_name:
        """
        super().__init__(repo, task_pool, parent_relpath, item_name)
        self.parent_dir_request = parent_dir_request

    def __repr__(self):
        return type(self).__name__ + '(%s)' % self.local_abspath

    def update_progress(self, curr_part, total_part):
        logging.info('Uploading file "%s": Part %d / %d.', self.local_abspath, curr_part, total_part)

    def handle(self):
        logging.info('Uploading file "%s" to OneDrive.', self.local_abspath)
        try:
            item_stat = os.stat(self.local_abspath)
            item_request = self.parent_dir_request.children[self.item_name]
            if item_stat.st_size < self.PUT_FILE_SIZE_THRESHOLD_BYTES:
                returned_item = item_request_call(self.repo, item_request.upload, self.local_abspath)
            else:
                returned_item = item_request_call(self.repo, item_request.upload_async,
                                                  self.local_abspath, upload_status=self.update_progress)
            self.update_timestamp_and_record(returned_item, item_stat)
            logging.info('Finished uploading file "%s".', self.local_abspath)
            return True
        except (onedrivesdk.error.OneDriveError, OSError) as e:
            logging.error('Error uploading file "%s": %s.', self.local_abspath, e)
            return False
<|endoftext|>


<commit_before>import math
import urllib.request as urlr
import pickle

class WeatherStation:
    def __init__(self):
        self.attributes = ["station_name", "latitude", "longitude", "xml_url"]
        self.station_info = {}

    def print_station_info(self):
        for attribute in self.attributes:
            print(attribute + ": ", self.station_info[attribute])

    def to_string(self):
        output = ""
        for attribute in self.attributes:
            output += attribute + ": " \
                      + self.station_info[attribute] + '\n'
        return output
class ZipRegion:
    def __init__(self, state, city, zip_code, latitude, longitude):
        self.state = state
        self.city = city
        self.zip_code = zip_code
        self.latitude = latitude
        self.longitude = longitude
    def print_info(self):
        print(self.state)
        print(self.city)
        print(self.zip_code)
        print(self.latitude)
        print(self.longitude)
        print()
def parse_zip_region_file(filename):
    region_list = []
    with open(filename,'r') as f:
        zip_regions = f.read().split('\n')
        #first line is header
        #State City Postal Latitude Longitude
        i = 1
        zip_regions = zip_regions[1:]
        for region_info in zip_regions:
            region_info = region_info.split(',')
            #remove Null strings
            region_info = list(filter(bool,region_info))
            if len(region_info) == 0:
                continue
            if len(region_info) != 5:
                print("Malformed sequence", i)
                print(region_info)
                continue
            state = region_info[0]
            city = region_info[1]
            zip_code = region_info[2]
            latitude = region_info[3]
            longitude = region_info[4]
            region = ZipRegion(state, city, zip_code, latitude, longitude)
            region_list.append(region)
            i = i+1
        print(i,"zip codes on file")
    return region_list

def parse_station_file(weather_xml):
    weather_station_list = []
    #last last after last station is just a footer
    for station in weather_xml.split("</station>")[:-1]:
        weather_station = WeatherStation()
        station_info = station.split("<station>")[-1]
        station_info = station_info.split("\n")
        for line in station_info:
            for attribute in weather_station.attributes:
                if '<' + attribute + '>' in line:
                    start_index = line.index('>') + 1
                    end_index = line.index('<', start_index)
                    weather_station.station_info[attribute] \
                        = line[start_index:end_index]
        weather_station_list.append(weather_station)
    print(len(weather_station_list), "weather stations found.")
    return weather_station_list

def get_station_file():
    station_index = "http://w1.weather.gov/xml/current_obs/index.xml"

    http_rsp_obj = urlr.urlopen(station_index)

    #reads as bytes
    weather_xml = http_rsp_obj.read()

    #turn weather_xml into a string from bytes
    weather_xml = weather_xml.decode(encoding='UTF-8')
    return weather_xml

# TODO update to a balanced tree
# TODO: restructure class to get rid of annoying checks for Nones
# TODO: should all attributes be kept as strings?  Even lat and long?
# Create a binary search tree from list of stations
class binary_tree:
    def __init__(self, station, sort_field):
        self.data = station
        self.sort_field = sort_field
        self.left = None
        self.right = None

    def insert(self, station):
        # place node right if greater
        if float(station.station_info[self.sort_field]) > \
         float(self.data.station_info[self.sort_field]):
            if self.right == None:
                self.right = binary_tree(station, self.sort_field)
            else:
                self.right.insert(station)
        #place node left if less
        else:
            if self.left == None:
                self.left = binary_tree(station, self.sort_field)
            else:
                self.left.insert(station)
    def size(self):
        if self.left == None:
            left_size = 0
        else:
            left_size = self.left.size()
        if self.right == None:
            right_size = 0
        else:
            right_size = self.right.size()
        return 1 + left_size + right_size
    # Searches the tree for stations within a specified distance
    # returns a list of stations that are within the specified distance
    def find_station(self, value, distance):
        if float(self.data.station_info[self.sort_field]) < (value - distance):
            if self.right is not None:
                return self.right.find_station(value, distance)
            else:
                return []
        elif float(self.data.station_info[self.sort_field]) > (value + distance):
            if self.left is not None:
                return self.left.find_station(value, distance)
            else:
                return []
        else:
            right_values = []
            left_values = []
            if self.right is not None:
                right_values = self.right.find_station(value, distance)
            if self.left is not None:
                left_values = self.left.find_station(value, distance)
            return [self.data] + right_values + left_values

# converts linear lists into binary serach trees
def station_list_to_tree(station_list, sort_field):
    station_tree = binary_tree(station_list[0], sort_field)
    for station in station_list[1:]:
        station_tree.insert(station)
    return station_tree

#Uses a binary search tree to locate close stations
def better_map_zip_to_station(region_list, weather_station_list):
    zip_to_station_map = {}
    # chane linear lists into binary search trees
    lat_station_tree = station_list_to_tree(weather_station_list, "latitude")
    long_station_tree = station_list_to_tree(weather_station_list, "longitude")
    print("lat map size:", lat_station_tree.size())
    print("long map size:", long_station_tree.size())
    for region in region_list:
        # make a list of all stations within distance degrees
        distance = 0.1
        close_stations = []
        # check for stations in a widening circle
        while(close_stations == []):
            long_stations = long_station_tree.find_station(float(region.longitude), distance)
            lat_stations = lat_station_tree.find_station(float(region.latitude), distance)
            close_stations = list(set(lat_stations) & set(long_stations))
            distance = distance + 0.1
        # start with large initial distance
        shortest_distance = 1e20
        for close_station in close_stations:
            distance = (float(region.latitude) - float(close_station.station_info["latitude"]))**2 + \
                               (float(region.longitude)- float(close_station.station_info["longitude"]))**2
            if distance < shortest_distance:
                zip_to_station_map[region.zip_code] \
                    = close_station.station_info["xml_url"]
                shortest_distance = distance
    return zip_to_station_map

def map_zip_to_station(region_list, weather_station_list):
    zip_to_station_map = {}
    for region in region_list:
        shortest_distance = 1000000
        for station in weather_station_list:
            # Using distance squared because actual distance doesn't matter.
            # only what is closest
            distance = (float(region.latitude) - float(station.station_info["latitude"]))**2 + \
                               (float(region.longitude)- float(station.station_info["longitude"]))**2
            if distance < shortest_distance:
                zip_to_station_map[region.zip_code] \
                    = station.station_info["xml_url"]
                shortest_distance = distance
    return zip_to_station_map

def main():
    region_list = parse_zip_region_file("cityzip.csv")
    weather_station_list = parse_station_file(get_station_file())
    zip_to_station_map = better_map_zip_to_station(region_list, weather_station_list)
    with open("zip_to_station_map","wb") as f:
        pickle.dump(zip_to_station_map, f)

if __name__ == "__main__":
    main()
<commit_msg>Adjusted waether station search to be exponentional widening function instead of linear<commit_after>import math
import urllib.request as urlr
import pickle

class WeatherStation:
    def __init__(self):
        self.attributes = ["station_name", "latitude", "longitude", "xml_url"]
        self.station_info = {}

    def print_station_info(self):
        for attribute in self.attributes:
            print(attribute + ": ", self.station_info[attribute])

    def to_string(self):
        output = ""
        for attribute in self.attributes:
            output += attribute + ": " \
                      + self.station_info[attribute] + '\n'
        return output
class ZipRegion:
    def __init__(self, state, city, zip_code, latitude, longitude):
        self.state = state
        self.city = city
        self.zip_code = zip_code
        self.latitude = latitude
        self.longitude = longitude
    def print_info(self):
        print(self.state)
        print(self.city)
        print(self.zip_code)
        print(self.latitude)
        print(self.longitude)
        print()
def parse_zip_region_file(filename):
    region_list = []
    with open(filename,'r') as f:
        zip_regions = f.read().split('\n')
        #first line is header
        #State City Postal Latitude Longitude
        i = 1
        zip_regions = zip_regions[1:]
        for region_info in zip_regions:
            region_info = region_info.split(',')
            #remove Null strings
            region_info = list(filter(bool,region_info))
            if len(region_info) == 0:
                continue
            if len(region_info) != 5:
                print("Malformed sequence", i)
                print(region_info)
                continue
            state = region_info[0]
            city = region_info[1]
            zip_code = region_info[2]
            latitude = region_info[3]
            longitude = region_info[4]
            region = ZipRegion(state, city, zip_code, latitude, longitude)
            region_list.append(region)
            i = i+1
        print(i,"zip codes on file")
    return region_list

def parse_station_file(weather_xml):
    weather_station_list = []
    #last last after last station is just a footer
    for station in weather_xml.split("</station>")[:-1]:
        weather_station = WeatherStation()
        station_info = station.split("<station>")[-1]
        station_info = station_info.split("\n")
        for line in station_info:
            for attribute in weather_station.attributes:
                if '<' + attribute + '>' in line:
                    start_index = line.index('>') + 1
                    end_index = line.index('<', start_index)
                    weather_station.station_info[attribute] \
                        = line[start_index:end_index]
        weather_station_list.append(weather_station)
    print(len(weather_station_list), "weather stations found.")
    return weather_station_list

def get_station_file():
    station_index = "http://w1.weather.gov/xml/current_obs/index.xml"

    http_rsp_obj = urlr.urlopen(station_index)

    #reads as bytes
    weather_xml = http_rsp_obj.read()

    #turn weather_xml into a string from bytes
    weather_xml = weather_xml.decode(encoding='UTF-8')
    return weather_xml

# TODO update to a balanced tree
# TODO: restructure class to get rid of annoying checks for Nones
# TODO: should all attributes be kept as strings?  Even lat and long?
# Create a binary search tree from list of stations
class binary_tree:
    def __init__(self, station, sort_field):
        self.data = station
        self.sort_field = sort_field
        self.left = None
        self.right = None

    def insert(self, station):
        # place node right if greater
        if float(station.station_info[self.sort_field]) > \
         float(self.data.station_info[self.sort_field]):
            if self.right == None:
                self.right = binary_tree(station, self.sort_field)
            else:
                self.right.insert(station)
        #place node left if less
        else:
            if self.left == None:
                self.left = binary_tree(station, self.sort_field)
            else:
                self.left.insert(station)
    def size(self):
        if self.left == None:
            left_size = 0
        else:
            left_size = self.left.size()
        if self.right == None:
            right_size = 0
        else:
            right_size = self.right.size()
        return 1 + left_size + right_size
    # Searches the tree for stations within a specified distance
    # returns a list of stations that are within the specified distance
    def find_station(self, value, distance):
        if float(self.data.station_info[self.sort_field]) < (value - distance):
            if self.right is not None:
                return self.right.find_station(value, distance)
            else:
                return []
        elif float(self.data.station_info[self.sort_field]) > (value + distance):
            if self.left is not None:
                return self.left.find_station(value, distance)
            else:
                return []
        else:
            right_values = []
            left_values = []
            if self.right is not None:
                right_values = self.right.find_station(value, distance)
            if self.left is not None:
                left_values = self.left.find_station(value, distance)
            return [self.data] + right_values + left_values

# converts linear lists into binary serach trees
def station_list_to_tree(station_list, sort_field):
    station_tree = binary_tree(station_list[0], sort_field)
    for station in station_list[1:]:
        station_tree.insert(station)
    return station_tree

#Uses a binary search tree to locate close stations
def better_map_zip_to_station(region_list, weather_station_list):
    zip_to_station_map = {}
    # chane linear lists into binary search trees
    lat_station_tree = station_list_to_tree(weather_station_list, "latitude")
    long_station_tree = station_list_to_tree(weather_station_list, "longitude")
    print("lat map size:", lat_station_tree.size())
    print("long map size:", long_station_tree.size())
    count = 0
    for region in region_list:
        # make a list of all stations within distance degrees
        distance = 0.1
        close_stations = []
        # check for stations in a widening circle
        while(close_stations == []):
            long_stations = long_station_tree.find_station(float(region.longitude), distance)
            lat_stations = lat_station_tree.find_station(float(region.latitude), distance)
            close_stations = list(set(lat_stations) & set(long_stations))
            distance = distance*2
        # start with large initial distance
        shortest_distance = 1e20
        for close_station in close_stations:
            distance = (float(region.latitude) - float(close_station.station_info["latitude"]))**2 + \
                               (float(region.longitude)- float(close_station.station_info["longitude"]))**2
            if distance < shortest_distance:
                zip_to_station_map[region.zip_code] \
                    = close_station.station_info["xml_url"]
                shortest_distance = distance
        count = count + 1
        if count % 1000 == 0:
            print(count, "Stations matched")
    return zip_to_station_map

def map_zip_to_station(region_list, weather_station_list):
    zip_to_station_map = {}
    for region in region_list:
        shortest_distance = 1000000
        for station in weather_station_list:
            # Using distance squared because actual distance doesn't matter.
            # only what is closest
            distance = (float(region.latitude) - float(station.station_info["latitude"]))**2 + \
                               (float(region.longitude)- float(station.station_info["longitude"]))**2
            if distance < shortest_distance:
                zip_to_station_map[region.zip_code] \
                    = station.station_info["xml_url"]
                shortest_distance = distance
    return zip_to_station_map

def main():
    region_list = parse_zip_region_file("cityzip.csv")
    weather_station_list = parse_station_file(get_station_file())
    zip_to_station_map = better_map_zip_to_station(region_list, weather_station_list)
    with open("zip_to_station_map","wb") as f:
        pickle.dump(zip_to_station_map, f)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import logging

from tornado import gen, concurrent

from zoonado import protocol, exc

from .recipes.proxy import RecipeProxy
from .session import Session
from .transaction import Transaction
from .features import Features

log = logging.getLogger(__name__)

class Zoonado(object):

    def __init__(
            self,
            servers,
            chroot=None,
            session_timeout=10,
            default_acl=None,
            retry_policy=None,
            allow_read_only=False,
    ):
        self.chroot = None
        if chroot:
            self.chroot = self.normalize_path(chroot)
            log.info("Using chroot '%s'", self.chroot)

        self.session = Session(
            servers, session_timeout, retry_policy, allow_read_only
        )

        self.default_acl = default_acl or [protocol.UNRESTRICTED_ACCESS]

        self.stat_cache = {}

        self.recipes = RecipeProxy(self)

    def normalize_path(self, path):
        if self.chroot:
            path = "/".join([self.chroot, path])

        normalized = "/".join([
            name for name in path.split("/")
            if name
        ])

        return "/" + normalized

    def denormalize_path(self, path):
        if self.chroot and path.startswith(self.chroot):
            path = path[len(self.chroot):]

        return path

    @gen.coroutine
    def start(self):
        yield self.session.start()

        if self.chroot:
            yield self.ensure_path("/")

    @property
    def features(self):
        if self.session.conn:
            return Features(self.session.conn.version_info)
        else:
            return Features((0, 0, 0))

    @gen.coroutine
    def send(self, request):
        response = yield self.session.send(request)

        if getattr(request, "path", None) and getattr(response, "stat", None):
            self.stat_cache[
                self.denormalize_path(request.path)
            ] = response.stat

        raise gen.Return(response)

    @gen.coroutine
    def close(self):
        yield self.session.close()

    def wait_for_event(self, event_type, path):
        path = self.normalize_path(path)

        f = concurrent.Future()

        def set_future(_):
            if not f.done():
                f.set_result(None)
            self.session.remove_watch_callback(event_type, path, set_future)

        self.session.add_watch_callback(event_type, path, set_future)

        return f

    @gen.coroutine
    def exists(self, path, watch=False):
        path = self.normalize_path(path)

        try:
            yield self.send(protocol.ExistsRequest(path=path, watch=watch))
        except exc.NoNode:
            raise gen.Return(False)

        raise gen.Return(True)

    @gen.coroutine
    def create(
            self, path, data=None, acl=None,
            ephemeral=False, sequential=False, container=False,
    ):
        if container and not self.features.containers:
            raise ValueError("Cannot create container, feature unavailable.")

        path = self.normalize_path(path)
        acl = acl or self.default_acl

        if self.features.create_with_stat:
            request_class = protocol.Create2Request
        else:
            request_class = protocol.CreateRequest

        request = request_class(path=path, data=data, acl=acl)
        request.set_flags(ephemeral, sequential, container)

        response = yield self.send(request)

        raise gen.Return(self.denormalize_path(response.path))

    @gen.coroutine
    def ensure_path(self, path, acl=None):
        path = self.normalize_path(path)

        acl = acl or self.default_acl

        paths_to_make = []
        for segment in path[1:].split("/"):
            if not paths_to_make:
                paths_to_make.append("/" + segment)
                continue

            paths_to_make.append("/".join([paths_to_make[-1], segment]))

        while paths_to_make:
            path = paths_to_make[0]

            if self.features.create_with_stat:
                request = protocol.Create2Request(path=path, acl=acl)
            else:
                request = protocol.CreateRequest(path=path, acl=acl)
            request.set_flags(
                ephemeral=False, sequential=False,
                container=self.features.containers
            )

            try:
                yield self.send(request)
            except exc.NodeExists:
                pass

            paths_to_make.pop(0)

    @gen.coroutine
    def delete(self, path, force=False):
        path = self.normalize_path(path)

        if not force and path in self.stat_cache:
            version = self.stat_cache[path].version
        else:
            version = -1

        yield self.send(protocol.DeleteRequest(path=path, version=version))

    @gen.coroutine
    def get_data(self, path, watch=False):
        path = self.normalize_path(path)

        response = yield self.send(
            protocol.GetDataRequest(path=path, watch=watch)
        )
        raise gen.Return(response.data)

    @gen.coroutine
    def set_data(self, path, data, force=False):
        path = self.normalize_path(path)

        if not force and path in self.stat_cache:
            version = self.stat_cache[path].version
        else:
            version = -1

        yield self.send(
            protocol.SetDataRequest(path=path, data=data, version=version)
        )

    @gen.coroutine
    def get_children(self, path, watch=False):
        path = self.normalize_path(path)

        response = yield self.send(
            protocol.GetChildren2Request(path=path, watch=watch)
        )
        raise gen.Return(response.children)

    @gen.coroutine
    def get_acl(self, path):
        path = self.normalize_path(path)

        response = yield self.send(protocol.GetACLRequest(path=path))
        raise gen.Return(response.acl)

    @gen.coroutine
    def set_acl(self, path, acl, force=False):
        path = self.normalize_path(path)

        if not force and path in self.stat_cache:
            version = self.stat_cache[path].version
        else:
            version = -1

        yield self.send(
            protocol.SetACLRequest(path=path, acl=acl, version=version)
        )

    def begin_transaction(self):
        return Transaction(self)
<commit_msg>Factor znode path normalization to a decorator.<commit_after>from __future__ import unicode_literals

import functools
import logging

from tornado import gen, concurrent

from zoonado import protocol, exc

from .recipes.proxy import RecipeProxy
from .session import Session
from .transaction import Transaction
from .features import Features

log = logging.getLogger(__name__)

def znode_method(fn):

    @functools.wraps(fn)
    def wrapper(client, path, *args, **kwargs):
        path = client.normalize_path(path)

        return fn(client, path, *args, **kwargs)

    return wrapper

class Zoonado(object):

    def __init__(
            self,
            servers,
            chroot=None,
            session_timeout=10,
            default_acl=None,
            retry_policy=None,
            allow_read_only=False,
    ):
        self.chroot = None
        if chroot:
            self.chroot = self.normalize_path(chroot)
            log.info("Using chroot '%s'", self.chroot)

        self.session = Session(
            servers, session_timeout, retry_policy, allow_read_only
        )

        self.default_acl = default_acl or [protocol.UNRESTRICTED_ACCESS]

        self.stat_cache = {}

        self.recipes = RecipeProxy(self)

    def normalize_path(self, path):
        if self.chroot:
            path = "/".join([self.chroot, path])

        normalized = "/".join([
            name for name in path.split("/")
            if name
        ])

        return "/" + normalized

    def denormalize_path(self, path):
        if self.chroot and path.startswith(self.chroot):
            path = path[len(self.chroot):]

        return path

    @gen.coroutine
    def start(self):
        yield self.session.start()

        if self.chroot:
            yield self.ensure_path("/")

    @property
    def features(self):
        if self.session.conn:
            return Features(self.session.conn.version_info)
        else:
            return Features((0, 0, 0))

    @gen.coroutine
    def send(self, request):
        response = yield self.session.send(request)

        if getattr(request, "path", None) and getattr(response, "stat", None):
            self.stat_cache[
                self.denormalize_path(request.path)
            ] = response.stat

        raise gen.Return(response)

    @gen.coroutine
    def close(self):
        yield self.session.close()

    def wait_for_event(self, event_type, path):
        path = self.normalize_path(path)

        f = concurrent.Future()

        def set_future(_):
            if not f.done():
                f.set_result(None)
            self.session.remove_watch_callback(event_type, path, set_future)

        self.session.add_watch_callback(event_type, path, set_future)

        return f

    @gen.coroutine
    @znode_method
    def exists(self, path, watch=False):
        try:
            yield self.send(protocol.ExistsRequest(path=path, watch=watch))
        except exc.NoNode:
            raise gen.Return(False)

        raise gen.Return(True)

    @gen.coroutine
    @znode_method
    def create(
            self, path, data=None, acl=None,
            ephemeral=False, sequential=False, container=False,
    ):
        if container and not self.features.containers:
            raise ValueError("Cannot create container, feature unavailable.")

        acl = acl or self.default_acl

        if self.features.create_with_stat:
            request_class = protocol.Create2Request
        else:
            request_class = protocol.CreateRequest

        request = request_class(path=path, data=data, acl=acl)
        request.set_flags(ephemeral, sequential, container)

        response = yield self.send(request)

        raise gen.Return(self.denormalize_path(response.path))

    @gen.coroutine
    @znode_method
    def ensure_path(self, path, acl=None):
        acl = acl or self.default_acl

        paths_to_make = []
        for segment in path[1:].split("/"):
            if not paths_to_make:
                paths_to_make.append("/" + segment)
                continue

            paths_to_make.append("/".join([paths_to_make[-1], segment]))

        while paths_to_make:
            path = paths_to_make[0]

            if self.features.create_with_stat:
                request = protocol.Create2Request(path=path, acl=acl)
            else:
                request = protocol.CreateRequest(path=path, acl=acl)
            request.set_flags(
                ephemeral=False, sequential=False,
                container=self.features.containers
            )

            try:
                yield self.send(request)
            except exc.NodeExists:
                pass

            paths_to_make.pop(0)

    @gen.coroutine
    @znode_method
    def delete(self, path, force=False):
        if not force and path in self.stat_cache:
            version = self.stat_cache[path].version
        else:
            version = -1

        yield self.send(protocol.DeleteRequest(path=path, version=version))

    @gen.coroutine
    @znode_method
    def get_data(self, path, watch=False):
        response = yield self.send(
            protocol.GetDataRequest(path=path, watch=watch)
        )
        raise gen.Return(response.data)

    @gen.coroutine
    @znode_method
    def set_data(self, path, data, force=False):

        if not force and path in self.stat_cache:
            version = self.stat_cache[path].version
        else:
            version = -1

        yield self.send(
            protocol.SetDataRequest(path=path, data=data, version=version)
        )

    @gen.coroutine
    @znode_method
    def get_children(self, path, watch=False):
        response = yield self.send(
            protocol.GetChildren2Request(path=path, watch=watch)
        )
        raise gen.Return(response.children)

    @gen.coroutine
    @znode_method
    def get_acl(self, path):
        response = yield self.send(protocol.GetACLRequest(path=path))
        raise gen.Return(response.acl)

    @gen.coroutine
    @znode_method
    def set_acl(self, path, acl, force=False):
        if not force and path in self.stat_cache:
            version = self.stat_cache[path].version
        else:
            version = -1

        yield self.send(
            protocol.SetACLRequest(path=path, acl=acl, version=version)
        )

    def begin_transaction(self):
        return Transaction(self)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-TODAY OpenERP S.A. <http://www.openerp.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
import logging
import threading
import psycopg2
from datetime import datetime
from dateutil.relativedelta import relativedelta

import netsvc
import openerp
import pooler
import tools
from openerp.cron import WAKE_UP_NOW
from osv import fields, osv
from tools import DEFAULT_SERVER_DATETIME_FORMAT
from tools.safe_eval import safe_eval as eval

def str2tuple(s):
    return eval('tuple(%s)' % (s or ''))

_intervalTypes = {
    'work_days': lambda interval: relativedelta(days=interval),
    'days': lambda interval: relativedelta(days=interval),
    'hours': lambda interval: relativedelta(hours=interval),
    'weeks': lambda interval: relativedelta(days=7*interval),
    'months': lambda interval: relativedelta(months=interval),
    'minutes': lambda interval: relativedelta(minutes=interval),
}

class ir_cron(osv.osv):
    """ Model describing cron jobs (also called actions or tasks).
    """

    # TODO: perhaps in the future we could consider a flag on ir.cron jobs
    # that would cause database wake-up even if the database has not been
    # loaded yet or was already unloaded (e.g. 'force_db_wakeup' or something)
    # See also openerp.cron

    _name = "ir.cron"
    _order = 'name'
    _columns = {
        'name': fields.char('Name', size=60, required=True),
        'user_id': fields.many2one('res.users', 'User', required=True),
        'active': fields.boolean('Active'),
        'interval_number': fields.integer('Interval Number',help="Repeat every x."),
        'interval_type': fields.selection( [('minutes', 'Minutes'),
            ('hours', 'Hours'), ('work_days','Work Days'), ('days', 'Days'),('weeks', 'Weeks'), ('months', 'Months')], 'Interval Unit'),
        'numbercall': fields.integer('Number of Calls', help='How many times the method is called,\na negative number indicates no limit.'),
        'doall' : fields.boolean('Repeat Missed', help="Specify if missed occurrences should be executed when the server restarts."),
        'nextcall' : fields.datetime('Next Execution Date', required=True, help="Next planned execution date for this job."),
        'model': fields.char('Object', size=64, help="Model name on which the method to be called is located, e.g. 'res.partner'."),
        'function': fields.char('Method', size=64, help="Name of the method to be called when this job is processed."),
        'args': fields.text('Arguments', help="Arguments to be passed to the method, e.g. (uid,)."),
        'priority': fields.integer('Priority', help='The priority of the job, as an integer: 0 means higher priority, 10 means lower priority.')
    }

    _defaults = {
        'nextcall' : lambda *a: time.strftime(DEFAULT_SERVER_DATETIME_FORMAT),
        'priority' : lambda *a: 5,
        'user_id' : lambda obj,cr,uid,context: uid,
        'interval_number' : lambda *a: 1,
        'interval_type' : lambda *a: 'months',
        'numbercall' : lambda *a: 1,
        'active' : lambda *a: 1,
        'doall' : lambda *a: 1
    }

    _logger = logging.getLogger('cron')

    def _check_args(self, cr, uid, ids, context=None):
        try:
            for this in self.browse(cr, uid, ids, context):
                str2tuple(this.args)
        except Exception:
            return False
        return True

    _constraints = [
        (_check_args, 'Invalid arguments', ['args']),
    ]

    def _handle_callback_exception(self, cr, uid, model_name, method_name, args, job_id, job_exception):
        """ Method called when an exception is raised by a job.

        Simply logs the exception and rollback the transaction.

        :param model_name: model name on which the job method is located.
        :param method_name: name of the method to call when this job is processed.
        :param args: arguments of the method (without the usual self, cr, uid).
        :param job_id: job id.
        :param job_exception: exception raised by the job.

        """
        cr.rollback()
        self._logger.exception("Call of self.pool.get('%s').%s(cr, uid, *%r) failed in Job %s" % (model_name, method_name, args, job_id))

    def _callback(self, cr, uid, model_name, method_name, args, job_id):
        """ Run the method associated to a given job

        It takes care of logging and exception handling.

        :param model_name: model name on which the job method is located.
        :param method_name: name of the method to call when this job is processed.
        :param args: arguments of the method (without the usual self, cr, uid).
        :param job_id: job id.
        """
        args = str2tuple(args)
        model = self.pool.get(model_name)
        if model and hasattr(model, method_name):
            method = getattr(model, method_name)
            try:
                netsvc.log('cron', (cr.dbname,uid,'*',model_name,method_name)+tuple(args), channel=logging.DEBUG,
                            depth=(None if self._logger.isEnabledFor(logging.DEBUG_RPC_ANSWER) else 1), fn='object.execute')
                logger = logging.getLogger('execution time')
                if logger.isEnabledFor(logging.DEBUG):
                    start_time = time.time()
                method(cr, uid, *args)
                if logger.isEnabledFor(logging.DEBUG):
                    end_time = time.time()
                    logger.log(logging.DEBUG, '%.3fs (%s, %s)' % (end_time - start_time, model_name, method_name))
            except Exception, e:
                self._handle_callback_exception(cr, uid, model_name, method_name, args, job_id, e)

    def _run_job(self, cr, job, now):
        """ Run a given job taking care of the repetition.

        The cursor has a lock on the job (aquired by _run_jobs_multithread()) and this
        method is run in a worker thread (spawned by _run_jobs_multithread())).

        :param job: job to be run (as a dictionary).
        :param now: timestamp (result of datetime.now(), no need to call it multiple time).

        """
        try:
            nextcall = datetime.strptime(job['nextcall'], DEFAULT_SERVER_DATETIME_FORMAT)
            numbercall = job['numbercall']

            ok = False
            while nextcall < now and numbercall:
                if numbercall > 0:
                    numbercall -= 1
                if not ok or job['doall']:
                    self._callback(cr, job['user_id'], job['model'], job['function'], job['args'], job['id'])
                if numbercall:
                    nextcall += _intervalTypes[job['interval_type']](job['interval_number'])
                ok = True
            addsql = ''
            if not numbercall:
                addsql = ', active=False'
            cr.execute("UPDATE ir_cron SET nextcall=%s, numbercall=%s"+addsql+" WHERE id=%s",
                       (nextcall.strftime(DEFAULT_SERVER_DATETIME_FORMAT), numbercall, job['id']))

            if numbercall:
                # Reschedule our own main cron thread if necessary.
                # This is really needed if this job runs longer than its rescheduling period.
                nextcall = time.mktime(nextcall.timetuple())
                openerp.cron.schedule_wakeup(nextcall, cr.dbname)
        finally:
            cr.commit()
            cr.close()
            openerp.cron.release_thread_slot()

    def _run_jobs_multithread(self):
        # TODO remove 'check' argument from addons/base_action_rule/base_action_rule.py
        """ Process the cron jobs by spawning worker threads.

        This selects in database all the jobs that should be processed. It then
        tries to lock each of them and, if it succeeds, spawns a thread to run
        the cron job (if it doesn't succeed, it means the job was already
        locked to be taken care of by another thread).

        The cursor used to lock the job in database is given to the worker
        thread (which has to close it itself).

        """
        db = self.pool.db
        cr = db.cursor()
        db_name = db.dbname
        try:
            jobs = {} # mapping job ids to jobs for all jobs being processed.
            now = datetime.now()
            # Careful to compare timestamps with 'UTC' - everything is UTC as of v6.1.
            cr.execute("""SELECT * FROM ir_cron
                          WHERE numbercall != 0
                              AND active AND nextcall <= (now() at time zone 'UTC')
                          ORDER BY priority""")
            for job in cr.dictfetchall():
                if not openerp.cron.get_thread_slots():
                    break
                jobs[job['id']] = job

                task_cr = db.cursor()
                try:
                    # Try to grab an exclusive lock on the job row from within the task transaction
                    acquired_lock = False
                    task_cr.execute("""SELECT *
                                       FROM ir_cron
                                       WHERE id=%s
                                       FOR UPDATE NOWAIT""",
                                   (job['id'],), log_exceptions=False)
                    acquired_lock = True
                except psycopg2.OperationalError, e:
                    if e.pgcode == '55P03':
                        # Class 55: Object not in prerequisite state; 55P03: lock_not_available
                        self._logger.debug('Another process/thread is already busy executing job `%s`, skipping it.', job['name'])
                        continue
                    else:
                        # Unexpected OperationalError
                        raise
                finally:
                    if not acquired_lock:
                        # we're exiting due to an exception while acquiring the lot
                        task_cr.close()

                # Got the lock on the job row, now spawn a thread to execute it in the transaction with the lock
                task_thread = threading.Thread(target=self._run_job, name=job['name'], args=(task_cr, job, now))
                # force non-daemon task threads (the runner thread must be daemon, and this property is inherited by default)
                task_thread.setDaemon(False)
                openerp.cron.take_thread_slot()
                task_thread.start()
                self._logger.debug('Cron execution thread for job `%s` spawned', job['name'])

            # Find next earliest job ignoring currently processed jobs (by this and other cron threads)
            find_next_time_query = """SELECT min(nextcall) AS min_next_call
                                      FROM ir_cron WHERE numbercall != 0 AND active"""
            if jobs:
                cr.execute(find_next_time_query + " AND id NOT IN %s", (tuple(jobs.keys()),))
            else:
                cr.execute(find_next_time_query)
            next_call = cr.dictfetchone()['min_next_call']

            if next_call:
                next_call = time.mktime(time.strptime(next_call, DEFAULT_SERVER_DATETIME_FORMAT))
            else:
                # no matching cron job found in database, re-schedule arbitrarily in 1 day,
                # this delay will likely be modified when running jobs complete their tasks
                next_call = time.time() + (24*3600)

            openerp.cron.schedule_wakeup(next_call, db_name)

        except Exception, ex:
            self._logger.warning('Exception in cron:', exc_info=True)

        finally:
            cr.commit()
            cr.close()

    def update_running_cron(self, cr):
        """ Schedule as soon as possible a wake-up for this database. """
        # Verify whether the server is already started and thus whether we need to commit
        # immediately our changes and restart the cron agent in order to apply the change
        # immediately. The commit() is needed because as soon as the cron is (re)started it
        # will query the database with its own cursor, possibly before the end of the
        # current transaction.
        # This commit() is not an issue in most cases, but we must absolutely avoid it
        # when the server is only starting or loading modules (hence the test on pool._init).
        if not self.pool._init:
            cr.commit()
            openerp.cron.schedule_wakeup(WAKE_UP_NOW, self.pool.db.dbname)

    def create(self, cr, uid, vals, context=None):
        res = super(ir_cron, self).create(cr, uid, vals, context=context)
        self.update_running_cron(cr)
        return res

    def write(self, cr, user, ids, vals, context=None):
        res = super(ir_cron, self).write(cr, user, ids, vals, context=context)
        self.update_running_cron(cr)
        return res

    def unlink(self, cr, uid, ids, context=None):
        res = super(ir_cron, self).unlink(cr, uid, ids, context=context)
        self.update_running_cron(cr)
        return res
ir_cron()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[IMP] ir.cron.write/unlink: try NOWAIT lock to avoid blocking user operations during cron executions<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-TODAY OpenERP S.A. <http://www.openerp.com>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
import logging
import threading
import psycopg2
from datetime import datetime
from dateutil.relativedelta import relativedelta

import netsvc
import openerp
import pooler
import tools
from openerp.cron import WAKE_UP_NOW
from osv import fields, osv
from tools import DEFAULT_SERVER_DATETIME_FORMAT
from tools.safe_eval import safe_eval as eval
from tools.translate import _

def str2tuple(s):
    return eval('tuple(%s)' % (s or ''))

_intervalTypes = {
    'work_days': lambda interval: relativedelta(days=interval),
    'days': lambda interval: relativedelta(days=interval),
    'hours': lambda interval: relativedelta(hours=interval),
    'weeks': lambda interval: relativedelta(days=7*interval),
    'months': lambda interval: relativedelta(months=interval),
    'minutes': lambda interval: relativedelta(minutes=interval),
}

class ir_cron(osv.osv):
    """ Model describing cron jobs (also called actions or tasks).
    """

    # TODO: perhaps in the future we could consider a flag on ir.cron jobs
    # that would cause database wake-up even if the database has not been
    # loaded yet or was already unloaded (e.g. 'force_db_wakeup' or something)
    # See also openerp.cron

    _name = "ir.cron"
    _order = 'name'
    _columns = {
        'name': fields.char('Name', size=60, required=True),
        'user_id': fields.many2one('res.users', 'User', required=True),
        'active': fields.boolean('Active'),
        'interval_number': fields.integer('Interval Number',help="Repeat every x."),
        'interval_type': fields.selection( [('minutes', 'Minutes'),
            ('hours', 'Hours'), ('work_days','Work Days'), ('days', 'Days'),('weeks', 'Weeks'), ('months', 'Months')], 'Interval Unit'),
        'numbercall': fields.integer('Number of Calls', help='How many times the method is called,\na negative number indicates no limit.'),
        'doall' : fields.boolean('Repeat Missed', help="Specify if missed occurrences should be executed when the server restarts."),
        'nextcall' : fields.datetime('Next Execution Date', required=True, help="Next planned execution date for this job."),
        'model': fields.char('Object', size=64, help="Model name on which the method to be called is located, e.g. 'res.partner'."),
        'function': fields.char('Method', size=64, help="Name of the method to be called when this job is processed."),
        'args': fields.text('Arguments', help="Arguments to be passed to the method, e.g. (uid,)."),
        'priority': fields.integer('Priority', help='The priority of the job, as an integer: 0 means higher priority, 10 means lower priority.')
    }

    _defaults = {
        'nextcall' : lambda *a: time.strftime(DEFAULT_SERVER_DATETIME_FORMAT),
        'priority' : lambda *a: 5,
        'user_id' : lambda obj,cr,uid,context: uid,
        'interval_number' : lambda *a: 1,
        'interval_type' : lambda *a: 'months',
        'numbercall' : lambda *a: 1,
        'active' : lambda *a: 1,
        'doall' : lambda *a: 1
    }

    _logger = logging.getLogger('cron')

    def _check_args(self, cr, uid, ids, context=None):
        try:
            for this in self.browse(cr, uid, ids, context):
                str2tuple(this.args)
        except Exception:
            return False
        return True

    _constraints = [
        (_check_args, 'Invalid arguments', ['args']),
    ]

    def _handle_callback_exception(self, cr, uid, model_name, method_name, args, job_id, job_exception):
        """ Method called when an exception is raised by a job.

        Simply logs the exception and rollback the transaction.

        :param model_name: model name on which the job method is located.
        :param method_name: name of the method to call when this job is processed.
        :param args: arguments of the method (without the usual self, cr, uid).
        :param job_id: job id.
        :param job_exception: exception raised by the job.

        """
        cr.rollback()
        self._logger.exception("Call of self.pool.get('%s').%s(cr, uid, *%r) failed in Job %s" % (model_name, method_name, args, job_id))

    def _callback(self, cr, uid, model_name, method_name, args, job_id):
        """ Run the method associated to a given job

        It takes care of logging and exception handling.

        :param model_name: model name on which the job method is located.
        :param method_name: name of the method to call when this job is processed.
        :param args: arguments of the method (without the usual self, cr, uid).
        :param job_id: job id.
        """
        args = str2tuple(args)
        model = self.pool.get(model_name)
        if model and hasattr(model, method_name):
            method = getattr(model, method_name)
            try:
                netsvc.log('cron', (cr.dbname,uid,'*',model_name,method_name)+tuple(args), channel=logging.DEBUG,
                            depth=(None if self._logger.isEnabledFor(logging.DEBUG_RPC_ANSWER) else 1), fn='object.execute')
                logger = logging.getLogger('execution time')
                if logger.isEnabledFor(logging.DEBUG):
                    start_time = time.time()
                method(cr, uid, *args)
                if logger.isEnabledFor(logging.DEBUG):
                    end_time = time.time()
                    logger.log(logging.DEBUG, '%.3fs (%s, %s)' % (end_time - start_time, model_name, method_name))
            except Exception, e:
                self._handle_callback_exception(cr, uid, model_name, method_name, args, job_id, e)

    def _run_job(self, cr, job, now):
        """ Run a given job taking care of the repetition.

        The cursor has a lock on the job (aquired by _run_jobs_multithread()) and this
        method is run in a worker thread (spawned by _run_jobs_multithread())).

        :param job: job to be run (as a dictionary).
        :param now: timestamp (result of datetime.now(), no need to call it multiple time).

        """
        try:
            nextcall = datetime.strptime(job['nextcall'], DEFAULT_SERVER_DATETIME_FORMAT)
            numbercall = job['numbercall']

            ok = False
            while nextcall < now and numbercall:
                if numbercall > 0:
                    numbercall -= 1
                if not ok or job['doall']:
                    self._callback(cr, job['user_id'], job['model'], job['function'], job['args'], job['id'])
                if numbercall:
                    nextcall += _intervalTypes[job['interval_type']](job['interval_number'])
                ok = True
            addsql = ''
            if not numbercall:
                addsql = ', active=False'
            cr.execute("UPDATE ir_cron SET nextcall=%s, numbercall=%s"+addsql+" WHERE id=%s",
                       (nextcall.strftime(DEFAULT_SERVER_DATETIME_FORMAT), numbercall, job['id']))

            if numbercall:
                # Reschedule our own main cron thread if necessary.
                # This is really needed if this job runs longer than its rescheduling period.
                nextcall = time.mktime(nextcall.timetuple())
                openerp.cron.schedule_wakeup(nextcall, cr.dbname)
        finally:
            cr.commit()
            cr.close()
            openerp.cron.release_thread_slot()

    def _run_jobs_multithread(self):
        # TODO remove 'check' argument from addons/base_action_rule/base_action_rule.py
        """ Process the cron jobs by spawning worker threads.

        This selects in database all the jobs that should be processed. It then
        tries to lock each of them and, if it succeeds, spawns a thread to run
        the cron job (if it doesn't succeed, it means the job was already
        locked to be taken care of by another thread).

        The cursor used to lock the job in database is given to the worker
        thread (which has to close it itself).

        """
        db = self.pool.db
        cr = db.cursor()
        db_name = db.dbname
        try:
            jobs = {} # mapping job ids to jobs for all jobs being processed.
            now = datetime.now()
            # Careful to compare timestamps with 'UTC' - everything is UTC as of v6.1.
            cr.execute("""SELECT * FROM ir_cron
                          WHERE numbercall != 0
                              AND active AND nextcall <= (now() at time zone 'UTC')
                          ORDER BY priority""")
            for job in cr.dictfetchall():
                if not openerp.cron.get_thread_slots():
                    break
                jobs[job['id']] = job

                task_cr = db.cursor()
                try:
                    # Try to grab an exclusive lock on the job row from within the task transaction
                    acquired_lock = False
                    task_cr.execute("""SELECT *
                                       FROM ir_cron
                                       WHERE id=%s
                                       FOR UPDATE NOWAIT""",
                                   (job['id'],), log_exceptions=False)
                    acquired_lock = True
                except psycopg2.OperationalError, e:
                    if e.pgcode == '55P03':
                        # Class 55: Object not in prerequisite state; 55P03: lock_not_available
                        self._logger.debug('Another process/thread is already busy executing job `%s`, skipping it.', job['name'])
                        continue
                    else:
                        # Unexpected OperationalError
                        raise
                finally:
                    if not acquired_lock:
                        # we're exiting due to an exception while acquiring the lot
                        task_cr.close()

                # Got the lock on the job row, now spawn a thread to execute it in the transaction with the lock
                task_thread = threading.Thread(target=self._run_job, name=job['name'], args=(task_cr, job, now))
                # force non-daemon task threads (the runner thread must be daemon, and this property is inherited by default)
                task_thread.setDaemon(False)
                openerp.cron.take_thread_slot()
                task_thread.start()
                self._logger.debug('Cron execution thread for job `%s` spawned', job['name'])

            # Find next earliest job ignoring currently processed jobs (by this and other cron threads)
            find_next_time_query = """SELECT min(nextcall) AS min_next_call
                                      FROM ir_cron WHERE numbercall != 0 AND active"""
            if jobs:
                cr.execute(find_next_time_query + " AND id NOT IN %s", (tuple(jobs.keys()),))
            else:
                cr.execute(find_next_time_query)
            next_call = cr.dictfetchone()['min_next_call']

            if next_call:
                next_call = time.mktime(time.strptime(next_call, DEFAULT_SERVER_DATETIME_FORMAT))
            else:
                # no matching cron job found in database, re-schedule arbitrarily in 1 day,
                # this delay will likely be modified when running jobs complete their tasks
                next_call = time.time() + (24*3600)

            openerp.cron.schedule_wakeup(next_call, db_name)

        except Exception, ex:
            self._logger.warning('Exception in cron:', exc_info=True)

        finally:
            cr.commit()
            cr.close()

    def update_running_cron(self, cr):
        """ Schedule as soon as possible a wake-up for this database. """
        # Verify whether the server is already started and thus whether we need to commit
        # immediately our changes and restart the cron agent in order to apply the change
        # immediately. The commit() is needed because as soon as the cron is (re)started it
        # will query the database with its own cursor, possibly before the end of the
        # current transaction.
        # This commit() is not an issue in most cases, but we must absolutely avoid it
        # when the server is only starting or loading modules (hence the test on pool._init).
        if not self.pool._init:
            cr.commit()
            openerp.cron.schedule_wakeup(WAKE_UP_NOW, self.pool.db.dbname)

    def _try_lock(self, cr, uid, ids, context=None):
        """Try to grab a dummy exclusive write-lock to the rows with the given ids,
           to make sure a following write() or unlink() will not block due
           to a process currently executing those cron tasks"""
        try:
            cr.execute("""SELECT id FROM "%s" WHERE id IN %%s FOR UPDATE NOWAIT""" % self._table,
                       (tuple(ids),), log_exceptions=False)
        except psycopg2.OperationalError:
            cr.rollback() # early rollback to allow translations to work for the user feedback
            raise osv.except_osv(_("Record cannot be modified right now"),
                                 _("This cron task is currently being executed and may not be modified, "
                                  "please try again in a few minutes"))

    def create(self, cr, uid, vals, context=None):
        res = super(ir_cron, self).create(cr, uid, vals, context=context)
        self.update_running_cron(cr)
        return res

    def write(self, cr, uid, ids, vals, context=None):
        self._try_lock(cr, uid, ids, context)
        res = super(ir_cron, self).write(cr, uid, ids, vals, context=context)
        self.update_running_cron(cr)
        return res

    def unlink(self, cr, uid, ids, context=None):
        self._try_lock(cr, uid, ids, context)
        res = super(ir_cron, self).unlink(cr, uid, ids, context=context)
        self.update_running_cron(cr)
        return res
ir_cron()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>