<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from odoo import fields, osv, models, _
from odoo.tools.translate import _
import pdb
import logging
_logger = logging.getLogger(__name__)

import json
from datetime import datetime

#from bottle import Bottle, run, template, route, request
#import json
from meli_oerp_config import *

from warning import warning

import melisdk
from melisdk.meli import Meli

class product_post(models.TransientModel):
    _name = "mercadolibre.product.post"
    _description = "Wizard de Product Posting en MercadoLibre"

    type = fields.Selection([('post','Alta'),('put','Editado'),('delete','Borrado')], string='Tipo de operación' );
    posting_date = fields.Date('Fecha del posting');
	    #'company_id': fields.many2one('res.company',string='Company'),
	    #'mercadolibre_state': fields.related( 'res.company', 'mercadolibre_state', string="State" )

    def pretty_json( self, data ):
        return json.dumps( data, sort_keys=False, indent=4 )

    def product_post(self, context):
        pdb.set_trace()
        company = self.env.user.company_id
        product_ids = context['active_ids']
        product_obj = self.env['product.product']

        #user_obj = self.pool.get('res.users').browse(cr, uid, uid)
        #user_obj.company_id.meli_login()
        #company = user_obj.company_id
        warningobj = self.env['warning']

        #company = self.pool.get('res.company').browse(cr,uid,1)

        REDIRECT_URI = company.mercadolibre_redirect_uri
        CLIENT_ID = company.mercadolibre_client_id
        CLIENT_SECRET = company.mercadolibre_secret_key
        ACCESS_TOKEN = company.mercadolibre_access_token
        REFRESH_TOKEN = company.mercadolibre_refresh_token

        meli = Meli(client_id=CLIENT_ID,client_secret=CLIENT_SECRET, access_token=ACCESS_TOKEN, refresh_token=REFRESH_TOKEN)

        if ACCESS_TOKEN=='' or ACCESS_TOKEN==False:
            meli = Meli(client_id=CLIENT_ID,client_secret=CLIENT_SECRET)
            url_login_meli = meli.auth_url(redirect_URI=REDIRECT_URI)
            return {
	            "type": "ir.actions.act_url",
	            "url": url_login_meli,
	            "target": "new",
            }

        for product_id in product_ids:
            product = product_obj.browse(product_id)

            if (product.meli_id):
                response = meli.get("/items/%s" % product.meli_id, {'access_token':meli.access_token})

            # print product.meli_category.meli_category_id

            if product.meli_title==False:
                # print 'Assigning title: product.meli_title: %s name: %s' % (product.meli_title, product.name)
                product.meli_title = product.name

            if product.meli_price==False:
                # print 'Assigning price: product.meli_price: %s standard_price: %s' % (product.meli_price, product.standard_price)
                product.meli_price = product.standard_price

            body = {
                "title": product.meli_title or '',
                "description": product.meli_description or '',
                "category_id": product.meli_category.meli_category_id or '0',
                "listing_type_id": product.meli_listing_type or '0',
                "buying_mode": product.meli_buying_mode or '',
                "price": product.meli_price  or '0',
                "currency_id": product.meli_currency  or '0',
                #"condition": product.meli_condition  or '',
                "available_quantity": product.meli_available_quantity  or '0',
                "warranty": product.meli_warranty or '',
                #"pictures": [ { 'source': product.meli_imagen_logo} ] ,
                "video_id": product.meli_video  or '',
            }
            print "product.meli_id before setting conditions ]"+product.meli_id+"["
            if (len(product.meli_id)==0):
                body["condition"] = product.meli_condition;

            # print body

            assign_img = False and product.meli_id

            #publicando imagen cargada en OpenERP
            if product.image==None:
                return warningobj.info(cr, uid, title='MELI WARNING', message="Debe cargar una imagen de base en el producto.", message_html="" )
            elif product.meli_imagen_id==False:
                # print "try uploading image..."
                resim = product.product_meli_upload_image()
                if "status" in resim:
                    if (resim["status"]=="error" or resim["status"]=="warning"):
                        error_msg = 'MELI: mensaje de error:   ', resim
                        _logger.error(error_msg)
                    else:
                        assign_img = True and product.meli_imagen_id

            #modificando datos si ya existe el producto en MLA
            if (product.meli_id):
                body = {
                    "title": product.meli_title or '',
                    #"description": product.meli_description or '',
                    #"category_id": product.meli_category.meli_category_id,
                    #"listing_type_id": product.meli_listing_type,
                    #"buying_mode": product.meli_buying_mode or '',
                    "price": product.meli_price or '0',
                    #"currency_id": product.meli_currency,
                    #"condition": product.meli_condition or '',
                    "available_quantity": product.meli_available_quantity or '0',
                    "warranty": product.meli_warranty or '',
                    "pictures": [],
                    #"pictures": [ { 'source': product.meli_imagen_logo} ] ,
                    "video_id": product.meli_video or '',
                }

            #publicando multiples imagenes
            multi_images_ids = {}
            if (product.images):
                # print 'website_multi_images presente:   ', product.images
                #recorrer las imagenes y publicarlas
                multi_images_ids = product.product_meli_upload_multi_images()

            #asignando imagen de logo (por source)
            if product.meli_imagen_logo:
                if product.meli_imagen_id:
                    if 'pictures' in body.keys():
                        body["pictures"]+= [ { 'id': product.meli_imagen_id } ]
                    else:
                        body["pictures"] = [ { 'id': product.meli_imagen_id } ]

                        if (multi_images_ids):
                            if 'pictures' in body.keys():
                                body["pictures"]+= multi_images_ids
                            else:
                                body["pictures"] = multi_images_ids

                    if 'pictures' in body.keys():
                        body["pictures"]+= [ { 'source': product.meli_imagen_logo} ]
                    else:
                        body["pictures"]+= [ { 'source': product.meli_imagen_logo} ]
                else:
                    imagen_producto = ""
                    if (product.meli_description!="" and product.meli_description!=False and product.meli_imagen_link!=""):
                        imgtag = "" % ( product.meli_imagen_link )
                        result = product.meli_description.replace( "[IMAGEN_PRODUCTO]", imgtag )
                        if (result):
                            _logger.info( "result: %s" % (result) )
                            product.meli_description = result
                        else:
                            result = product.meli_description

            else:
                return warningobj.info(cr, uid, title='MELI WARNING', message="Debe completar el campo 'Imagen_Logo' con el url: http://www.nuevohorizonte-sa.com.ar/images/logo1.png", message_html="")

            #check fields
            if product.meli_description==False:
                return warningobj.info(cr, uid, title='MELI WARNING', message="Debe completar el campo 'description' (en html)", message_html="")

            #put for editing, post for creating
            if product.meli_id:
                response = meli.put("/items/"+product.meli_id, body, {'access_token':meli.access_token})
            else:
                assign_img = True and product.meli_imagen_id
                response = meli.post("/items", body, {'access_token':meli.access_token})

            #check response
            # print response.content
            rjson = response.json()

            #check error
            if "error" in rjson:
                #print "Error received: %s " % rjson["error"]
                error_msg = 'MELI: mensaje de error:  %s , mensaje: %s, status: %s, cause: %s ' % (rjson["error"], rjson["message"], rjson["status"], rjson["cause"])
                _logger.error(error_msg)

                missing_fields = error_msg

                #expired token
                if "message" in rjson and (rjson["message"]=='invalid_token' or rjson["message"]=="expired_token"):
                    meli = Meli(client_id=CLIENT_ID,client_secret=CLIENT_SECRET)
                    url_login_meli = meli.auth_url(redirect_URI=REDIRECT_URI)
                    #print "url_login_meli:", url_login_meli
                    #raise osv.except_osv( _('MELI WARNING'), _('INVALID TOKEN or EXPIRED TOKEN (must login, go to Edit Company and login):  error: %s, message: %s, status: %s') % ( rjson["error"], rjson["message"],rjson["status"],))
                    return warningobj.info(title='MELI WARNING', message="Debe iniciar sesión en MELI.  ", message_html="")
                else:
                     #Any other errors
                    return warningobj.info(title='MELI WARNING', message="Completar todos los campos!  ", message_html="

"+missing_fields )

            #last modifications if response is OK
            if "id" in rjson:
                product.write( { 'meli_id': rjson["id"]} )

            posting_fields = {'posting_date': str(datetime.now()),'meli_id':rjson['id'],'product_id':product.id,'name': 'Post: ' + product.meli_title }

            posting_id = self.env['mercadolibre.posting'].search([('meli_id','=',rjson['id'])])

            if not posting_id:
	            posting_id = self.env['mercadolibre.posting'].create((posting_fields))

        return {}

product_post()
<commit_msg>Fixing syntax<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from odoo import fields, osv, models, _
from odoo.tools.translate import _
import pdb
import logging
_logger = logging.getLogger(__name__)

import json
from datetime import datetime

#from bottle import Bottle, run, template, route, request
#import json
from meli_oerp_config import *

from warning import warning

import melisdk
from melisdk.meli import Meli

class product_post(models.TransientModel):
    _name = "mercadolibre.product.post"
    _description = "Wizard de Product Posting en MercadoLibre"

    type = fields.Selection([('post','Alta'),('put','Editado'),('delete','Borrado')], string='Tipo de operación' );
    posting_date = fields.Date('Fecha del posting');
	    #'company_id': fields.many2one('res.company',string='Company'),
	    #'mercadolibre_state': fields.related( 'res.company', 'mercadolibre_state', string="State" )

    def pretty_json( self, data ):
        return json.dumps( data, sort_keys=False, indent=4 )

    def product_post(self, context):
        pdb.set_trace()
        company = self.env.user.company_id
        product_ids = context['active_ids']
        product_obj = self.env['product.product']

        #user_obj = self.pool.get('res.users').browse(cr, uid, uid)
        #user_obj.company_id.meli_login()
        #company = user_obj.company_id
        warningobj = self.env['warning']

        #company = self.pool.get('res.company').browse(cr,uid,1)

        REDIRECT_URI = company.mercadolibre_redirect_uri
        CLIENT_ID = company.mercadolibre_client_id
        CLIENT_SECRET = company.mercadolibre_secret_key
        ACCESS_TOKEN = company.mercadolibre_access_token
        REFRESH_TOKEN = company.mercadolibre_refresh_token

        meli = Meli(client_id=CLIENT_ID,client_secret=CLIENT_SECRET, access_token=ACCESS_TOKEN, refresh_token=REFRESH_TOKEN)

        if ACCESS_TOKEN=='' or ACCESS_TOKEN==False:
            meli = Meli(client_id=CLIENT_ID,client_secret=CLIENT_SECRET)
            url_login_meli = meli.auth_url(redirect_URI=REDIRECT_URI)
            return {
	            "type": "ir.actions.act_url",
	            "url": url_login_meli,
	            "target": "new",
            }

        for product_id in product_ids:
            product = product_obj.browse(product_id)

            if (product.meli_id):
                response = meli.get("/items/%s" % product.meli_id, {'access_token':meli.access_token})

            # print product.meli_category.meli_category_id

            if product.meli_title==False:
                # print 'Assigning title: product.meli_title: %s name: %s' % (product.meli_title, product.name)
                product.meli_title = product.name

            if product.meli_price==False:
                # print 'Assigning price: product.meli_price: %s standard_price: %s' % (product.meli_price, product.standard_price)
                product.meli_price = product.standard_price

            body = {
                "title": product.meli_title or '',
                "description": product.meli_description or '',
                "category_id": product.meli_category.meli_category_id or '0',
                "listing_type_id": product.meli_listing_type or '0',
                "buying_mode": product.meli_buying_mode or '',
                "price": product.meli_price  or '0',
                "currency_id": product.meli_currency  or '0',
                #"condition": product.meli_condition  or '',
                "available_quantity": product.meli_available_quantity  or '0',
                "warranty": product.meli_warranty or '',
                #"pictures": [ { 'source': product.meli_imagen_logo} ] ,
                "video_id": product.meli_video  or '',
            }
            print "product.meli_id before setting conditions ]"+product.meli_id+"["
            if (len(product.meli_id)==0):
                body["condition"] = product.meli_condition;

            # print body

            assign_img = False and product.meli_id

            #publicando imagen cargada en OpenERP
            if product.image==None:
                return warningobj.info(cr, uid, title='MELI WARNING', message="Debe cargar una imagen de base en el producto.", message_html="" )
            elif product.meli_imagen_id==False:
                # print "try uploading image..."
                resim = product.product_meli_upload_image()
                if "status" in resim:
                    if (resim["status"]=="error" or resim["status"]=="warning"):
                        error_msg = 'MELI: mensaje de error:   ', resim
                        _logger.error(error_msg)
                    else:
                        assign_img = True and product.meli_imagen_id

            #modificando datos si ya existe el producto en MLA
            if (product.meli_id):
                body = {
                    "title": product.meli_title or '',
                    #"description": product.meli_description or '',
                    #"category_id": product.meli_category.meli_category_id,
                    #"listing_type_id": product.meli_listing_type,
                    #"buying_mode": product.meli_buying_mode or '',
                    "price": product.meli_price or '0',
                    #"currency_id": product.meli_currency,
                    #"condition": product.meli_condition or '',
                    "available_quantity": product.meli_available_quantity or '0',
                    "warranty": product.meli_warranty or '',
                    "pictures": [],
                    #"pictures": [ { 'source': product.meli_imagen_logo} ] ,
                    "video_id": product.meli_video or '',
                }

            #publicando multiples imagenes
            multi_images_ids = {}
            if "images" in product:
                # print 'website_multi_images presente:   ', product.images
                #recorrer las imagenes y publicarlas
                multi_images_ids = product.product_meli_upload_multi_images()

            #asignando imagen de logo (por source)
            if product.meli_imagen_logo:
                if product.meli_imagen_id:
                    if 'pictures' in body.keys():
                        body["pictures"]+= [ { 'id': product.meli_imagen_id } ]
                    else:
                        body["pictures"] = [ { 'id': product.meli_imagen_id } ]

                        if (multi_images_ids):
                            if 'pictures' in body.keys():
                                body["pictures"]+= multi_images_ids
                            else:
                                body["pictures"] = multi_images_ids

                    if 'pictures' in body.keys():
                        body["pictures"]+= [ { 'source': product.meli_imagen_logo} ]
                    else:
                        body["pictures"]+= [ { 'source': product.meli_imagen_logo} ]
                else:
                    imagen_producto = ""
                    if (product.meli_description!="" and product.meli_description!=False and product.meli_imagen_link!=""):
                        imgtag = "" % ( product.meli_imagen_link )
                        result = product.meli_description.replace( "[IMAGEN_PRODUCTO]", imgtag )
                        if (result):
                            _logger.info( "result: %s" % (result) )
                            product.meli_description = result
                        else:
                            result = product.meli_description

            else:
                return warningobj.info(cr, uid, title='MELI WARNING', message="Debe completar el campo 'Imagen_Logo' con el url: http://www.nuevohorizonte-sa.com.ar/images/logo1.png", message_html="")

            #check fields
            if product.meli_description==False:
                return warningobj.info(cr, uid, title='MELI WARNING', message="Debe completar el campo 'description' (en html)", message_html="")

            #put for editing, post for creating
            if product.meli_id:
                response = meli.put("/items/"+product.meli_id, body, {'access_token':meli.access_token})
            else:
                assign_img = True and product.meli_imagen_id
                response = meli.post("/items", body, {'access_token':meli.access_token})

            #check response
            # print response.content
            rjson = response.json()

            #check error
            if "error" in rjson:
                #print "Error received: %s " % rjson["error"]
                error_msg = 'MELI: mensaje de error:  %s , mensaje: %s, status: %s, cause: %s ' % (rjson["error"], rjson["message"], rjson["status"], rjson["cause"])
                _logger.error(error_msg)

                missing_fields = error_msg

                #expired token
                if "message" in rjson and (rjson["message"]=='invalid_token' or rjson["message"]=="expired_token"):
                    meli = Meli(client_id=CLIENT_ID,client_secret=CLIENT_SECRET)
                    url_login_meli = meli.auth_url(redirect_URI=REDIRECT_URI)
                    #print "url_login_meli:", url_login_meli
                    #raise osv.except_osv( _('MELI WARNING'), _('INVALID TOKEN or EXPIRED TOKEN (must login, go to Edit Company and login):  error: %s, message: %s, status: %s') % ( rjson["error"], rjson["message"],rjson["status"],))
                    return warningobj.info(title='MELI WARNING', message="Debe iniciar sesión en MELI.  ", message_html="")
                else:
                     #Any other errors
                    return warningobj.info(title='MELI WARNING', message="Completar todos los campos!  ", message_html="

"+missing_fields )

            #last modifications if response is OK
            if "id" in rjson:
                product.write( { 'meli_id': rjson["id"]} )

            posting_fields = {'posting_date': str(datetime.now()),'meli_id':rjson['id'],'product_id':product.id,'name': 'Post: ' + product.meli_title }

            posting_id = self.env['mercadolibre.posting'].search([('meli_id','=',rjson['id'])])

            if not posting_id:
	            posting_id = self.env['mercadolibre.posting'].create((posting_fields))

        return {}

product_post()
<|endoftext|>


<commit_before>#example survey based on https://github.com/etosch/SurveyMan/blob/master/data/samples/sample3.csv
#outputs JSON representation

from surveyman.survey.survey_representation import *
from surveyman.survey.survey_exceptions import *

def createSurvey():

    b1 = Block([])

    q1 = Question("radio","Block 1a", [])
    q1.addOption("a")
    q1.addOption("b")

    b1.addQuestion(q1)

    q2 = Question("radio","Block 1.1a",[])
    q2.addOption("c")
    q2.addOption("d")

    #trying different ways to add options
    q3 = Question("radio","Block 1.1b",[Option("e"),Option("f")])

    b1_1 = Block([q2,q3])
    b1.addSubblock(b1_1)

    q4 = Question("radio","Block 1b",[Option("g")])

    b1.addQuestion(q4)

    q5 = Question("radio","Block 1.2a",[Option("h")])

    b1_2 = Block([q5])

    q6 = Question("radio","Block 1.2.1b",[Option("X"),Option("Y")])

    q7 = Question("radio","Block 1.2.1a",[Option("j")])

    q8 = Question("radio","Block 1.2.2a",[Option("k")])

    q9 = Question("radio","Block 1.2.2b",[Option("l")])

    q10 = Question("radio","Block 1.2b",[Option("m")])

    b1_2_1 = Block([q6,q7])
    b1_2_2 = Block([q8,q9])

    b1_2.addSubblock(b1_2_1)
    b1_2.addSubblock(b1_2_2)

    q11 = Question("radio","Block 1.3a",[Option("n")])

    q12 = Question("radio","Block 1.3b",[Option("o")])

    q13 = Question("radio","Block 1.3c",[Option("p")])

    b1_3 = Block([q11,q12,q13])

    b1.addSubblock(b1_3)

    q14 = Question("radio","Block 2a",[Option("q"), Option("r")])

    q15 = Question("radio","Block 2c",[Option("s"), Option("t")])

    q16 = Question("radio","Block 3a",[Option("u"), Option("v")])

    b2 = Block([q14,q15])

    b3 = Block([q16])

    q17 = Question("radio","Block 4a", [])
    q17.addOption("w")
    q17.addOption("x")
    q17.addOption("y")
    q17.addOption("z")

    b4 = Block([q17])

    q18 = Question("radio","Block 5.1a",[Option("1"),Option("2"),Option("3")])

    q19 = Question("radio","Block 5.1b",[Option("3"),Option("4")])

    b5_1 = Block([q18,q19])

    q20 = Question("radio","Block 5.2a",[Option("5"),Option("6")])

    q21 = Question("radio","Block 5.2b",[Option("7"),Option("8")])

    b5_2 = Block([q20,q21])

    b5 = Block([b5_1,b5_2])

    survey = Survey([b1,b2,b3,b4,b5],[])

    return survey

def main():
    survey = createSurvey()
    #survey.jsonize()

if  __name__ =='__main__':
    main()
<commit_msg>more PEP whitespace<commit_after>#example survey based on https://github.com/etosch/SurveyMan/blob/master/data/samples/sample3.csv
#outputs JSON representation

from surveyman.survey.survey_representation import *

def create_survey():

    b1 = Block([])

    oneof = "oneof"

    q1 = Question(oneof,"Block 1a", [])
    q1.addOption("a")
    q1.addOption("b")

    b1.addQuestion(q1)

    q2 = Question(oneof, "Block 1.1a", [])
    q2.addOption("c")
    q2.addOption("d")

    #trying different ways to add options
    q3 = Question(oneof, "Block 1.1b", [Option("e"), Option("f")])

    b1_1 = Block([q2,q3])
    b1.addSubblock(b1_1)

    q4 = Question(oneof, "Block 1b", [Option("g")])

    b1.addQuestion(q4)

    q5 = Question(oneof, "Block 1.2a", [Option("h")])

    b1_2 = Block([q5])

    q6 = Question(oneof, "Block 1.2.1b", [Option("X"), Option("Y")])

    q7 = Question(oneof, "Block 1.2.1a", [Option("j")])

    q8 = Question(oneof, "Block 1.2.2a", [Option("k")])

    q9 = Question(oneof, "Block 1.2.2b", [Option("l")])

    q10 = Question(oneof, "Block 1.2b", [Option("m")])

    b1_2_1 = Block([q6, q7])
    b1_2_2 = Block([q8, q9])

    b1_2.addSubblock(b1_2_1)
    b1_2.addSubblock(b1_2_2)

    q11 = Question(oneof, "Block 1.3a", [Option("n")])

    q12 = Question(oneof, "Block 1.3b", [Option("o")])

    q13 = Question(oneof, "Block 1.3c", [Option("p")])

    b1_3 = Block([q11, q12, q13])

    b1.addSubblock(b1_3)

    q14 = Question(oneof, "Block 2a", [Option("q"), Option("r")])

    q15 = Question(oneof, "Block 2c", [Option("s"), Option("t")])

    q16 = Question(oneof, "Block 3a", [Option("u"), Option("v")])

    b2 = Block([q14, q15])

    b3 = Block([q16])

    q17 = Question(oneof, "Block 4a", [])
    q17.addOption("w")
    q17.addOption("x")
    q17.addOption("y")
    q17.addOption("z")

    b4 = Block([q17])

    q18 = Question(oneof, "Block 5.1a", [Option("1"),Option("2"),Option("3")])

    q19 = Question(oneof, "Block 5.1b", [Option("3"),Option("4")])

    b5_1 = Block([q18, q19])

    q20 = Question(oneof, "Block 5.2a", [Option("5"), Option("6")])

    q21 = Question(oneof, "Block 5.2b", [Option("7"), Option("8")])

    b5_2 = Block([q20, q21])

    b5 = Block([b5_1, b5_2])

    survey = Survey([b1, b2, b3, b4, b5], [])

    return survey<|endoftext|>


<commit_before>
# Copyright (c) 2015, 2014 Computational Molecular Biology Group, Free University
# Berlin, 14195 Berlin, Germany.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''
Created on 13.03.2015

@author: marscher
'''
from mdtraj.utils.validation import cast_indices
from mdtraj.core.trajectory import load, _parse_topology, _TOPOLOGY_EXTS, _get_extension, open

def iterload(filename, chunk=100, **kwargs):
    """An iterator over a trajectory from one or more files on disk, in fragments

    This may be more memory efficient than loading an entire trajectory at
    once

    Parameters
    ----------
    filename : str
        Path to the trajectory file on disk
    chunk : int
        Number of frames to load at once from disk per iteration.  If 0, load all.

    Other Parameters
    ----------------
    top : {str, Trajectory, Topology}
        Most trajectory formats do not contain topology information. Pass in
        either the path to a RCSB PDB file, a trajectory, or a topology to
        supply this information. This option is not required for the .h5, .lh5,
        and .pdb formats, which already contain topology information.
    stride : int, default=None
        Only read every stride-th frame.
    atom_indices : array_like, optional
        If not none, then read only a subset of the atoms coordinates from the
        file. This may be slightly slower than the standard read because it
        requires an extra copy, but will save memory.

    See Also
    --------
    load, load_frame

    Examples
    --------

    >>> import mdtraj as md
    >>> for chunk in md.iterload('output.xtc', top='topology.pdb')
    >>>     print chunk

    <mdtraj.Trajectory with 100 frames, 423 atoms at 0x110740a90>
    <mdtraj.Trajectory with 100 frames, 423 atoms at 0x110740a90>
    <mdtraj.Trajectory with 100 frames, 423 atoms at 0x110740a90>
    <mdtraj.Trajectory with 100 frames, 423 atoms at 0x110740a90>
    <mdtraj.Trajectory with 100 frames, 423 atoms at 0x110740a90>

    """
    stride = kwargs.pop('stride', 1)
    atom_indices = cast_indices(kwargs.pop('atom_indices', None))
    top = kwargs.pop('top', None)
    skip = kwargs.pop('skip', 0)

    extension = _get_extension(filename)
    if extension not in _TOPOLOGY_EXTS:
        topology = _parse_topology(top)

    if chunk % stride != 0:
        raise ValueError('Stride must be a divisor of chunk. stride=%d does not go '
                         'evenly into chunk=%d' % (stride, chunk))
    if chunk == 0:
        # If chunk was 0 then we want to avoid filetype-specific code
        # in case of undefined behavior in various file parsers.
        # TODO: this will first apply stride, then skip!
        if extension not in _TOPOLOGY_EXTS:
            kwargs['top'] = top
        yield load(filename, **kwargs)[skip:]
    elif extension in ('.pdb', '.pdb.gz'):
        # the PDBTrajectortFile class doesn't follow the standard API. Fixing it
        # to support iterload could be worthwhile, but requires a deep refactor.
        t = load(filename, stride=stride, atom_indices=atom_indices)
        for i in range(0, len(t), chunk):
            yield t[i:i+chunk]

    else:
        with (lambda x: open(x, n_atoms=topology.n_atoms)
              if extension in ('.crd', '.mdcrd')
              else open(filename))(filename) as f:
            if skip > 0:
                f.seek(skip)
            while True:
                if extension not in _TOPOLOGY_EXTS:
                    traj = f.read_as_traj(topology, n_frames=chunk*stride, stride=stride, atom_indices=atom_indices, **kwargs)
                else:
                    traj = f.read_as_traj(n_frames=chunk*stride, stride=stride, atom_indices=atom_indices, **kwargs)

                if len(traj) == 0:
                    raise StopIteration()

                yield traj
<commit_msg>[coor/util/patches] removed check if stride is divisor of chunksize (so far no effects with multiple tests for lag/stride/chunksize combinations). Asked this mdtraj/mdtraj#881<commit_after>
# Copyright (c) 2015, 2014 Computational Molecular Biology Group, Free University
# Berlin, 14195 Berlin, Germany.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''
Created on 13.03.2015

@author: marscher
'''
from mdtraj.utils.validation import cast_indices
from mdtraj.core.trajectory import load, _parse_topology, _TOPOLOGY_EXTS, _get_extension, open

def iterload(filename, chunk=100, **kwargs):
    """An iterator over a trajectory from one or more files on disk, in fragments

    This may be more memory efficient than loading an entire trajectory at
    once

    Parameters
    ----------
    filename : str
        Path to the trajectory file on disk
    chunk : int
        Number of frames to load at once from disk per iteration.  If 0, load all.

    Other Parameters
    ----------------
    top : {str, Trajectory, Topology}
        Most trajectory formats do not contain topology information. Pass in
        either the path to a RCSB PDB file, a trajectory, or a topology to
        supply this information. This option is not required for the .h5, .lh5,
        and .pdb formats, which already contain topology information.
    stride : int, default=None
        Only read every stride-th frame.
    atom_indices : array_like, optional
        If not none, then read only a subset of the atoms coordinates from the
        file. This may be slightly slower than the standard read because it
        requires an extra copy, but will save memory.

    See Also
    --------
    load, load_frame

    Examples
    --------

    >>> import mdtraj as md
    >>> for chunk in md.iterload('output.xtc', top='topology.pdb')
    >>>     print chunk

    <mdtraj.Trajectory with 100 frames, 423 atoms at 0x110740a90>
    <mdtraj.Trajectory with 100 frames, 423 atoms at 0x110740a90>
    <mdtraj.Trajectory with 100 frames, 423 atoms at 0x110740a90>
    <mdtraj.Trajectory with 100 frames, 423 atoms at 0x110740a90>
    <mdtraj.Trajectory with 100 frames, 423 atoms at 0x110740a90>

    """
    stride = kwargs.pop('stride', 1)
    atom_indices = cast_indices(kwargs.pop('atom_indices', None))
    top = kwargs.pop('top', None)
    skip = kwargs.pop('skip', 0)

    extension = _get_extension(filename)
    if extension not in _TOPOLOGY_EXTS:
        topology = _parse_topology(top)

    if chunk % stride != 0:
        pass
        #raise ValueError('Stride must be a divisor of chunk. stride=%d does not go '
        #                 'evenly into chunk=%d' % (stride, chunk))
    if chunk == 0:
        # If chunk was 0 then we want to avoid filetype-specific code
        # in case of undefined behavior in various file parsers.
        # TODO: this will first apply stride, then skip!
        if extension not in _TOPOLOGY_EXTS:
            kwargs['top'] = top
        yield load(filename, **kwargs)[skip:]
    elif extension in ('.pdb', '.pdb.gz'):
        # the PDBTrajectortFile class doesn't follow the standard API. Fixing it
        # to support iterload could be worthwhile, but requires a deep refactor.
        t = load(filename, stride=stride, atom_indices=atom_indices)
        for i in range(0, len(t), chunk):
            yield t[i:i+chunk]

    else:
        with (lambda x: open(x, n_atoms=topology.n_atoms)
              if extension in ('.crd', '.mdcrd')
              else open(filename))(filename) as f:
            if skip > 0:
                f.seek(skip)
            while True:
                if extension not in _TOPOLOGY_EXTS:
                    traj = f.read_as_traj(topology, n_frames=chunk*stride, stride=stride, atom_indices=atom_indices, **kwargs)
                else:
                    traj = f.read_as_traj(n_frames=chunk*stride, stride=stride, atom_indices=atom_indices, **kwargs)

                if len(traj) == 0:
                    raise StopIteration()

                yield traj
<|endoftext|>


<commit_before># coding=utf-8
import ast
import typing as t
from pathlib import Path
from collections import Counter

from nltk import pos_tag
from nltk.downloader import Downloader

__version__ = '0.0.1'

def flat(_list: t.Iterable) -> list:
    """
    >>> flat([(1,2), (3,4)])
    ... [1, 2, 3, 4]
    """
    return sum([list(item) for item in _list], [])

def is_verb(word: str) -> bool:
    if not word:
        return False
    pos_info = pos_tag([word])
    return pos_info[0][1] == 'VB'

def get_all_files(path: Path) -> t.Iterator[Path]:
    for item in path.iterdir():
        if item.is_dir():
            yield from get_all_files(item)
        elif item.is_file() and item.name.endswith('.py'):
            yield item

def get_trees(path: Path) -> t.Iterator[ast.AST]:
    for file in get_all_files(path):
        content = file.read_text()
        try:
            tree = ast.parse(content)
        except SyntaxError:
            pass
        else:
            yield tree

def get_verbs_from_function_name(function_name):
    return [word for word in function_name.split('_') if is_verb(word)]

def get_all_verbs_in_path(path: Path) -> t.Iterator[str]:
    trees = get_trees(path)
    functions = (node for tree in trees for node in ast.walk(tree)
                 if isinstance(node, ast.FunctionDef) and not (node.name.startswith('__') or node.name.endswith('__')))

    return flat(get_verbs_from_function_name(func.name) for func in functions)

def most_common_verbs(paths: list, count: int) -> t.List[t.Tuple]:
    nltk_downloader = Downloader()
    if not nltk_downloader.is_installed('all'):
        nltk_downloader.download('all')

    words = flat(get_all_verbs_in_path(Path(p)) for p in paths)
    counter = Counter(words)

    return counter.most_common(count)

def show_most_common_verbs(paths: list, count: int):
    commons = most_common_verbs(paths, count)
    for word, quantity in commons:
        print(word, quantity)
<commit_msg>refactor flat<commit_after># coding=utf-8
import ast
import typing as t
from pathlib import Path
from collections import Counter

from nltk import pos_tag
from nltk.downloader import Downloader

__version__ = '0.0.2'

def flat(source: t.Iterable) -> list:
    """
    >>> flat([(1,2), (3,4)])
    ... [1, 2, 3, 4]
    """
    ret = []
    for item in source:
        if isinstance(item, list):
            ret.extend(item)
        else:
            ret.append(item)
    return ret

def is_verb(word: str) -> bool:
    if not word:
        return False
    pos_info = pos_tag([word])
    return pos_info[0][1] == 'VB'

def get_all_files(path: Path) -> t.Iterator[Path]:
    for item in path.iterdir():
        if item.is_dir():
            yield from get_all_files(item)
        elif item.is_file() and item.name.endswith('.py'):
            yield item

def get_trees(path: Path) -> t.Iterator[ast.AST]:
    for file in get_all_files(path):
        content = file.read_text()
        try:
            tree = ast.parse(content)
        except SyntaxError:
            pass
        else:
            yield tree

def get_verbs_from_function_name(function_name):
    return [word for word in function_name.split('_') if is_verb(word)]

def get_all_verbs_in_path(path: Path) -> t.Iterator[str]:
    trees = get_trees(path)
    functions = (node for tree in trees for node in ast.walk(tree)
                 if isinstance(node, ast.FunctionDef) and not (node.name.startswith('__') or node.name.endswith('__')))

    return flat(get_verbs_from_function_name(func.name) for func in functions)

def most_common_verbs(paths: list, count: int) -> t.List[t.Tuple]:
    nltk_downloader = Downloader()
    if not nltk_downloader.is_installed('all'):
        nltk_downloader.download('all')

    words = flat(get_all_verbs_in_path(Path(p)) for p in paths)
    counter = Counter(words)

    return counter.most_common(count)

def show_most_common_verbs(paths: list, count: int):
    commons = most_common_verbs(paths, count)
    for word, quantity in commons:
        print(word, quantity)
<|endoftext|>


<commit_before># coding: utf-8
__version__ = '0.7.4'

# Standard error codes that can be returned from any tool.
OK = 0
USER_ERROR = 4711
CONFIG_ERROR = 4712
SERVER_ERROR = 4713
INTERNAL_ERROR = 4714

import acmd.logger
init_log = acmd.logger.init_log
log = acmd.logger.log
warning = acmd.logger.warning
error = acmd.logger.error

import acmd.server
Server = acmd.server.Server

import acmd.config
read_config = acmd.config.read_config
get_current_config = acmd.config.get_current_config
get_rcfilename = acmd.config.get_rcfilename

import acmd.deploy
setup_rcfile = acmd.deploy.setup_rcfile
deploy_bash_completion = acmd.deploy.deploy_bash_completion

get_current_version = acmd.deploy.get_current_version

import acmd.props
parse_properties = acmd.props.parse_properties

import acmd.tool_repo
import_tools = acmd.tool_repo.import_tools
tool = acmd.tool_repo.tool
list_tools = acmd.tool_repo.list_tools
get_tool = acmd.tool_repo.get_tool
register_tool = acmd.tool_repo.register_tool
set_current_project = acmd.tool_repo.set_current_project

# Has to be last if tools init depend on this file.
import acmd.tools
<commit_msg>Release of import tool demarks 0.8<commit_after># coding: utf-8
__version__ = '0.8.0'

# Standard error codes that can be returned from any tool.
OK = 0
USER_ERROR = 4711
CONFIG_ERROR = 4712
SERVER_ERROR = 4713
INTERNAL_ERROR = 4714

import acmd.logger
init_log = acmd.logger.init_log
log = acmd.logger.log
warning = acmd.logger.warning
error = acmd.logger.error

import acmd.server
Server = acmd.server.Server

import acmd.config
read_config = acmd.config.read_config
get_current_config = acmd.config.get_current_config
get_rcfilename = acmd.config.get_rcfilename

import acmd.deploy
setup_rcfile = acmd.deploy.setup_rcfile
deploy_bash_completion = acmd.deploy.deploy_bash_completion

get_current_version = acmd.deploy.get_current_version

import acmd.props
parse_properties = acmd.props.parse_properties

import acmd.tool_repo
import_tools = acmd.tool_repo.import_tools
tool = acmd.tool_repo.tool
list_tools = acmd.tool_repo.list_tools
get_tool = acmd.tool_repo.get_tool
register_tool = acmd.tool_repo.register_tool
set_current_project = acmd.tool_repo.set_current_project

# Has to be last if tools init depend on this file.
import acmd.tools
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  'variables': {
    'chromium_code': 1,
  },
  'targets': [
    {
      'target_name': 'url',
      'type': '<(component)',
      'dependencies': [
        '../base/base.gyp:base',
        '../third_party/icu/icu.gyp:icudata',
        '../third_party/icu/icu.gyp:icui18n',
        '../third_party/icu/icu.gyp:icuuc',
      ],
      'sources': [
        'gurl.cc',
        'gurl.h',
        'url_canon.h',
        'url_canon_etc.cc',
        'url_canon_fileurl.cc',
        'url_canon_filesystemurl.cc',
        'url_canon_host.cc',
        'url_canon_icu.cc',
        'url_canon_icu.h',
        'url_canon_internal.cc',
        'url_canon_internal.h',
        'url_canon_internal_file.h',
        'url_canon_ip.cc',
        'url_canon_ip.h',
        'url_canon_mailtourl.cc',
        'url_canon_path.cc',
        'url_canon_pathurl.cc',
        'url_canon_query.cc',
        'url_canon_relative.cc',
        'url_canon_stdstring.cc',
        'url_canon_stdstring.h',
        'url_canon_stdurl.cc',
        'url_file.h',
        'url_parse.cc',
        'url_parse.h',
        'url_parse_file.cc',
        'url_parse_internal.h',
        'url_util.cc',
        'url_util.h',
      ],
      'direct_dependent_settings': {
        'include_dirs': [
          '..',
        ],
      },
      'defines': [
        'FULL_FILESYSTEM_URL_SUPPORT=1',
      ],
      'conditions': [
        ['component=="shared_library"',
          {
            'defines': [
              'GURL_DLL',
              'GURL_IMPLEMENTATION=1',
            ],
            'direct_dependent_settings': {
              'defines': [
                'GURL_DLL',
              ],
            },
          }
        ],
      ],
      # TODO(jschuh): crbug.com/167187 fix size_t to int truncations.
      'msvs_disabled_warnings': [4267, ],
    },
    {
      'target_name': 'url_unittests',
      'type': 'executable',
      'dependencies': [
        'url',
        '../base/base.gyp:base_i18n',
        '../base/base.gyp:run_all_unittests',
        '../testing/gtest.gyp:gtest',
        '../third_party/icu/icu.gyp:icuuc',
      ],
      'sources': [
        'gurl_unittest.cc',
        'url_canon_unittest.cc',
        'url_parse_unittest.cc',
        'url_test_utils.h',
        'url_util_unittest.cc',
      ],
      'defines': [
        'FULL_FILESYSTEM_URL_SUPPORT=1',
      ],
      'conditions': [
        ['os_posix==1 and OS!="mac" and OS!="ios"',
          {
            'conditions': [
              ['linux_use_tcmalloc==1',
                {
                  'dependencies': [
                    '../base/allocator/allocator.gyp:allocator',
                  ],
                }
              ],
            ],
          }
        ],
      ],
      # TODO(jschuh): crbug.com/167187 fix size_t to int truncations.
      'msvs_disabled_warnings': [4267, ],
    },
    {
      'target_name': 'googleurl_unittests',
      'type': 'none',
      'dependencies': [
        'url_unittests',
      ],
      'conditions': [
        ['OS != "ios"',
          {
            'actions': [
              {
                'message': 'TEMPORARY: Copying url_unittests to googleurl_unittests',
                'variables': {
                  'url_copy_target': '<(PRODUCT_DIR)/url_unittests<(EXECUTABLE_SUFFIX)',
                  'url_copy_dest': '<(PRODUCT_DIR)/googleurl_unittests<(EXECUTABLE_SUFFIX)',
                },
                'inputs': ['<(url_copy_target)'],
                'outputs': ['<(url_copy_dest)'],
                'action_name': 'TEMP_copy_url_unittests',
                'action': [
                  'python', '-c',
                  'import os, shutil, stat; ' \
                  'shutil.copyfile(\'<(url_copy_target)\', \'<(url_copy_dest)\'); ' \
                  'os.chmod(\'<(url_copy_dest)\', ' \
                  'stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)'
                ],
              },
            ],
          }
        ],
        ['OS == "ios"',
          {
            'actions': [
              {
                'message': 'TEMPORARY: Copying url_unittests to googleurl_unittests',
                'variables': {
                  'url_copy_target': '<(PRODUCT_DIR)/url_unittests.app/url_unittests',
                  'url_copy_dest': '<(PRODUCT_DIR)/googleurl_unittests.app/googleurl_unittests',
                },
                'inputs': ['<(url_copy_target)'],
                'outputs': ['<(url_copy_dest)'],
                'action_name': 'TEMP_copy_url_unittests',
                'action': [
                  'python', '-c',
                  'import os, shutil, stat; ' \
                  'shutil.copyfile(\'<(url_copy_target)\', \'<(url_copy_dest)\'); ' \
                  'os.chmod(\'<(url_copy_dest)\', ' \
                  'stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)'
                ],
              },
            ],
          }
        ]
      ],
    },
  ],
}
<commit_msg>Revert 194082 "url: Copy 'googleurl_unittests' target from 'url_..."<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  'variables': {
    'chromium_code': 1,
  },
  'targets': [
    {
      'target_name': 'url',
      'type': '<(component)',
      'dependencies': [
        '../base/base.gyp:base',
        '../third_party/icu/icu.gyp:icudata',
        '../third_party/icu/icu.gyp:icui18n',
        '../third_party/icu/icu.gyp:icuuc',
      ],
      'sources': [
        'gurl.cc',
        'gurl.h',
        'url_canon.h',
        'url_canon_etc.cc',
        'url_canon_fileurl.cc',
        'url_canon_filesystemurl.cc',
        'url_canon_host.cc',
        'url_canon_icu.cc',
        'url_canon_icu.h',
        'url_canon_internal.cc',
        'url_canon_internal.h',
        'url_canon_internal_file.h',
        'url_canon_ip.cc',
        'url_canon_ip.h',
        'url_canon_mailtourl.cc',
        'url_canon_path.cc',
        'url_canon_pathurl.cc',
        'url_canon_query.cc',
        'url_canon_relative.cc',
        'url_canon_stdstring.cc',
        'url_canon_stdstring.h',
        'url_canon_stdurl.cc',
        'url_file.h',
        'url_parse.cc',
        'url_parse.h',
        'url_parse_file.cc',
        'url_parse_internal.h',
        'url_util.cc',
        'url_util.h',
      ],
      'direct_dependent_settings': {
        'include_dirs': [
          '..',
        ],
      },
      'defines': [
        'FULL_FILESYSTEM_URL_SUPPORT=1',
      ],
      'conditions': [
        ['component=="shared_library"', {
          'defines': [
            'GURL_DLL',
            'GURL_IMPLEMENTATION=1',
          ],
          'direct_dependent_settings': {
            'defines': [
              'GURL_DLL',
            ],
          },
        }],
      ],
      # TODO(jschuh): crbug.com/167187 fix size_t to int truncations.
      'msvs_disabled_warnings': [4267, ],
    },
    {
      'target_name': 'googleurl_unittests',
      'type': 'executable',
      'dependencies': [
        'url',
        '../base/base.gyp:base_i18n',
        '../base/base.gyp:run_all_unittests',
        '../testing/gtest.gyp:gtest',
        '../third_party/icu/icu.gyp:icuuc',
      ],
      'sources': [
        'gurl_unittest.cc',
        'url_canon_unittest.cc',
        'url_parse_unittest.cc',
        'url_test_utils.h',
        'url_util_unittest.cc',
      ],
      'defines': [
        'FULL_FILESYSTEM_URL_SUPPORT=1',
      ],
      'conditions': [
        ['os_posix==1 and OS!="mac" and OS!="ios"', {
          'conditions': [
            ['linux_use_tcmalloc==1', {
              'dependencies': [
                '../base/allocator/allocator.gyp:allocator',
              ],
            }],
          ],
        }],
      ],
      # TODO(jschuh): crbug.com/167187 fix size_t to int truncations.
      'msvs_disabled_warnings': [4267, ],
    },
  ],
}
<|endoftext|>


<commit_before>## Alarm Server
## Supporting Envisalink 2DS/3
##
## This code is under the terms of the GPL v3 license.

evl_Commands = {
    'KeepAlive' : '000',
    'StatusReport' : '001',
    'DumpZoneTimers' : '008',
    'PartitionKeypress' : '071',
    'Disarm' : '040',
    'ArmStay' : '031',
    'ArmAway' : '030',
    'ArmMax' : '032',
    'Login' : '005',
    'Panic' : '060',
    'SendCode' : '200'
}

evl_PanicTypes = {
    'Fire' : '1',
    'Ambulance' : '2',
    'Police' : '3'
}

evl_ArmModes = {
        '0' : {'name' : 'Arm Away', 'status':{'armed_away': True, 'alpha':'Arm Away', 'exit_delay':False}},
        '1' : {'name' : 'Arm Stay', 'status':{'armed_stay': True, 'alpha':'Arm Stay', 'exit_delay':False}},
        '2' : {'name' : 'Arm Zero Entry Away', 'status':{'armed_away': True, 'armed_zero_entry_delay': True, 'alpha':'Arm Zero Entry Away', 'exit_delay':False}},
        '3' : {'name' : 'Arm Zero Entry Stay', 'status':{'armed_stay': True, 'armed_zero_entry_delay': True, 'alpha':'Arm Zero Entry Stay', 'exit_delay':False}}
    }

evl_ResponseTypes = {
    '505' : {'name':'Login Prompt', 'handler':'login'},
    '615' : {'name':'Envisalink Zone Timer Dump', 'handler':'zone_timer_dump'},
    '500' : {'name':'Poll', 'handler':'poll_response'},
    '501' : {'name':'Checksum', 'handler':'command_response_error'},
    '900' : {'name':'EnterCode', 'handler':'send_code'},

#ZONE UPDATES

    '601' : {'name':'Zone Alarm', 'handler':'zone_state_change', 'status':{'alarm' : True}},
    '602' : {'name':'Zone Alarm Restore', 'handler':'zone_state_change', 'status':{'alarm' : False}},
    '603' : {'name':'Zone Tamper', 'handler':'zone_state_change', 'status':{'tamper' : True}},
    '604' : {'name':'Zone Tamper Restore', 'handler':'zone_state_change', 'status':{'tamper' : False}},
    '605' : {'name':'Zone Fault', 'handler':'zone_state_change', 'status':{'fault' : True}},
    '606' : {'name':'Zone Fault Restore', 'handler':'zone_state_change', 'status':{'fault' : False}},
    '609' : {'name':'Zone Open', 'handler':'zone_state_change', 'status':{'open' : True}},
    '610' : {'name':'Zone Restored', 'handler':'zone_state_change', 'status':{'open' : False}},

#PARTITION UPDATES
    '650' : {'name':'Ready', 'handler':'partition_state_change', 'status':{'ready' : True, 'alpha' : 'Ready'}},
    '651' : {'name':'Not Ready', 'handler':'partition_state_change', 'status':{'ready' : False, 'alpha' : 'Not Ready'}},
    '652' : {'name':'Armed', 'handler':'partition_state_change'},
    '653' : {'name':'Ready - Force Arming Enabled', 'handler':'partition_state_change', 'status':{'ready': True, 'alpha' : 'Ready - Force Arm'}},
    '654' : {'name':'Alarm', 'handler':'partition_state_change', 'status':{'alarm' : True, 'alpha' : 'Alarm'}},
    '655' : {'name':'Disarmed', 'handler':'partition_state_change', 'status' : {'alarm' : False, 'armed_stay' : False, 'armed_away' : False, 'exit_delay' : False, 'entry_delay' : False, 'alpha' : 'Disarmed'}},
    '656' : {'name':'Exit Delay in Progress', 'handler':'partition_state_change', 'status':{'exit_delay' : True, 'alpha' : 'Exit Delay In Progress'}},
    '657' : {'name':'Entry Delay in Progress', 'handler':'partition_state_change', 'status':{'entry_delay' : True, 'alpha' : 'Entry Delay in Progress'}},
    '663' : {'name':'ChimeOn', 'handler':'partition_state_change', 'status': {'chime': True}},
    '664' : {'name':'ChimeOff', 'handler':'partition_state_change', 'status': {'chime': False}},
    '673' : {'name':'Busy', 'handler':'partition_state_change', 'status': {'alpha': 'Busy'}},
    '750' : {'name':'Disarmed by user', 'handler':'partition_state_change', 'status' : {'alarm' : False, 'armed_stay' : False, 'armed_away' : False, 'exit_delay' : False, 'entry_delay' : False, 'alpha' : 'Disarmed'}},
    '751' : {'name':'Disarmed special', 'handler':'partition_state_change', 'status' : {'alarm' : False, 'armed_stay' : False, 'armed_away' : False, 'exit_delay' : False, 'entry_delay' : False, 'alpha' : 'Disarmed'}},
    '840' : {'name':'Trouble LED', 'handler':'partition_state_change', 'status':{'trouble' : True}},
    '841' : {'name':'Trouble Clear', 'handler':'partition_state_change', 'status':{'trouble' : False}},

#GENERAL UPDATES
    '621' : {'name':'FireAlarmButton', 'handler':'keypad_update', 'status':{'fire' : True, 'alarm': True, 'alpha' : 'Fire Alarm'}},
    '622' : {'name':'FireAlarmButtonOff', 'handler':'keypad_update', 'status':{'fire' : False, 'alarm': False, 'alpha' : 'Fire Alarm Cleared'}},
    '623' : {'name':'AuxAlarmButton', 'handler':'keypad_update', 'status':{'alarm': True, 'alpha' : 'Aux Alarm'}},
    '624' : {'name':'AuxAlarmButtonOff', 'handler':'keypad_update', 'status':{'alarm': False, 'alpha' : 'Aux Alarm Cleared'}},
    '625' : {'name':'PanicAlarmButton', 'handler':'keypad_update', 'status':{'alarm': True, 'alpha' : 'Panic Alarm'}},
    '626' : {'name':'PanicAlarmButtonOff', 'handler':'keypad_update', 'status':{'alarm': False, 'alpha' : 'Panic Alarm Cleared'}},
    '631' : {'name':'SmokeAlarmButton', 'handler':'keypad_update', 'status':{'alarm': True, 'alpha' : 'Smoke Alarm'}},
    '632' : {'name':'SmokeAlarmButtonOff', 'handler':'keypad_update', 'status':{'alarm': False, 'alpha' : 'Smoke Alarm Cleared'}},
    '800' : {'name':'LowBatTrouble', 'handler':'keypad_update', 'status':{'bat_trouble': True, 'alpha' : 'Low Battery'}},
    '801' : {'name':'LowBatTroubleOff', 'handler':'keypad_update', 'status':{'bat_trouble': False, 'alpha' : 'Low Battery Cleared'}},
    '802' : {'name':'ACTrouble', 'handler':'keypad_update', 'status':{'ac_present': False, 'alpha' : 'AC Power Lost'}},
    '803' : {'name':'ACTroubleOff', 'handler':'keypad_update', 'status':{'ac_present': True, 'alpha' : 'AC Power Restored'}},
    '829' : {'name':'SystemTamper', 'handler':'keypad_update', 'status':{'alpha' : 'System tamper'}},
    '830' : {'name':'SystemTamperOff', 'handler':'keypad_update', 'status':{'alpha' : 'System tamper Restored'}},
}
<commit_msg>Add Command Output Control command<commit_after>## Alarm Server
## Supporting Envisalink 2DS/3
##
## This code is under the terms of the GPL v3 license.

evl_Commands = {
    'KeepAlive' : '000',
    'StatusReport' : '001',
    'DumpZoneTimers' : '008',
    'PartitionKeypress' : '071',
    'Disarm' : '040',
    'ArmStay' : '031',
    'ArmAway' : '030',
    'ArmMax' : '032',
    'Login' : '005',
    'Panic' : '060',
    'SendCode' : '200',
    'CommandOutput' : '020'
}

evl_PanicTypes = {
    'Fire' : '1',
    'Ambulance' : '2',
    'Police' : '3'
}

evl_ArmModes = {
        '0' : {'name' : 'Arm Away', 'status':{'armed_away': True, 'alpha':'Arm Away', 'exit_delay':False}},
        '1' : {'name' : 'Arm Stay', 'status':{'armed_stay': True, 'alpha':'Arm Stay', 'exit_delay':False}},
        '2' : {'name' : 'Arm Zero Entry Away', 'status':{'armed_away': True, 'armed_zero_entry_delay': True, 'alpha':'Arm Zero Entry Away', 'exit_delay':False}},
        '3' : {'name' : 'Arm Zero Entry Stay', 'status':{'armed_stay': True, 'armed_zero_entry_delay': True, 'alpha':'Arm Zero Entry Stay', 'exit_delay':False}}
    }

evl_ResponseTypes = {
    '505' : {'name':'Login Prompt', 'handler':'login'},
    '615' : {'name':'Envisalink Zone Timer Dump', 'handler':'zone_timer_dump'},
    '500' : {'name':'Poll', 'handler':'poll_response'},
    '501' : {'name':'Checksum', 'handler':'command_response_error'},
    '900' : {'name':'EnterCode', 'handler':'send_code'},

#ZONE UPDATES

    '601' : {'name':'Zone Alarm', 'handler':'zone_state_change', 'status':{'alarm' : True}},
    '602' : {'name':'Zone Alarm Restore', 'handler':'zone_state_change', 'status':{'alarm' : False}},
    '603' : {'name':'Zone Tamper', 'handler':'zone_state_change', 'status':{'tamper' : True}},
    '604' : {'name':'Zone Tamper Restore', 'handler':'zone_state_change', 'status':{'tamper' : False}},
    '605' : {'name':'Zone Fault', 'handler':'zone_state_change', 'status':{'fault' : True}},
    '606' : {'name':'Zone Fault Restore', 'handler':'zone_state_change', 'status':{'fault' : False}},
    '609' : {'name':'Zone Open', 'handler':'zone_state_change', 'status':{'open' : True}},
    '610' : {'name':'Zone Restored', 'handler':'zone_state_change', 'status':{'open' : False}},

#PARTITION UPDATES
    '650' : {'name':'Ready', 'handler':'partition_state_change', 'status':{'ready' : True, 'alpha' : 'Ready'}},
    '651' : {'name':'Not Ready', 'handler':'partition_state_change', 'status':{'ready' : False, 'alpha' : 'Not Ready'}},
    '652' : {'name':'Armed', 'handler':'partition_state_change'},
    '653' : {'name':'Ready - Force Arming Enabled', 'handler':'partition_state_change', 'status':{'ready': True, 'alpha' : 'Ready - Force Arm'}},
    '654' : {'name':'Alarm', 'handler':'partition_state_change', 'status':{'alarm' : True, 'alpha' : 'Alarm'}},
    '655' : {'name':'Disarmed', 'handler':'partition_state_change', 'status' : {'alarm' : False, 'armed_stay' : False, 'armed_away' : False, 'exit_delay' : False, 'entry_delay' : False, 'alpha' : 'Disarmed'}},
    '656' : {'name':'Exit Delay in Progress', 'handler':'partition_state_change', 'status':{'exit_delay' : True, 'alpha' : 'Exit Delay In Progress'}},
    '657' : {'name':'Entry Delay in Progress', 'handler':'partition_state_change', 'status':{'entry_delay' : True, 'alpha' : 'Entry Delay in Progress'}},
    '663' : {'name':'ChimeOn', 'handler':'partition_state_change', 'status': {'chime': True}},
    '664' : {'name':'ChimeOff', 'handler':'partition_state_change', 'status': {'chime': False}},
    '673' : {'name':'Busy', 'handler':'partition_state_change', 'status': {'alpha': 'Busy'}},
    '750' : {'name':'Disarmed by user', 'handler':'partition_state_change', 'status' : {'alarm' : False, 'armed_stay' : False, 'armed_away' : False, 'exit_delay' : False, 'entry_delay' : False, 'alpha' : 'Disarmed'}},
    '751' : {'name':'Disarmed special', 'handler':'partition_state_change', 'status' : {'alarm' : False, 'armed_stay' : False, 'armed_away' : False, 'exit_delay' : False, 'entry_delay' : False, 'alpha' : 'Disarmed'}},
    '840' : {'name':'Trouble LED', 'handler':'partition_state_change', 'status':{'trouble' : True}},
    '841' : {'name':'Trouble Clear', 'handler':'partition_state_change', 'status':{'trouble' : False}},

#GENERAL UPDATES
    '621' : {'name':'FireAlarmButton', 'handler':'keypad_update', 'status':{'fire' : True, 'alarm': True, 'alpha' : 'Fire Alarm'}},
    '622' : {'name':'FireAlarmButtonOff', 'handler':'keypad_update', 'status':{'fire' : False, 'alarm': False, 'alpha' : 'Fire Alarm Cleared'}},
    '623' : {'name':'AuxAlarmButton', 'handler':'keypad_update', 'status':{'alarm': True, 'alpha' : 'Aux Alarm'}},
    '624' : {'name':'AuxAlarmButtonOff', 'handler':'keypad_update', 'status':{'alarm': False, 'alpha' : 'Aux Alarm Cleared'}},
    '625' : {'name':'PanicAlarmButton', 'handler':'keypad_update', 'status':{'alarm': True, 'alpha' : 'Panic Alarm'}},
    '626' : {'name':'PanicAlarmButtonOff', 'handler':'keypad_update', 'status':{'alarm': False, 'alpha' : 'Panic Alarm Cleared'}},
    '631' : {'name':'SmokeAlarmButton', 'handler':'keypad_update', 'status':{'alarm': True, 'alpha' : 'Smoke Alarm'}},
    '632' : {'name':'SmokeAlarmButtonOff', 'handler':'keypad_update', 'status':{'alarm': False, 'alpha' : 'Smoke Alarm Cleared'}},
    '800' : {'name':'LowBatTrouble', 'handler':'keypad_update', 'status':{'bat_trouble': True, 'alpha' : 'Low Battery'}},
    '801' : {'name':'LowBatTroubleOff', 'handler':'keypad_update', 'status':{'bat_trouble': False, 'alpha' : 'Low Battery Cleared'}},
    '802' : {'name':'ACTrouble', 'handler':'keypad_update', 'status':{'ac_present': False, 'alpha' : 'AC Power Lost'}},
    '803' : {'name':'ACTroubleOff', 'handler':'keypad_update', 'status':{'ac_present': True, 'alpha' : 'AC Power Restored'}},
    '829' : {'name':'SystemTamper', 'handler':'keypad_update', 'status':{'alpha' : 'System tamper'}},
    '830' : {'name':'SystemTamperOff', 'handler':'keypad_update', 'status':{'alpha' : 'System tamper Restored'}},
}
<|endoftext|>


<commit_before>"""Script for provisioning build server.

Use: `$ python cltk/utils/download_misc_dependencies.py`
"""

import os
from typing import Any, List

from stanfordnlp.utils.resources import download

from cltkv1.embeddings.embeddings import FastTextEmbeddings

def get_stanfordnlp_models(force_update: bool = True, lang: str = "all") -> None:
    """Download language models, from the ``stanfordnlp`` project,
    that are supported by the CLTK or in scope. More here:
    `<https://stanfordnlp.github.io/stanfordnlp/models.html>_.
    """
    all_ud_models_for_cltk = [
        "grc_perseus",
        "grc_proiel",
        "la_ittb",
        "la_perseus",
        "la_proiel",
        "cu_proiel",  # Old Church Slavonic
        "fro_srcmf",  # Old French
        "got_proiel",
    ]  # type: List[str]
    ud_models_for_dl = list()
    if lang == "all":
        ud_models_for_dl = all_ud_models_for_cltk
    elif lang == "chu":
        ud_models_for_dl = ud_models_for_dl + ["cu_proiel"]
    elif lang == "fro":
        ud_models_for_dl = ud_models_for_dl + ["fro_srcmf"]
    elif lang == "got_proiel":
        ud_models_for_dl = ud_models_for_dl + ["got_proiel"]
    elif lang == "grc":
        ud_models_for_dl = ud_models_for_dl + ["grc_perseus", "grc_proiel"]
    elif lang == "lat":
        ud_models_for_dl = ud_models_for_dl + ["la_ittb", "la_perseus", "la_proiel"]
    else:
        raise ValueError(f"No models for lang  '{lang}'.")
    # TODO: rm this check
    stanford_dir = os.path.expanduser("~/stanfordnlp_resources/")  # type: str
    for model in ud_models_for_dl:
        download(
            download_label=model,
            resource_dir=stanford_dir,
            confirm_if_exists=True,
            force=force_update,
        )

def get_fasttext_models(force: bool = True, lang: str = "all"):
    all_wiki_models = ["arb", "arc", "got", "lat", "pli", "san", "xno"]
    # all_common_crawl_models = ["arb", "lat", "san"]
    # wiki_models = list()
    # common_crawl_models = list()
    # if lang == "all":
    #     wiki_models = all_wiki_models
    #     common_crawl_models = all_common_crawl_models
    # else:
    #     assert (
    #         lang in all_wiki_models or lang in all_common_crawl_models
    #     ), f"ISO code '{lang}' either not among valid either 'wiki' or 'common_crawl' models."
    #     wiki_models.append(lang)
    #     common_crawl_models.append(lang)
    for lang in all_wiki_models:
        pass
        # fasttext_class = FastTextEmbeddings(iso_code=lang)
        # fasttext_class.download_fasttext_models()

if __name__ == "__main__":
    # TODO: add command line params for what langs (all or just one); useful for build server
    # get_stanfordnlp_models(force_update=True, lang="all")
    # get_fasttext_models(force=False, lang="lat")
    get_fasttext_models()
<commit_msg>re-enable stanfordnlp model download<commit_after>"""Script for provisioning build server.

Use: `$ python cltk/utils/download_misc_dependencies.py`
"""

import os
from typing import Any, List

from stanfordnlp.utils.resources import download

from cltkv1.embeddings.embeddings import FastTextEmbeddings

def get_stanfordnlp_models(force_update: bool = True, lang: str = "all") -> None:
    """Download language models, from the ``stanfordnlp`` project,
    that are supported by the CLTK or in scope. More here:
    `<https://stanfordnlp.github.io/stanfordnlp/models.html>_.
    """
    all_ud_models_for_cltk = [
        "grc_perseus",
        "grc_proiel",
        "la_ittb",
        "la_perseus",
        "la_proiel",
        "cu_proiel",  # Old Church Slavonic
        "fro_srcmf",  # Old French
        "got_proiel",
    ]  # type: List[str]
    ud_models_for_dl = list()
    if lang == "all":
        ud_models_for_dl = all_ud_models_for_cltk
    elif lang == "chu":
        ud_models_for_dl = ud_models_for_dl + ["cu_proiel"]
    elif lang == "fro":
        ud_models_for_dl = ud_models_for_dl + ["fro_srcmf"]
    elif lang == "got_proiel":
        ud_models_for_dl = ud_models_for_dl + ["got_proiel"]
    elif lang == "grc":
        ud_models_for_dl = ud_models_for_dl + ["grc_perseus", "grc_proiel"]
    elif lang == "lat":
        ud_models_for_dl = ud_models_for_dl + ["la_ittb", "la_perseus", "la_proiel"]
    else:
        raise ValueError(f"No models for lang  '{lang}'.")
    # TODO: rm this check
    stanford_dir = os.path.expanduser("~/stanfordnlp_resources/")  # type: str
    for model in ud_models_for_dl:
        download(
            download_label=model,
            resource_dir=stanford_dir,
            confirm_if_exists=True,
            force=force_update,
        )

def get_fasttext_models(force: bool = True, lang: str = "all"):
    all_wiki_models = ["arb", "arc", "got", "lat", "pli", "san", "xno"]
    # all_common_crawl_models = ["arb", "lat", "san"]
    # wiki_models = list()
    # common_crawl_models = list()
    # if lang == "all":
    #     wiki_models = all_wiki_models
    #     common_crawl_models = all_common_crawl_models
    # else:
    #     assert (
    #         lang in all_wiki_models or lang in all_common_crawl_models
    #     ), f"ISO code '{lang}' either not among valid either 'wiki' or 'common_crawl' models."
    #     wiki_models.append(lang)
    #     common_crawl_models.append(lang)
    for lang in all_wiki_models:
        pass
        # fasttext_class = FastTextEmbeddings(iso_code=lang)
        # fasttext_class.download_fasttext_models()

if __name__ == "__main__":
    # TODO: add command line params for what langs (all or just one); useful for build server
    get_stanfordnlp_models(force_update=True, lang="all")
    # get_fasttext_models(force=False, lang="lat")
    # get_fasttext_models()
<|endoftext|>


<commit_before>import requests
import dataset
import os
import re
import datetime as dt
import time
import traceback
from tqdm import tqdm
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, ElementNotVisibleException
from dateutil.parser import parse as dtparse

# Number of seconds to sleep between crawls
CRAWL_TIMEOUT = 10

THIRTY_DAY_REVIEW_REGEX = re.compile(r'^([0-9]+)% of the ([,0-9]+) user reviews in the last 30 days')
ALL_TIME_REVIEW_REGEX = re.compile(r'^([0-9]+)% of the ([,0-9]+) user reviews for this game')
DETAILS_BOX_REGEX = re.compile(r'^Title: ([^\n]+)(?:\nGenre: ([^\n]+))?(?:\nDeveloper: ([^\n]+))?(?:\nPublisher: ([^\n]+))?')
NUM_ACHIEVEMENTS_REGEX = re.compile(r'Includes ([,[0-9]+) Steam Achievements')

def upsert_all_apps(db):
    '''
    Get the full list of steam apps and upsert them in our database
    on the basis of steam's app ID.
    '''
    json = requests.get('http://api.steampowered.com/ISteamApps/GetAppList/v0001/').json()

    apps = json['applist']['apps']['app']

    db.begin()

    for app in tqdm(apps):
        db['game'].upsert({
            'steam_app_id': app['appid'],
            'game_name': app['name'],
        }, keys=['app_id'])

    db.commit()

def scrape_store_page(app_id):
    '''
    Extract all the information we can from the store page for a given app ID.
    '''
    results = {'steam_app_id': app_id}
    driver = webdriver.Chrome()
    store_base_url = "http://store.steampowered.com"
    app_url = "{}/app/{}".format(store_base_url, app_id)
    driver.get(app_url)

    try:
        try:
            # If this succeeds, we need to pass through the age gate.
            driver.find_element_by_id('agegate_box')

            select_element = driver.find_element_by_id('ageYear')
            # open year dialog
            select_element.click()
            # select correct year
            select_element.find_element_by_css_selector('option[value="1993"]').click()
            # close year dialog
            select_element.click()
            # submit the form
            driver.find_element_by_id('agecheck_form').submit()
        except NoSuchElementException:
            # No age gate; we're good to continue
            pass

        if driver.current_url in (store_base_url, '{}/'.format(store_base_url)):
            # We were redirected; the app doesn't have a store page.
            driver.close()
            return results
        elif 'store.steampowered.com/video' in driver.current_url:
            # This is a trailer for something else; we'll get the actual app later.
            driver.close()
            return results

        try:
            # If this succeeds, we've got a steam store error
            error_element = driver.find_element_by_id('error_box')
            error_text = error_element.find_element_by_class_name('error')
            if error_text.text == 'This item is currently unavailable in your region':
                # We can't see this app; ignore it
                driver.close()
                return results
        except NoSuchElementException:
            pass

        try:
            # If this succeeds, Chrome is showing us an error
            error_element = driver.find_element_by_class('error-code')
            if error_element.text == 'ERR_TOO_MANY_REDIRECTS':
                # Something wonky with the server response for this store page;
                # it's redirecting infinitely to itself.  Ignore it
                driver.close()
                return results
        except NoSuchElementException:
            pass

        # Get the description first, since it tells us whether the app is streaming video
        # (which means we don't care about it)
        descriptions = driver.find_elements_by_class_name('game_area_description')
        for description in descriptions:
            if description.text.startswith('ABOUT THIS GAME'):
                results['is_dlc'] = False
                results['long_description'] = description.text
            elif description.text.startswith('ABOUT THIS CONTENT'):
                results['is_dlc'] = True
                results['long_description'] = description.text
            elif description.text.startswith('ABOUT THIS SERIES'):
                # This is streaming video; we don't care about it, so just return
                driver.close()
                return results
            elif description.text.startswith('ABOUT THIS SOFTWARE'):
                # This is computer software; ignore it
                driver.close()
                return results
        if 'long_description' not in results and len(descriptions) > 0:
            raise RuntimeError('Unable to parse description for app_id {}'.format(app_id))

        results['game_name'] = (driver
                                .find_element_by_class_name('apphub_AppName')
                                .text)

        try:
            results['short_description'] = (driver
                                            .find_element_by_class_name('game_description_snippet')
                                            .text)
        except NoSuchElementException:
            # DLC doesn't have a short description
            pass

        reviews_texts = [element.get_attribute('data-store-tooltip')
                        for element in (driver
                                        .find_elements_by_class_name('user_reviews_summary_row')
                                        )]

        for text in reviews_texts:
            thirty_day_match = THIRTY_DAY_REVIEW_REGEX.match(text)

            if thirty_day_match:
                results['pct_positive_reviews_last_30_days'] = int(thirty_day_match.group(1))
                results['reviews_last_30_days'] = int(thirty_day_match.group(2).replace(',', ''))
            else:
                all_time_match = ALL_TIME_REVIEW_REGEX.match(text)

                if all_time_match:
                    results['pct_positive_reviews_all_time'] = int(all_time_match.group(1))
                    results['reviews_all_time'] = int(all_time_match.group(2).replace(',', ''))

        try:
            results['release_date'] = dtparse(driver.find_element_by_css_selector('.release_date .date').text)
        except NoSuchElementException:
            # This app doesn't have a release date for some reason
            pass

        # There's additional detail about VR stuff here, but we're not worried about that for now
        details_text = driver.find_elements_by_css_selector('.details_block:not(.vrsupport)')[0].text
        details_match = DETAILS_BOX_REGEX.match(details_text)
        results['title'] = details_match.group(1)
        raw_genre = details_match.group(2)
        if raw_genre is not None:
            results['genres'] = raw_genre.split(', ')
        results['developer'] = details_match.group(3)
        results['publisher'] = details_match.group(4)

        block_titles_texts = [element.text
                            for element in (driver
                                            .find_elements_by_class_name('block_title')
                                            )]

        for text in block_titles_texts:
            num_achievements_match = NUM_ACHIEVEMENTS_REGEX.match(text)

            if num_achievements_match:
                results['num_achievements'] = int(num_achievements_match.group(1))

        try:
            raw_metacritic_score = driver.find_element_by_class_name('score').text
            if raw_metacritic_score != 'NA':
                results['metacritic_score'] = int(raw_metacritic_score)
        except NoSuchElementException:
            # Some games don't have metascores
            pass

        try:
            # Not on sale or Free to Play
            raw_price = driver.find_element_by_class_name('game_purchase_price').text
            if raw_price in ('Free to Play', 'Free'):
                price = 0
            elif raw_price == 'Third-party':
                # For all examples thus far, this has meant "free", but I don't think
                # we can assume that if the source is a 3rd party
                price = None
            else:
                price = float(raw_price.replace('$', ''))
            results['full_price'] = price
        except NoSuchElementException:
            try:
                # On sale
                raw_price = driver.find_element_by_class_name('discount_original_price').text
                results['full_price'] = float(raw_price.replace('$', ''))
            except NoSuchElementException:
                # No price on the page
                pass

        results['game_details'] = []
        game_details_elements = driver.find_elements_by_class_name('game_area_details_specs')
        for element in game_details_elements:
            results['game_details'].append(element.find_element_by_css_selector('a.name').text)

        results['tags'] = []
        try:
            # Try to get the big list of tags if it's there
            driver.find_element_by_css_selector('.app_tag.add_button').click()
            tag_elements = driver.find_elements_by_css_selector('#app_tagging_modal a.app_tag')
        except (NoSuchElementException, ElementNotVisibleException):
            # Settle for the short list if not
            tag_elements = driver.find_elements_by_css_selector('a.app_tag')
        for element in tag_elements:
            results['tags'].append(element.text)

        driver.close()

        return results
    except:
        # Make sure our driver gets closed if we threw an error
        # so we don't leave a million windows open
        driver.close()
        raise

def insert_with_mapping(*, db, descrs, entity_table, pk_name, join_table, mapping, app_id, crawl_time):
    '''
    Given a db connection, some list of description data, a table of entities, the name of the PK,
    a many-to-many join table for the entities to game crawls,
    a mapping from descrs to entity table IDs, and an app_id/crawl_time,
    update the entity table/mapping if necessary and insert
    the entity in the many-to-many join table.

    Mutates the mapping.
    '''
    for descr in descrs:
        try:
            entity_id = mapping[descr]
        except KeyError:
            entity_id = db[entity_table].insert({'descr': descr})
            mapping[descr] = entity_id

        db[join_table].insert({
            'steam_app_id': app_id,
            'crawl_time': crawl_time,
            pk_name: entity_id,
        })

def do_crawl(app_ids, db):
    '''
    Given a list of steam app IDs and a db connection, do a crawl for the app IDs
    and append the results to our list of crawls in the database.
    '''
    tag_mapping = {r['descr']: r['tag_id'] for r in db['steam_tag'].find()}
    detail_mapping = {r['descr']: r['detail_id'] for r in db['steam_game_detail'].find()}
    genre_mapping = {r['descr']: r['genre_id'] for r in db['steam_genre'].find()}

    # Add a handler here to allow us to gracefully save our work and quit
    # if the user halts a crawl early via Ctrl + C.
    # NOTE: since background jobs ignore SIGINT, this has no effect if
    # the script is launched in the background; see
    # http://stackoverflow.com/questions/1112343/how-do-i-capture-sigint-in-python#comment68802096_1112357
    should_quit = False

    for app_id in tqdm(app_ids):
        try:
            db.begin()
            if should_quit:
                break

            time.sleep(CRAWL_TIMEOUT)
            results = scrape_store_page(app_id)

            crawl_time = dt.datetime.now()
            results['crawl_time'] = crawl_time

            # Default to empty list if the results don't contain any of these
            tags, details, genres = [], [], []

            # Pull the lists off before we insert the main crawl record.
            if 'tags' in results:
                tags = results['tags']
                del results['tags']
            if 'game_details' in results:
                details = results['game_details']
                del results['game_details']
            if 'genres' in results:
                genres = results['genres']
                del results['genres']

            db['game_crawl'].insert(results)

            if len(tags) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=tags,
                    entity_table='steam_tag',
                    pk_name='tag_id',
                    join_table='game_crawl_tag',
                    mapping=tag_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )

            if len(details) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=details,
                    entity_table='steam_game_detail',
                    pk_name='detail_id',
                    join_table='game_crawl_detail',
                    mapping=detail_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )

            if len(genres) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=genres,
                    entity_table='steam_genre',
                    pk_name='genre_id',
                    join_table='game_crawl_genre',
                    mapping=genre_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )
            db.commit()
        except Exception as e:
            # Problem app; pass along our failure and continue to the next one
            print('Failed to load app ID {}; continuing'.format(app_id))
            traceback.print_exc()

            # Ensure Postgres lets us continue by rolling back the current transaction
            db.rollback()

def run():
    db = dataset.connect(os.environ['POSTGRES_URI'], ensure_schema=False)

    # Since we're not super worried about having an up-to-date list of apps,
    # run this only if the table is empty
    if db['game'].count() == 0:
        upsert_all_apps(db)

    # For now, just crawl the apps we don't already have
    missing_crawl_query = '''
    SELECT g.steam_app_id
    FROM game g
      LEFT JOIN game_crawl gc
        USING (steam_app_id)
    WHERE gc.steam_app_id IS NULL;
    '''

    missing_app_ids = [r['steam_app_id'] for r in db.query(missing_crawl_query)]

    do_crawl(missing_app_ids, db)

if __name__ == '__main__':
    run()
<commit_msg>Print failure notices to stderr<commit_after>import requests
import dataset
import os
import sys
import re
import datetime as dt
import time
import traceback
from tqdm import tqdm
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, ElementNotVisibleException
from dateutil.parser import parse as dtparse

# Number of seconds to sleep between crawls
CRAWL_TIMEOUT = 10

THIRTY_DAY_REVIEW_REGEX = re.compile(r'^([0-9]+)% of the ([,0-9]+) user reviews in the last 30 days')
ALL_TIME_REVIEW_REGEX = re.compile(r'^([0-9]+)% of the ([,0-9]+) user reviews for this game')
DETAILS_BOX_REGEX = re.compile(r'^Title: ([^\n]+)(?:\nGenre: ([^\n]+))?(?:\nDeveloper: ([^\n]+))?(?:\nPublisher: ([^\n]+))?')
NUM_ACHIEVEMENTS_REGEX = re.compile(r'Includes ([,[0-9]+) Steam Achievements')

def upsert_all_apps(db):
    '''
    Get the full list of steam apps and upsert them in our database
    on the basis of steam's app ID.
    '''
    json = requests.get('http://api.steampowered.com/ISteamApps/GetAppList/v0001/').json()

    apps = json['applist']['apps']['app']

    db.begin()

    for app in tqdm(apps):
        db['game'].upsert({
            'steam_app_id': app['appid'],
            'game_name': app['name'],
        }, keys=['app_id'])

    db.commit()

def scrape_store_page(app_id):
    '''
    Extract all the information we can from the store page for a given app ID.
    '''
    results = {'steam_app_id': app_id}
    driver = webdriver.Chrome()
    store_base_url = "http://store.steampowered.com"
    app_url = "{}/app/{}".format(store_base_url, app_id)
    driver.get(app_url)

    try:
        try:
            # If this succeeds, we need to pass through the age gate.
            driver.find_element_by_id('agegate_box')

            select_element = driver.find_element_by_id('ageYear')
            # open year dialog
            select_element.click()
            # select correct year
            select_element.find_element_by_css_selector('option[value="1993"]').click()
            # close year dialog
            select_element.click()
            # submit the form
            driver.find_element_by_id('agecheck_form').submit()
        except NoSuchElementException:
            # No age gate; we're good to continue
            pass

        if driver.current_url in (store_base_url, '{}/'.format(store_base_url)):
            # We were redirected; the app doesn't have a store page.
            driver.close()
            return results
        elif 'store.steampowered.com/video' in driver.current_url:
            # This is a trailer for something else; we'll get the actual app later.
            driver.close()
            return results

        try:
            # If this succeeds, we've got a steam store error
            error_element = driver.find_element_by_id('error_box')
            error_text = error_element.find_element_by_class_name('error')
            if error_text.text == 'This item is currently unavailable in your region':
                # We can't see this app; ignore it
                driver.close()
                return results
        except NoSuchElementException:
            pass

        try:
            # If this succeeds, Chrome is showing us an error
            error_element = driver.find_element_by_class('error-code')
            if error_element.text == 'ERR_TOO_MANY_REDIRECTS':
                # Something wonky with the server response for this store page;
                # it's redirecting infinitely to itself.  Ignore it
                driver.close()
                return results
        except NoSuchElementException:
            pass

        # Get the description first, since it tells us whether the app is streaming video
        # (which means we don't care about it)
        descriptions = driver.find_elements_by_class_name('game_area_description')
        for description in descriptions:
            if description.text.startswith('ABOUT THIS GAME'):
                results['is_dlc'] = False
                results['long_description'] = description.text
            elif description.text.startswith('ABOUT THIS CONTENT'):
                results['is_dlc'] = True
                results['long_description'] = description.text
            elif description.text.startswith('ABOUT THIS SERIES'):
                # This is streaming video; we don't care about it, so just return
                driver.close()
                return results
            elif description.text.startswith('ABOUT THIS SOFTWARE'):
                # This is computer software; ignore it
                driver.close()
                return results
        if 'long_description' not in results and len(descriptions) > 0:
            raise RuntimeError('Unable to parse description for app_id {}'.format(app_id))

        results['game_name'] = (driver
                                .find_element_by_class_name('apphub_AppName')
                                .text)

        try:
            results['short_description'] = (driver
                                            .find_element_by_class_name('game_description_snippet')
                                            .text)
        except NoSuchElementException:
            # DLC doesn't have a short description
            pass

        reviews_texts = [element.get_attribute('data-store-tooltip')
                        for element in (driver
                                        .find_elements_by_class_name('user_reviews_summary_row')
                                        )]

        for text in reviews_texts:
            thirty_day_match = THIRTY_DAY_REVIEW_REGEX.match(text)

            if thirty_day_match:
                results['pct_positive_reviews_last_30_days'] = int(thirty_day_match.group(1))
                results['reviews_last_30_days'] = int(thirty_day_match.group(2).replace(',', ''))
            else:
                all_time_match = ALL_TIME_REVIEW_REGEX.match(text)

                if all_time_match:
                    results['pct_positive_reviews_all_time'] = int(all_time_match.group(1))
                    results['reviews_all_time'] = int(all_time_match.group(2).replace(',', ''))

        try:
            results['release_date'] = dtparse(driver.find_element_by_css_selector('.release_date .date').text)
        except NoSuchElementException:
            # This app doesn't have a release date for some reason
            pass

        # There's additional detail about VR stuff here, but we're not worried about that for now
        details_text = driver.find_elements_by_css_selector('.details_block:not(.vrsupport)')[0].text
        details_match = DETAILS_BOX_REGEX.match(details_text)
        results['title'] = details_match.group(1)
        raw_genre = details_match.group(2)
        if raw_genre is not None:
            results['genres'] = raw_genre.split(', ')
        results['developer'] = details_match.group(3)
        results['publisher'] = details_match.group(4)

        block_titles_texts = [element.text
                            for element in (driver
                                            .find_elements_by_class_name('block_title')
                                            )]

        for text in block_titles_texts:
            num_achievements_match = NUM_ACHIEVEMENTS_REGEX.match(text)

            if num_achievements_match:
                results['num_achievements'] = int(num_achievements_match.group(1))

        try:
            raw_metacritic_score = driver.find_element_by_class_name('score').text
            if raw_metacritic_score != 'NA':
                results['metacritic_score'] = int(raw_metacritic_score)
        except NoSuchElementException:
            # Some games don't have metascores
            pass

        try:
            # Not on sale or Free to Play
            raw_price = driver.find_element_by_class_name('game_purchase_price').text
            if raw_price in ('Free to Play', 'Free'):
                price = 0
            elif raw_price == 'Third-party':
                # For all examples thus far, this has meant "free", but I don't think
                # we can assume that if the source is a 3rd party
                price = None
            else:
                price = float(raw_price.replace('$', ''))
            results['full_price'] = price
        except NoSuchElementException:
            try:
                # On sale
                raw_price = driver.find_element_by_class_name('discount_original_price').text
                results['full_price'] = float(raw_price.replace('$', ''))
            except NoSuchElementException:
                # No price on the page
                pass

        results['game_details'] = []
        game_details_elements = driver.find_elements_by_class_name('game_area_details_specs')
        for element in game_details_elements:
            results['game_details'].append(element.find_element_by_css_selector('a.name').text)

        results['tags'] = []
        try:
            # Try to get the big list of tags if it's there
            driver.find_element_by_css_selector('.app_tag.add_button').click()
            tag_elements = driver.find_elements_by_css_selector('#app_tagging_modal a.app_tag')
        except (NoSuchElementException, ElementNotVisibleException):
            # Settle for the short list if not
            tag_elements = driver.find_elements_by_css_selector('a.app_tag')
        for element in tag_elements:
            results['tags'].append(element.text)

        driver.close()

        return results
    except:
        # Make sure our driver gets closed if we threw an error
        # so we don't leave a million windows open
        driver.close()
        raise

def insert_with_mapping(*, db, descrs, entity_table, pk_name, join_table, mapping, app_id, crawl_time):
    '''
    Given a db connection, some list of description data, a table of entities, the name of the PK,
    a many-to-many join table for the entities to game crawls,
    a mapping from descrs to entity table IDs, and an app_id/crawl_time,
    update the entity table/mapping if necessary and insert
    the entity in the many-to-many join table.

    Mutates the mapping.
    '''
    for descr in descrs:
        try:
            entity_id = mapping[descr]
        except KeyError:
            entity_id = db[entity_table].insert({'descr': descr})
            mapping[descr] = entity_id

        db[join_table].insert({
            'steam_app_id': app_id,
            'crawl_time': crawl_time,
            pk_name: entity_id,
        })

def do_crawl(app_ids, db):
    '''
    Given a list of steam app IDs and a db connection, do a crawl for the app IDs
    and append the results to our list of crawls in the database.
    '''
    tag_mapping = {r['descr']: r['tag_id'] for r in db['steam_tag'].find()}
    detail_mapping = {r['descr']: r['detail_id'] for r in db['steam_game_detail'].find()}
    genre_mapping = {r['descr']: r['genre_id'] for r in db['steam_genre'].find()}

    # Add a handler here to allow us to gracefully save our work and quit
    # if the user halts a crawl early via Ctrl + C.
    # NOTE: since background jobs ignore SIGINT, this has no effect if
    # the script is launched in the background; see
    # http://stackoverflow.com/questions/1112343/how-do-i-capture-sigint-in-python#comment68802096_1112357
    should_quit = False

    for app_id in tqdm(app_ids):
        try:
            db.begin()
            if should_quit:
                break

            time.sleep(CRAWL_TIMEOUT)
            results = scrape_store_page(app_id)

            crawl_time = dt.datetime.now()
            results['crawl_time'] = crawl_time

            # Default to empty list if the results don't contain any of these
            tags, details, genres = [], [], []

            # Pull the lists off before we insert the main crawl record.
            if 'tags' in results:
                tags = results['tags']
                del results['tags']
            if 'game_details' in results:
                details = results['game_details']
                del results['game_details']
            if 'genres' in results:
                genres = results['genres']
                del results['genres']

            db['game_crawl'].insert(results)

            if len(tags) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=tags,
                    entity_table='steam_tag',
                    pk_name='tag_id',
                    join_table='game_crawl_tag',
                    mapping=tag_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )

            if len(details) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=details,
                    entity_table='steam_game_detail',
                    pk_name='detail_id',
                    join_table='game_crawl_detail',
                    mapping=detail_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )

            if len(genres) > 0:
                insert_with_mapping(
                    db=db,
                    descrs=genres,
                    entity_table='steam_genre',
                    pk_name='genre_id',
                    join_table='game_crawl_genre',
                    mapping=genre_mapping,
                    app_id=app_id,
                    crawl_time=crawl_time
                )
            db.commit()
        except Exception as e:
            # Problem app; pass along our failure and continue to the next one
            print('Failed to load app ID {}; continuing'.format(app_id), file=sys.stderr)
            traceback.print_exc()

            # Ensure Postgres lets us continue by rolling back the current transaction
            db.rollback()

def run():
    db = dataset.connect(os.environ['POSTGRES_URI'], ensure_schema=False)

    # Since we're not super worried about having an up-to-date list of apps,
    # run this only if the table is empty
    if db['game'].count() == 0:
        upsert_all_apps(db)

    # For now, just crawl the apps we don't already have
    missing_crawl_query = '''
    SELECT g.steam_app_id
    FROM game g
      LEFT JOIN game_crawl gc
        USING (steam_app_id)
    WHERE gc.steam_app_id IS NULL;
    '''

    missing_app_ids = [r['steam_app_id'] for r in db.query(missing_crawl_query)]

    do_crawl(missing_app_ids, db)

if __name__ == '__main__':
    run()
<|endoftext|>