<commit_before>from protorpc import messages
from google.appengine.ext import ndb
from iomodels.crmengine.opportunitystage import Opportunitystage ,OpportunitystageSchema
from iomodels.crmengine.opportunities import Opportunity
from iomodels.crmengine.leads import Lead
from iomodels.crmengine.leadstatuses import Leadstatus
from model import User
from iograph import Edge
# 13.10.2014 LAR This file is about reporting on Iogrow

srcs=[None,'ioGrow Live','Social Media','Web Site','Phone Inquiry','Partner Referral','Purchased List','Other']
class stageOppSchema(messages.Message):
    entity_key=messages.StringField(1)
    name=messages.StringField(2)
    nbr=messages.IntegerField(3)
    amount=messages.IntegerField(4)
    probability=messages.IntegerField(5)
class LeadStatusSchema(messages.Message):
    name=messages.StringField(1)
    nbr_leads=messages.IntegerField(2)
    entity_key=messages.StringField(3)

class ReportSchema(messages.Message):
    owner=messages.StringField(1)
    organization=messages.StringField(2)
    organization_opportunity_amount=messages.IntegerField(3)
    organization_opportunity_nbr=messages.IntegerField(4)
    nbr_lead=messages.IntegerField(5)
    nbr_contact=messages.IntegerField(6)
    nbr_account=messages.IntegerField(7)
    org_oppo_stage=messages.MessageField(OpportunitystageSchema,8,repeated=True)
    leads_status_org=messages.MessageField(LeadStatusSchema,9,repeated=True)
    leads_source_org=messages.MessageField(LeadStatusSchema,10,repeated=True)
    leads_owner_org=messages.MessageField(LeadStatusSchema,11,repeated=True)
    org_oppo_owner=messages.MessageField(OpportunitystageSchema,12,repeated=True)

class stage_opportunity(ndb.Expando):
    entity_key=ndb.KeyProperty()
    name=ndb.StringProperty()
    nbr=ndb.IntegerProperty()
    amount=ndb.IntegerProperty()
    probability=ndb.IntegerProperty()

class Reports(ndb.Expando):
    owner = ndb.StringProperty()
    created_at = ndb.DateTimeProperty(auto_now_add=True)
    organization = ndb.KeyProperty()
    total_amount =ndb.IntegerProperty()
    nbr_lead=ndb.IntegerProperty()
    nbr_contact=ndb.IntegerProperty()
    nbr_account=ndb.IntegerProperty()
    nbr_opportunity=ndb.IntegerProperty()

    # stages_amount=ndb.StructuredProperty()
    @classmethod
    def get(cls, user_from_email):
        items=[]
        organization_opportunity_amount=0
        organization_opportunity_nbr=0
    	stages=Opportunitystage.query(Opportunitystage.organization==user_from_email.organization).fetch()
    	if stages :
            for stage in stages :
                organization_opportunity_amount=organization_opportunity_amount+stage.amount_opportunity
                organization_opportunity_nbr=organization_opportunity_nbr+stage.nbr_opportunity
                items.append(OpportunitystageSchema(
                    name=stage.name,
                    probability=stage.probability,
                    nbr_opportunity=stage.nbr_opportunity,
                    amount_opportunity=stage.amount_opportunity
                    )
                )
        return ReportSchema(
                            organization=user_from_email.organization.urlsafe(),
                            organization_opportunity_amount=organization_opportunity_amount,
                            organization_opportunity_nbr=organization_opportunity_nbr,
                            org_oppo_stage=items,

            )

    @classmethod
    def get_schema(cls,user_from_email):
        report=cls.get(user_from_email)
        item=[]
        stages=iograph.Edge.list(start_node= report.key,kind= "report_stage")
        stages= stages["items"]
        for stage in stages:
            stage=stage.end_node.get()
            item.append(stageOppSchema(
                entity_key=stage.entity_key.urlsafe(),
                name=stage.name,
                nbr=stage.nbr,
                amount=stage.amount,
                probability=stage.probability

                )
            )

        report_schema = ReportSchema(
                                  owner = report.owner,
                                  total_amount = report.total_amount,
                                  nbr_lead = report.nbr_lead,
                                  nbr_account=report.nbr_account,
                                  nbr_contact=report.nbr_contact ,
                                  opp_stage=item
                                )
        return  report_schema

    @classmethod
    def create(cls,user_from_email):
        print "***********************************************"
        print user_from_email
        print "+++++++++++++++++++++++++++++++++++++++++++++++"
        print user_from_email.google_user_id
        exist=cls.get(user_from_email)
        if not exist:

            report = cls(
                        owner = user_from_email.google_user_id,
                        organization = user_from_email.organization,
                        total_amount = 0,
                        nbr_lead=0,
                        nbr_contact=0,
                        nbr_account=0,
                        nbr_opportunity=0
                        )
            report_key=report.put()
            cls.init_stage(user_from_email,report_key)

    @classmethod
    def add_lead(cls,user_from_email,nbr=1):
        try:
            report=cls.get(user_from_email)
            report.nbr_lead= report.nbr_lead+nbr
            report.put()
        except :
            print"###########################################################"
            print"((((((((((((((  error in lead reports line 105   ))))))))))"
            print"###########################################################"
    @classmethod
    def add_account(cls,user_from_email,nbr=1):
        try:
            report=cls.get(user_from_email)
            report.nbr_account= report.nbr_account+nbr
            report.put()
        except :
            print"###########################################################"
            print"((((((((((((((  error in account reports line 115  ))))))))))"
            print"###########################################################"
    @classmethod
    def add_contact(cls,user_from_email,nbr=1):
        try:
            report=cls.get(user_from_email)
            report.nbr_contact= report.nbr_contact+nbr
            report.put()
        except :
            print"###########################################################"
            print"((((((((((((((  error in contact reports line 125   ))))))))))"
            print"###########################################################"
    @classmethod
    def add_opportunity(cls,stage_key,amount=0):
        stage=stage_key.get()
        stage.nbr_opportunity=stage.nbr_opportunity+1
        stage.amount_opportunity=stage.amount_opportunity+amount
        stage.put()
    @classmethod
    def remove_opportunity(cls,opp):
        from iograph import Edge
        query=Edge.list(start_node=opp.key, kind="stages",limit=1)
        if query["items"] :
            stage=query["items"][0].end_node.get()
            stage.nbr_opportunity=stage.nbr_opportunity-1
            stage.amount_opportunity=stage.amount_opportunity-opp.amount_total
            stage.put()
    def update_opportunity(cls):
        pass

    @classmethod
    def init_stage(cls,user_from_email,report):
        import iograph
        stages=Opportunitystage.query(Opportunitystage.organization==user_from_email.organization).fetch()
        array=[]
        for stage in stages:
            node=stage_opportunity(name=stage.name,nbr=0,probability=stage.probability,amount=0,entity_key=stage.key)
            node_key=node.put()
            iograph.Edge.insert(start_node=report,end_node=node_key,kind="report_stage",inverse_edge="stage_report")

    @classmethod
    def init_reports(cls):
        from model import User
        users=User.query()
        for user in users.iter(keys_only=True):

            cls.create(user.get())

    @classmethod
    def init_reports(cls):
        from model import User
        users=User.query()
        for user in users.iter(keys_only=True):
            cls.create(user.get())
    @classmethod
    def big_query(cls,org):
        users=User.query(User.organization==org)
        for user in users.iter(keys_only=True):
            print(user.get())
    @classmethod

    def lead_by_owner(cls,org):
        users=User.query(User.organization==org)
        for user in users.iter(keys_only=True):
            print(user.get())

    def opp_by_owner(cls,org):
        from model import Organization
        org=ndb.kind(urlsafe=org)
        users=User.query(User.organization==org)
        for user in users.iter(keys_only=True):
            print(user.get())
    @classmethod
    def add_stage(cls,user_from_email,stage=None):
        import iograph
        report=cls.get(user_from_email)
        node=stage_opportunity(name=stage.name,nbr=0,probability=stage.probability,amount=0,entity_key=stage.key)
        node_key=node.put()
        iograph.Edge.insert(start_node=report.key,end_node=node_key,kind="report_stage",inverse_edge="stage_report")

    @classmethod
    def remove_stage(cls,user_from_email,stage_key=None):
        import iograph
        report=cls.get(user_from_email)
        result=iograph.Edge.list(start_node=report.key,kind="report_stage")
        for edge in result["items"]:
            stage=edge.end_node.get()
            print stage_key
            print "----------------------------------------------------------------"
            print stage.name
            if  stage.entity_key== stage_key :
                print "#################### delete ###############################"
                print stage.name
                stage.key.delete()
                iograph.Edge.delete_all(stage.key)
    @classmethod
    def reportQuery(cls,user_from_email):
        org=user_from_email.organization
        query_lead=Lead.query(Lead.organization==org)
        leads=query_lead.fetch()
        total_lead=len(leads)
        statuses=Leadstatus.query(Leadstatus.organization==org).fetch()
        lead_by_status=[]
        for status in statuses :
            lead_by_status.append(
                    LeadStatusSchema(
                        entity_key=status.key.urlsafe(),
                        name=status.status,
                        nbr_leads=len(query_lead.filter(Lead.status==status.status).fetch())
                        )
                    )

        query_user=User.query(User.organization==org)
        users=query_user.fetch()
        total_user=len(users)
        lead_by_owner=[]
        for user in users:
            lead_by_owner.append(
                    LeadStatusSchema(
                        entity_key=user.key.urlsafe(),
                        name=user.google_display_name,
                        nbr_leads=len(query_lead.filter(Lead.owner==user.google_user_id).fetch())
                        )
                )

        lead_by_source=[]
        for src in srcs:
            lead_by_source.append(
                    LeadStatusSchema(
                        entity_key=u"",
                        name=src or "Other",
                        nbr_leads=len(query_lead.filter(Lead.source==src).fetch())
                        )
                )

        oppo_stage=[]
        total_amount=0
        total_nbr=0
        # stages=Opportunitystage.query(Opportunitystage.organization==org).fetch()
        # for stage in stages:
        #     amount_oppo_stage=0
        #     result= Edge.list(start_node=stage.key,kind="related_opportunities")["items"]
        #     nbr_oppo_stage=len(result)
        #     for oppo in result:
        #         oppo=oppo.end_node.get()
        #         if oppo :
        #             amount_oppo_stage=amount_oppo_stage+oppo.amount_total
        #     oppo_stage.append(
        #             OpportunitystageSchema(
        #                 name=stage.name,
        #                 probability=stage.probability,
        #                 nbr_opportunity=nbr_oppo_stage,
        #                 amount_opportunity=amount_oppo_stage
        #                 )
        #         )
        #     total_nbr=total_nbr+nbr_oppo_stage
        #     total_amount=total_amount+amount_oppo_stage
        stage_names=[]
        query_oppo=Opportunity.query(Opportunity.organization==org)
        stages=Opportunitystage.query(Opportunitystage.organization==org).fetch()
        for stage in stages:
            stage.nbr_opportunity=0
            stage.amount_opportunity=0
            stage.put()
        oppo_stages=query_oppo.fetch()
        for oppo in oppo_stages:
            result= Edge.list(start_node=oppo.key,kind="stages")["items"]
            total_amount=total_amount+oppo.amount_total
            total_nbr=total_nbr+1
            if result:
                stage=result[0].end_node.get()
                stage.nbr_opportunity=stage.nbr_opportunity+1
                if oppo.amount_total :
                    stage.amount_opportunity=stage.amount_opportunity+oppo.amount_total
                stage.put()
        for s in stages:
            oppo_stage.append(
                    OpportunitystageSchema(
                        name=s.name,
                        probability=s.probability,
                        nbr_opportunity=s.nbr_opportunity,
                        amount_opportunity=s.amount_opportunity
                        )
                )

        oppo_by_owner=[]
        for user in users:
            opportunities=query_oppo.filter(Opportunity.owner==user.google_user_id).fetch()
            amount=0
            nbr=len(opportunities)
            for opportunity in opportunities:
                amount=amount+opportunity.amount_total
            oppo_by_owner.append(
                    OpportunitystageSchema(
                        name=user.google_display_name,
                        nbr_opportunity=nbr,
                        amount_opportunity=amount
                        )
                )

        return ReportSchema(
                organization=org.urlsafe(),
                organization_opportunity_amount=total_amount,
                organization_opportunity_nbr=total_nbr,
                nbr_lead=total_lead,
                nbr_contact=0,
                nbr_account=0,
                org_oppo_stage=oppo_stage,
                org_oppo_owner=oppo_by_owner,
                leads_status_org=lead_by_status,
                leads_source_org=lead_by_source,
                leads_owner_org=lead_by_owner,
            )

<commit_msg>correction the bug in reporting api on our organization<commit_after>from protorpc import messages
from google.appengine.ext import ndb
from iomodels.crmengine.opportunitystage import Opportunitystage ,OpportunitystageSchema
from iomodels.crmengine.opportunities import Opportunity
from iomodels.crmengine.leads import Lead
from iomodels.crmengine.leadstatuses import Leadstatus
from model import User
from iograph import Edge
# 13.10.2014 LAR This file is about reporting on Iogrow

srcs=[None,'ioGrow Live','Social Media','Web Site','Phone Inquiry','Partner Referral','Purchased List','Other']
class stageOppSchema(messages.Message):
    entity_key=messages.StringField(1)
    name=messages.StringField(2)
    nbr=messages.IntegerField(3)
    amount=messages.IntegerField(4)
    probability=messages.IntegerField(5)
class LeadStatusSchema(messages.Message):
    name=messages.StringField(1)
    nbr_leads=messages.IntegerField(2)
    entity_key=messages.StringField(3)

class ReportSchema(messages.Message):
    owner=messages.StringField(1)
    organization=messages.StringField(2)
    organization_opportunity_amount=messages.IntegerField(3)
    organization_opportunity_nbr=messages.IntegerField(4)
    nbr_lead=messages.IntegerField(5)
    nbr_contact=messages.IntegerField(6)
    nbr_account=messages.IntegerField(7)
    org_oppo_stage=messages.MessageField(OpportunitystageSchema,8,repeated=True)
    leads_status_org=messages.MessageField(LeadStatusSchema,9,repeated=True)
    leads_source_org=messages.MessageField(LeadStatusSchema,10,repeated=True)
    leads_owner_org=messages.MessageField(LeadStatusSchema,11,repeated=True)
    org_oppo_owner=messages.MessageField(OpportunitystageSchema,12,repeated=True)

class stage_opportunity(ndb.Expando):
    entity_key=ndb.KeyProperty()
    name=ndb.StringProperty()
    nbr=ndb.IntegerProperty()
    amount=ndb.IntegerProperty()
    probability=ndb.IntegerProperty()

class Reports(ndb.Expando):
    owner = ndb.StringProperty()
    created_at = ndb.DateTimeProperty(auto_now_add=True)
    organization = ndb.KeyProperty()
    total_amount =ndb.IntegerProperty()
    nbr_lead=ndb.IntegerProperty()
    nbr_contact=ndb.IntegerProperty()
    nbr_account=ndb.IntegerProperty()
    nbr_opportunity=ndb.IntegerProperty()

    # stages_amount=ndb.StructuredProperty()
    @classmethod
    def get(cls, user_from_email):
        items=[]
        organization_opportunity_amount=0
        organization_opportunity_nbr=0
    	stages=Opportunitystage.query(Opportunitystage.organization==user_from_email.organization).fetch()
    	if stages :
            for stage in stages :
                organization_opportunity_amount=organization_opportunity_amount+stage.amount_opportunity
                organization_opportunity_nbr=organization_opportunity_nbr+stage.nbr_opportunity
                items.append(OpportunitystageSchema(
                    name=stage.name,
                    probability=stage.probability,
                    nbr_opportunity=stage.nbr_opportunity,
                    amount_opportunity=stage.amount_opportunity
                    )
                )
        return ReportSchema(
                            organization=user_from_email.organization.urlsafe(),
                            organization_opportunity_amount=organization_opportunity_amount,
                            organization_opportunity_nbr=organization_opportunity_nbr,
                            org_oppo_stage=items,

            )

    @classmethod
    def get_schema(cls,user_from_email):
        report=cls.get(user_from_email)
        item=[]
        stages=iograph.Edge.list(start_node= report.key,kind= "report_stage")
        stages= stages["items"]
        for stage in stages:
            stage=stage.end_node.get()
            item.append(stageOppSchema(
                entity_key=stage.entity_key.urlsafe(),
                name=stage.name,
                nbr=stage.nbr,
                amount=stage.amount,
                probability=stage.probability

                )
            )

        report_schema = ReportSchema(
                                  owner = report.owner,
                                  total_amount = report.total_amount,
                                  nbr_lead = report.nbr_lead,
                                  nbr_account=report.nbr_account,
                                  nbr_contact=report.nbr_contact ,
                                  opp_stage=item
                                )
        return  report_schema

    @classmethod
    def create(cls,user_from_email):
        print "***********************************************"
        print user_from_email
        print "+++++++++++++++++++++++++++++++++++++++++++++++"
        print user_from_email.google_user_id
        exist=cls.get(user_from_email)
        if not exist:

            report = cls(
                        owner = user_from_email.google_user_id,
                        organization = user_from_email.organization,
                        total_amount = 0,
                        nbr_lead=0,
                        nbr_contact=0,
                        nbr_account=0,
                        nbr_opportunity=0
                        )
            report_key=report.put()
            cls.init_stage(user_from_email,report_key)

    @classmethod
    def add_lead(cls,user_from_email,nbr=1):
        try:
            report=cls.get(user_from_email)
            report.nbr_lead= report.nbr_lead+nbr
            report.put()
        except :
            print"###########################################################"
            print"((((((((((((((  error in lead reports line 105   ))))))))))"
            print"###########################################################"
    @classmethod
    def add_account(cls,user_from_email,nbr=1):
        try:
            report=cls.get(user_from_email)
            report.nbr_account= report.nbr_account+nbr
            report.put()
        except :
            print"###########################################################"
            print"((((((((((((((  error in account reports line 115  ))))))))))"
            print"###########################################################"
    @classmethod
    def add_contact(cls,user_from_email,nbr=1):
        try:
            report=cls.get(user_from_email)
            report.nbr_contact= report.nbr_contact+nbr
            report.put()
        except :
            print"###########################################################"
            print"((((((((((((((  error in contact reports line 125   ))))))))))"
            print"###########################################################"
    @classmethod
    def add_opportunity(cls,stage_key,amount=0):
        stage=stage_key.get()
        stage.nbr_opportunity=stage.nbr_opportunity+1
        stage.amount_opportunity=stage.amount_opportunity+amount
        stage.put()
    @classmethod
    def remove_opportunity(cls,opp):
        from iograph import Edge
        query=Edge.list(start_node=opp.key, kind="stages",limit=1)
        if query["items"] :
            stage=query["items"][0].end_node.get()
            stage.nbr_opportunity=stage.nbr_opportunity-1
            stage.amount_opportunity=stage.amount_opportunity-opp.amount_total
            stage.put()
    def update_opportunity(cls):
        pass

    @classmethod
    def init_stage(cls,user_from_email,report):
        import iograph
        stages=Opportunitystage.query(Opportunitystage.organization==user_from_email.organization).fetch()
        array=[]
        for stage in stages:
            node=stage_opportunity(name=stage.name,nbr=0,probability=stage.probability,amount=0,entity_key=stage.key)
            node_key=node.put()
            iograph.Edge.insert(start_node=report,end_node=node_key,kind="report_stage",inverse_edge="stage_report")

    @classmethod
    def init_reports(cls):
        from model import User
        users=User.query()
        for user in users.iter(keys_only=True):

            cls.create(user.get())

    @classmethod
    def init_reports(cls):
        from model import User
        users=User.query()
        for user in users.iter(keys_only=True):
            cls.create(user.get())
    @classmethod
    def big_query(cls,org):
        users=User.query(User.organization==org)
        for user in users.iter(keys_only=True):
            print(user.get())
    @classmethod

    def lead_by_owner(cls,org):
        users=User.query(User.organization==org)
        for user in users.iter(keys_only=True):
            print(user.get())

    def opp_by_owner(cls,org):
        from model import Organization
        org=ndb.kind(urlsafe=org)
        users=User.query(User.organization==org)
        for user in users.iter(keys_only=True):
            print(user.get())
    @classmethod
    def add_stage(cls,user_from_email,stage=None):
        import iograph
        report=cls.get(user_from_email)
        node=stage_opportunity(name=stage.name,nbr=0,probability=stage.probability,amount=0,entity_key=stage.key)
        node_key=node.put()
        iograph.Edge.insert(start_node=report.key,end_node=node_key,kind="report_stage",inverse_edge="stage_report")

    @classmethod
    def remove_stage(cls,user_from_email,stage_key=None):
        import iograph
        report=cls.get(user_from_email)
        result=iograph.Edge.list(start_node=report.key,kind="report_stage")
        for edge in result["items"]:
            stage=edge.end_node.get()
            print stage_key
            print "----------------------------------------------------------------"
            print stage.name
            if  stage.entity_key== stage_key :
                print "#################### delete ###############################"
                print stage.name
                stage.key.delete()
                iograph.Edge.delete_all(stage.key)
    @classmethod
    def reportQuery(cls,user_from_email):
        org=user_from_email.organization
        query_lead=Lead.query(Lead.organization==org)
        leads=query_lead.fetch()
        total_lead=len(leads)
        statuses=Leadstatus.query(Leadstatus.organization==org).fetch()
        lead_by_status=[]
        for status in statuses :
            lead_by_status.append(
                    LeadStatusSchema(
                        entity_key=status.key.urlsafe(),
                        name=status.status,
                        nbr_leads=len(query_lead.filter(Lead.status==status.status).fetch())
                        )
                    )

        query_user=User.query(User.organization==org)
        users=query_user.fetch()
        total_user=len(users)
        lead_by_owner=[]
        for user in users:
            lead_by_owner.append(
                    LeadStatusSchema(
                        entity_key=user.key.urlsafe(),
                        name=user.google_display_name,
                        nbr_leads=len(query_lead.filter(Lead.owner==user.google_user_id).fetch())
                        )
                )

        lead_by_source=[]
        for src in srcs:
            lead_by_source.append(
                    LeadStatusSchema(
                        entity_key=u"",
                        name=src or "Other",
                        nbr_leads=len(query_lead.filter(Lead.source==src).fetch())
                        )
                )

        oppo_stage=[]
        total_amount=0
        total_nbr=0
        # stages=Opportunitystage.query(Opportunitystage.organization==org).fetch()
        # for stage in stages:
        #     amount_oppo_stage=0
        #     result= Edge.list(start_node=stage.key,kind="related_opportunities")["items"]
        #     nbr_oppo_stage=len(result)
        #     for oppo in result:
        #         oppo=oppo.end_node.get()
        #         if oppo :
        #             amount_oppo_stage=amount_oppo_stage+oppo.amount_total
        #     oppo_stage.append(
        #             OpportunitystageSchema(
        #                 name=stage.name,
        #                 probability=stage.probability,
        #                 nbr_opportunity=nbr_oppo_stage,
        #                 amount_opportunity=amount_oppo_stage
        #                 )
        #         )
        #     total_nbr=total_nbr+nbr_oppo_stage
        #     total_amount=total_amount+amount_oppo_stage
        stage_names=[]
        query_oppo=Opportunity.query(Opportunity.organization==org)
        stages=Opportunitystage.query(Opportunitystage.organization==org).fetch()
        for stage in stages:
            stage.nbr_opportunity=0
            stage.amount_opportunity=0
            stage.put()
        oppo_stages=query_oppo.fetch()
        for oppo in oppo_stages:
            result= Edge.list(start_node=oppo.key,kind="stages")["items"]
            if oppo.amount_total :
                total_amount=total_amount+oppo.amount_total
            total_nbr=total_nbr+1
            if result:
                stage=result[0].end_node.get()
                stage.nbr_opportunity=stage.nbr_opportunity+1
                if oppo.amount_total :
                    stage.amount_opportunity=stage.amount_opportunity+oppo.amount_total
                stage.put()
        for s in stages:
            oppo_stage.append(
                    OpportunitystageSchema(
                        name=s.name,
                        probability=s.probability,
                        nbr_opportunity=s.nbr_opportunity,
                        amount_opportunity=s.amount_opportunity
                        )
                )

        oppo_by_owner=[]
        for user in users:
            opportunities=query_oppo.filter(Opportunity.owner==user.google_user_id).fetch()
            amount=0
            nbr=len(opportunities)
            for opportunity in opportunities:
                amount=amount+opportunity.amount_total
            oppo_by_owner.append(
                    OpportunitystageSchema(
                        name=user.google_display_name,
                        nbr_opportunity=nbr,
                        amount_opportunity=amount
                        )
                )

        return ReportSchema(
                organization=org.urlsafe(),
                organization_opportunity_amount=total_amount,
                organization_opportunity_nbr=total_nbr,
                nbr_lead=total_lead,
                nbr_contact=0,
                nbr_account=0,
                org_oppo_stage=oppo_stage,
                org_oppo_owner=oppo_by_owner,
                leads_status_org=lead_by_status,
                leads_source_org=lead_by_source,
                leads_owner_org=lead_by_owner,
            )

<|endoftext|>


<commit_before>from . import app
from datetime import date
from flask import url_for
from markdown import markdown
from smartypants import smartyPants
import bleach
import bs4
import codecs
import datetime
import jwt
import os
import os.path
import random
import re
import requests
import shutil
import unicodedata
import urllib
import hmac
import hashlib

bleach.ALLOWED_TAGS += ['img', 'p', 'br', 'marquee', 'blink']
bleach.ALLOWED_ATTRIBUTES.update({
    'img': ['src', 'alt', 'title']
})

TWITTER_PROFILE_RE = re.compile(r'https?://(?:www\.)?twitter\.com/(\w+)')
TWITTER_RE = re.compile(r'https?://(?:www\.|mobile\.)?twitter\.com/(\w+)/status(?:es)?/(\w+)')
FACEBOOK_PROFILE_RE = re.compile(r'https?://(?:www\.)?facebook\.com/([a-zA-Z0-9._-]+)')
FACEBOOK_RE = re.compile(r'https?://(?:www\.)?facebook\.com/([a-zA-Z0-9._-]+)/\w+/(\w+)')
YOUTUBE_RE = re.compile(r'https?://(?:www.)?youtube\.com/watch\?v=(\w+)')
INSTAGRAM_RE = re.compile(r'https?://instagram\.com/p/(\w+)')

PEOPLE_RE = re.compile(r"\[\[([\w ]+)(?:\|([\w\-'. ]+))?\]\]")
RELATIVE_PATH_RE = re.compile('\[([^\]]*)\]\(([^/)]+)\)')

AT_USERNAME_RE = re.compile(r"""(?<!\w)@([a-zA-Z0-9_]+)(?=($|[\s,:;.'"]))""")
LINK_RE = re.compile(
    # optional schema
    r'\b([a-z]{3,9}://)?'
    # hostname and port
    r'((?:[a-z0-9\-]+\.)+[a-z]{2,4}(?::\d{2,6})?'
    # path
    r'(?:(?:/(?:[a-zA-Z0-9\-_~.;:$?&%#@()/=]*[a-zA-Z0-9\-_$?#/])?)|\b))'
)

def isoparse(s):
    """Parse (UTC) datetimes in ISO8601 format"""
    if s:
        try:
            return datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S')
        except:
            return datetime.datetime.strptime(s, '%Y-%m-%d')

def isoparse_with_tz(s):
    """Parse datetimes with a timezone in ISO8601 format"""
    return s and datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S%z')

def isoformat(date):
    if date:
        if (isinstance(date, datetime.date)
            and not isinstance(date, datetime.datetime)):
            return date.isoformat()
        if date.tzinfo:
            date = date.astimezone(datetime.timezone.utc)
            date = date.replace(tzinfo=None)
        date = date.replace(microsecond=0)
        return date.isoformat('T')

def isoformat_with_tz(date):
    if hasattr(date, 'tzinfo') and not date.tzinfo:
        date = date.replace(tzinfo=datetime.timezone.utc)
    return date.isoformat(sep='T')

def normalize_tag(tag):
    # lowercase and remove spaces, dashes, and underscores
    tag = unicodedata.normalize('NFKD', tag).lower()
    tag = re.sub(r'[ _\-]', '', tag)
    return tag

def filter_empty_keys(data):
    if isinstance(data, list):
        return list(filter_empty_keys(v) for v in data if filter_empty_keys(v))
    if isinstance(data, dict):
        return dict((k, filter_empty_keys(v)) for k, v in data.items()
                    if filter_empty_keys(v))
    return data

def download_resource(url, path):
    from .models import get_settings
    app.logger.debug("downloading {} to {}".format(url, path))
    response = requests.get(urllib.parse.urljoin(get_settings().site_url, url),
                            stream=True, timeout=10)
    response.raise_for_status()
    if not os.path.exists(os.path.dirname(path)):
        os.makedirs(os.path.dirname(path))

    with open(path, 'wb') as f:
        for chunk in response.iter_content(512):
            f.write(chunk)

def urls_match(url1, url2):
    if url1 == url2:
        return True
    p1 = urllib.parse.urlparse(url1)
    p2 = urllib.parse.urlparse(url2)
    return p1.netloc == p2.netloc and p1.path == p2.path

def url_to_link(url, soup):
    a = soup.new_tag('a', href=url)
    a.string = prettify_url(url)
    return a

def person_to_microcard(contact, nick, soup):
    if contact:
        url = contact.url or url_for('contact_by_name', nick)
        a_tag = soup.new_tag('a', href=url)
        a_tag['class'] = ['microcard', 'h-card']

        image = contact.image
        if image:
            image = construct_imageproxy_url(image, 26)
            image_tag = soup.new_tag('img', src=image)
            a_tag.append(image_tag)
            a_tag.append(contact.name)
        else:
            a_tag.append('@' + contact.name)

    else:
        a_tag = soup.new_tag('a', href='https://twitter.com/' + nick)
        a_tag.string = '@' + nick

    return a_tag

def autolink(plain, url_processor=url_to_link,
             person_processor=person_to_microcard):
    """Replace bare URLs in a document with an HTML <a> representation
    """
    blacklist = ('a', 'script', 'pre', 'code', 'embed', 'object',
                      'audio', 'video')
    soup = bs4.BeautifulSoup(plain)

    def bs4_sub(regex, repl):
        """Process text elements in a BeautifulSoup document with a regex and
        replacement string.

        :param BeautifulSoup soup: the BeautifulSoup document to be
         edited in place
        :param string regex: a regular expression whose matches will
         be replaced
        :param function repl: a function that a Match object, and
         returns text or a new HTML node
        :param list blacklist: a list of tags whose children should
         not be modified (<pre> for example)
        """
        for txt in soup.find_all(text=True):
            if any(p.name in blacklist for p in txt.parents):
                continue
            nodes = []
            start = 0
            for m in regex.finditer(txt):
                nodes.append(txt[start:m.start()])
                nodes.append(repl(m))
                start = m.end()
            if not nodes:
                continue
            nodes.append(txt[start:])
            parent = txt.parent
            ii = parent.contents.index(txt)
            txt.extract()
            for offset, node in enumerate(nodes):
                parent.insert(ii + offset, node)

    def link_repl(m):
        url = (m.group(1) or 'http://') + m.group(2)
        return url_processor(url, soup)

    def process_nick(m):
        from . import db
        from .models import Nick
        name = m.group(1)
        nick = Nick.query.filter(
            db.func.lower(Nick.name) == db.func.lower(name)).first()
        contact = nick and nick.contact
        processed = person_processor(contact, name, soup)
        if processed:
            return processed
        return m.group(0)

    if url_processor:
        bs4_sub(LINK_RE, link_repl)

    if person_processor:
        bs4_sub(AT_USERNAME_RE, process_nick)

    return ''.join(str(t) for t in soup.body.contents) if soup.body else ''

TAG_TO_TYPE = {
    'n': 'note',
    'a': 'article',
    'r': 'reply',
    's': 'share',
    'l': 'like',
    'c': 'checkin',
    'p': 'photo',
    'b': 'bookmark',
}

TYPE_TO_TAG = {v: k for k, v in TAG_TO_TYPE.items()}

BASE_ORDINAL = date(1970, 1, 1).toordinal()

def parse_type(tag):
    type_enc = tag[0]
    return TAG_TO_TYPE.get(type_enc)

def parse_date(tag):
    try:
        date_enc = tag[1:4]
        ordinal = base60_decode(date_enc)
        if ordinal:
            return date_from_ordinal(ordinal)
    except ValueError:
        app.logger.warn("Could not parse base60 date %s", tag)

def parse_index(tag):
    index_enc = tag[4:]
    return index_enc

def date_to_ordinal(date0):
    return date0.toordinal() - BASE_ORDINAL

def date_from_ordinal(ordinal):
    return date.fromordinal(ordinal + BASE_ORDINAL)

def tag_for_post_type(post_type):
    return TYPE_TO_TAG.get(post_type)

# use tantek's NewBase60 http://tantek.pbworks.com/w/page/19402946/NewBase60
RADIX = list("0123456789ABCDEFGHJKLMNPQRSTUVWXYZ_abcdefghijkmnopqrstuvwxyz")

def base60_encode(n):
    arr = []
    base = len(RADIX)
    while n > 0:
        c = RADIX[n % base]
        n = n // base
        arr.append(c)

    arr.reverse()
    return ''.join(arr)

def base60_decode(s):
    base = len(RADIX)
    n = 0
    for c in s:
        n *= base
        n += RADIX.index(c)
    return n

def slugify(s, limit=256):
    slug = unicodedata.normalize('NFKD', s).lower()
    slug = re.sub(r'[^a-z0-9]+', '-', slug).strip('-')
    slug = re.sub(r'[-]+', '-', slug)
    # trim to first - after the limit
    if len(slug) > limit:
        idx = slug.find('-', limit)
        if idx >= 0:
            slug = slug[:idx]
    return slug

def multiline_string_to_list(s):
    return [l.strip() for l in s.split('\n') if l.strip()]

def image_root_path():
    return app.config.get('IMAGE_ROOT_PATH', app.root_path)

def proxy_all_images(html):
    def repl(m):
        return m.group(1) + construct_imageproxy_url(m.group(2)) + m.group(3)

    regex = re.compile(r'(<img[^>]+src=")([^">]+)(")')
    return regex.sub(repl, html)

def construct_imageproxy_url(src, side=None):
    pilbox_url = app.config.get('PILBOX_URL')
    if not pilbox_url:
        # cannot resize without pilbox
        app.logger.warn('No pilbox server configured')
        return src

    query = {}
    query['url'] = src
    if side:
        query['w'] = side
        query['h'] = side
    else:
        query['op'] = 'noop'

    pilbox_key = app.config.get('PILBOX_KEY')
    if pilbox_key:
        qs = urllib.parse.urlencode(query)
        h = hmac.new(pilbox_key.encode(), qs.encode(), hashlib.sha1)
        qs += '&sig=' + h.hexdigest()

    return pilbox_url + '?' + qs

def markdown_filter(data, img_path=None, url_processor=url_to_link,
                    person_processor=person_to_microcard):
    if data is None:
        return ''

    if img_path:
        # replace relative paths to images with absolute
        data = RELATIVE_PATH_RE.sub('[\g<1>](' + img_path + '/\g<2>)', data)

    data = convert_legacy_people_to_at_names(data)
    result = markdown(data, extensions=['codehilite', 'fenced_code'])
    if url_processor or person_processor:
        result = autolink(result, url_processor, person_processor)
    result = smartyPants(result)
    return result

def convert_legacy_people_to_at_names(data):
    from .models import Contact

    def process_name(m):
        fullname = m.group(1)
        displayname = m.group(2)
        contact = Contact.query.filter_by(name=fullname).first()
        if contact and contact.nicks:
            return '@' + contact.nicks[0].name
        return '@' + displayname

    data = PEOPLE_RE.sub(process_name, data)
    return data

def format_as_text(html, link_fn=None):
    if html is None:
        return ''
    soup = bs4.BeautifulSoup(html)

    # replace links with the URL
    for a in soup.find_all('a'):
        if link_fn:
            link_fn(a)
        else:
            a.replace_with(a.get('href') or '[link]')

    # and remove images
    for i in soup.find_all('img'):
        i.hidden = True

    return soup.get_text().strip()

def is_cached_current(original, cached):
    """Compare a file and the processed, cached version to see if the cached
    version is up to date.
    """
    return (os.path.exists(cached)
            and os.stat(cached).st_mtime >= os.stat(original).st_mtime)

def prettify_url(url):
    """Return a URL without its schema
    """
    if not url:
        return url
    split = url.split('//', 1)
    if len(split) == 2:
        schema, path = split
    else:
        path = url
    return path

def fetch_html(url):
    """Utility to fetch HTML from an external site. If the Content-Type
    header does not explicitly list a charset, Requests will assume a
    bad one, so we ahve to use 'get_encodings_from_content` to find
    the meta charset or other indications in the actual response body.

    Return a requests.Response
    """
    response = requests.get(url, timeout=30)
    if response.status_code // 2 == 100:
        # requests ignores <meta charset> when a Content-Type header
        # is provided, even if the header does not define a charset
        if 'charset' not in response.headers.get('content-type', ''):
            encodings = requests.utils.get_encodings_from_content(
                response.text)
            if encodings:
                response.encoding = encodings[0]
    else:
        app.logger.warn('failed to fetch url %s. got response %s.',
                        url, response)
    return response

def clean_foreign_html(html):
    return bleach.clean(html, strip=True)

def jwt_encode(obj):
    obj['nonce'] = random.randint(1000000, 2 ** 31)
    return jwt.encode(obj, app.config['SECRET_KEY'])

def jwt_decode(s):
    return jwt.decode(s, app.config['SECRET_KEY'])
<commit_msg>Patch bc pilbox chokes on "data:" urls<commit_after>from . import app
from datetime import date
from flask import url_for
from markdown import markdown
from smartypants import smartyPants
import bleach
import bs4
import codecs
import datetime
import jwt
import os
import os.path
import random
import re
import requests
import shutil
import unicodedata
import urllib
import hmac
import hashlib

bleach.ALLOWED_TAGS += ['img', 'p', 'br', 'marquee', 'blink']
bleach.ALLOWED_ATTRIBUTES.update({
    'img': ['src', 'alt', 'title']
})

TWITTER_PROFILE_RE = re.compile(r'https?://(?:www\.)?twitter\.com/(\w+)')
TWITTER_RE = re.compile(r'https?://(?:www\.|mobile\.)?twitter\.com/(\w+)/status(?:es)?/(\w+)')
FACEBOOK_PROFILE_RE = re.compile(r'https?://(?:www\.)?facebook\.com/([a-zA-Z0-9._-]+)')
FACEBOOK_RE = re.compile(r'https?://(?:www\.)?facebook\.com/([a-zA-Z0-9._-]+)/\w+/(\w+)')
YOUTUBE_RE = re.compile(r'https?://(?:www.)?youtube\.com/watch\?v=(\w+)')
INSTAGRAM_RE = re.compile(r'https?://instagram\.com/p/(\w+)')

PEOPLE_RE = re.compile(r"\[\[([\w ]+)(?:\|([\w\-'. ]+))?\]\]")
RELATIVE_PATH_RE = re.compile('\[([^\]]*)\]\(([^/)]+)\)')

AT_USERNAME_RE = re.compile(r"""(?<!\w)@([a-zA-Z0-9_]+)(?=($|[\s,:;.'"]))""")
LINK_RE = re.compile(
    # optional schema
    r'\b([a-z]{3,9}://)?'
    # hostname and port
    r'((?:[a-z0-9\-]+\.)+[a-z]{2,4}(?::\d{2,6})?'
    # path
    r'(?:(?:/(?:[a-zA-Z0-9\-_~.;:$?&%#@()/=]*[a-zA-Z0-9\-_$?#/])?)|\b))'
)

def isoparse(s):
    """Parse (UTC) datetimes in ISO8601 format"""
    if s:
        try:
            return datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S')
        except:
            return datetime.datetime.strptime(s, '%Y-%m-%d')

def isoparse_with_tz(s):
    """Parse datetimes with a timezone in ISO8601 format"""
    return s and datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S%z')

def isoformat(date):
    if date:
        if (isinstance(date, datetime.date)
            and not isinstance(date, datetime.datetime)):
            return date.isoformat()
        if date.tzinfo:
            date = date.astimezone(datetime.timezone.utc)
            date = date.replace(tzinfo=None)
        date = date.replace(microsecond=0)
        return date.isoformat('T')

def isoformat_with_tz(date):
    if hasattr(date, 'tzinfo') and not date.tzinfo:
        date = date.replace(tzinfo=datetime.timezone.utc)
    return date.isoformat(sep='T')

def normalize_tag(tag):
    # lowercase and remove spaces, dashes, and underscores
    tag = unicodedata.normalize('NFKD', tag).lower()
    tag = re.sub(r'[ _\-]', '', tag)
    return tag

def filter_empty_keys(data):
    if isinstance(data, list):
        return list(filter_empty_keys(v) for v in data if filter_empty_keys(v))
    if isinstance(data, dict):
        return dict((k, filter_empty_keys(v)) for k, v in data.items()
                    if filter_empty_keys(v))
    return data

def download_resource(url, path):
    from .models import get_settings
    app.logger.debug("downloading {} to {}".format(url, path))
    response = requests.get(urllib.parse.urljoin(get_settings().site_url, url),
                            stream=True, timeout=10)
    response.raise_for_status()
    if not os.path.exists(os.path.dirname(path)):
        os.makedirs(os.path.dirname(path))

    with open(path, 'wb') as f:
        for chunk in response.iter_content(512):
            f.write(chunk)

def urls_match(url1, url2):
    if url1 == url2:
        return True
    p1 = urllib.parse.urlparse(url1)
    p2 = urllib.parse.urlparse(url2)
    return p1.netloc == p2.netloc and p1.path == p2.path

def url_to_link(url, soup):
    a = soup.new_tag('a', href=url)
    a.string = prettify_url(url)
    return a

def person_to_microcard(contact, nick, soup):
    if contact:
        url = contact.url or url_for('contact_by_name', nick)
        a_tag = soup.new_tag('a', href=url)
        a_tag['class'] = ['microcard', 'h-card']

        image = contact.image
        if image:
            image = construct_imageproxy_url(image, 26)
            image_tag = soup.new_tag('img', src=image)
            a_tag.append(image_tag)
            a_tag.append(contact.name)
        else:
            a_tag.append('@' + contact.name)

    else:
        a_tag = soup.new_tag('a', href='https://twitter.com/' + nick)
        a_tag.string = '@' + nick

    return a_tag

def autolink(plain, url_processor=url_to_link,
             person_processor=person_to_microcard):
    """Replace bare URLs in a document with an HTML <a> representation
    """
    blacklist = ('a', 'script', 'pre', 'code', 'embed', 'object',
                      'audio', 'video')
    soup = bs4.BeautifulSoup(plain)

    def bs4_sub(regex, repl):
        """Process text elements in a BeautifulSoup document with a regex and
        replacement string.

        :param BeautifulSoup soup: the BeautifulSoup document to be
         edited in place
        :param string regex: a regular expression whose matches will
         be replaced
        :param function repl: a function that a Match object, and
         returns text or a new HTML node
        :param list blacklist: a list of tags whose children should
         not be modified (<pre> for example)
        """
        for txt in soup.find_all(text=True):
            if any(p.name in blacklist for p in txt.parents):
                continue
            nodes = []
            start = 0
            for m in regex.finditer(txt):
                nodes.append(txt[start:m.start()])
                nodes.append(repl(m))
                start = m.end()
            if not nodes:
                continue
            nodes.append(txt[start:])
            parent = txt.parent
            ii = parent.contents.index(txt)
            txt.extract()
            for offset, node in enumerate(nodes):
                parent.insert(ii + offset, node)

    def link_repl(m):
        url = (m.group(1) or 'http://') + m.group(2)
        return url_processor(url, soup)

    def process_nick(m):
        from . import db
        from .models import Nick
        name = m.group(1)
        nick = Nick.query.filter(
            db.func.lower(Nick.name) == db.func.lower(name)).first()
        contact = nick and nick.contact
        processed = person_processor(contact, name, soup)
        if processed:
            return processed
        return m.group(0)

    if url_processor:
        bs4_sub(LINK_RE, link_repl)

    if person_processor:
        bs4_sub(AT_USERNAME_RE, process_nick)

    return ''.join(str(t) for t in soup.body.contents) if soup.body else ''

TAG_TO_TYPE = {
    'n': 'note',
    'a': 'article',
    'r': 'reply',
    's': 'share',
    'l': 'like',
    'c': 'checkin',
    'p': 'photo',
    'b': 'bookmark',
}

TYPE_TO_TAG = {v: k for k, v in TAG_TO_TYPE.items()}

BASE_ORDINAL = date(1970, 1, 1).toordinal()

def parse_type(tag):
    type_enc = tag[0]
    return TAG_TO_TYPE.get(type_enc)

def parse_date(tag):
    try:
        date_enc = tag[1:4]
        ordinal = base60_decode(date_enc)
        if ordinal:
            return date_from_ordinal(ordinal)
    except ValueError:
        app.logger.warn("Could not parse base60 date %s", tag)

def parse_index(tag):
    index_enc = tag[4:]
    return index_enc

def date_to_ordinal(date0):
    return date0.toordinal() - BASE_ORDINAL

def date_from_ordinal(ordinal):
    return date.fromordinal(ordinal + BASE_ORDINAL)

def tag_for_post_type(post_type):
    return TYPE_TO_TAG.get(post_type)

# use tantek's NewBase60 http://tantek.pbworks.com/w/page/19402946/NewBase60
RADIX = list("0123456789ABCDEFGHJKLMNPQRSTUVWXYZ_abcdefghijkmnopqrstuvwxyz")

def base60_encode(n):
    arr = []
    base = len(RADIX)
    while n > 0:
        c = RADIX[n % base]
        n = n // base
        arr.append(c)

    arr.reverse()
    return ''.join(arr)

def base60_decode(s):
    base = len(RADIX)
    n = 0
    for c in s:
        n *= base
        n += RADIX.index(c)
    return n

def slugify(s, limit=256):
    slug = unicodedata.normalize('NFKD', s).lower()
    slug = re.sub(r'[^a-z0-9]+', '-', slug).strip('-')
    slug = re.sub(r'[-]+', '-', slug)
    # trim to first - after the limit
    if len(slug) > limit:
        idx = slug.find('-', limit)
        if idx >= 0:
            slug = slug[:idx]
    return slug

def multiline_string_to_list(s):
    return [l.strip() for l in s.split('\n') if l.strip()]

def image_root_path():
    return app.config.get('IMAGE_ROOT_PATH', app.root_path)

def proxy_all_images(html):
    def repl(m):
        return m.group(1) + construct_imageproxy_url(m.group(2)) + m.group(3)

    regex = re.compile(r'(<img[^>]+src=")([^">]+)(")')
    return regex.sub(repl, html)

def construct_imageproxy_url(src, side=None):
    pilbox_url = app.config.get('PILBOX_URL')
    if not pilbox_url or src.lower().startswith('data:'):
        # cannot resize without pilbox
        # pilbox cannot resize "data:" urls
        app.logger.warn('No pilbox server configured')
        return src

    query = {}
    query['url'] = src
    if side:
        query['w'] = side
        query['h'] = side
    else:
        query['op'] = 'noop'

    pilbox_key = app.config.get('PILBOX_KEY')
    if pilbox_key:
        qs = urllib.parse.urlencode(query)
        h = hmac.new(pilbox_key.encode(), qs.encode(), hashlib.sha1)
        qs += '&sig=' + h.hexdigest()

    return pilbox_url + '?' + qs

def markdown_filter(data, img_path=None, url_processor=url_to_link,
                    person_processor=person_to_microcard):
    if data is None:
        return ''

    if img_path:
        # replace relative paths to images with absolute
        data = RELATIVE_PATH_RE.sub('[\g<1>](' + img_path + '/\g<2>)', data)

    data = convert_legacy_people_to_at_names(data)
    result = markdown(data, extensions=['codehilite', 'fenced_code'])
    if url_processor or person_processor:
        result = autolink(result, url_processor, person_processor)
    result = smartyPants(result)
    return result

def convert_legacy_people_to_at_names(data):
    from .models import Contact

    def process_name(m):
        fullname = m.group(1)
        displayname = m.group(2)
        contact = Contact.query.filter_by(name=fullname).first()
        if contact and contact.nicks:
            return '@' + contact.nicks[0].name
        return '@' + displayname

    data = PEOPLE_RE.sub(process_name, data)
    return data

def format_as_text(html, link_fn=None):
    if html is None:
        return ''
    soup = bs4.BeautifulSoup(html)

    # replace links with the URL
    for a in soup.find_all('a'):
        if link_fn:
            link_fn(a)
        else:
            a.replace_with(a.get('href') or '[link]')

    # and remove images
    for i in soup.find_all('img'):
        i.hidden = True

    return soup.get_text().strip()

def is_cached_current(original, cached):
    """Compare a file and the processed, cached version to see if the cached
    version is up to date.
    """
    return (os.path.exists(cached)
            and os.stat(cached).st_mtime >= os.stat(original).st_mtime)

def prettify_url(url):
    """Return a URL without its schema
    """
    if not url:
        return url
    split = url.split('//', 1)
    if len(split) == 2:
        schema, path = split
    else:
        path = url
    return path

def fetch_html(url):
    """Utility to fetch HTML from an external site. If the Content-Type
    header does not explicitly list a charset, Requests will assume a
    bad one, so we ahve to use 'get_encodings_from_content` to find
    the meta charset or other indications in the actual response body.

    Return a requests.Response
    """
    response = requests.get(url, timeout=30)
    if response.status_code // 2 == 100:
        # requests ignores <meta charset> when a Content-Type header
        # is provided, even if the header does not define a charset
        if 'charset' not in response.headers.get('content-type', ''):
            encodings = requests.utils.get_encodings_from_content(
                response.text)
            if encodings:
                response.encoding = encodings[0]
    else:
        app.logger.warn('failed to fetch url %s. got response %s.',
                        url, response)
    return response

def clean_foreign_html(html):
    return bleach.clean(html, strip=True)

def jwt_encode(obj):
    obj['nonce'] = random.randint(1000000, 2 ** 31)
    return jwt.encode(obj, app.config['SECRET_KEY'])

def jwt_decode(s):
    return jwt.decode(s, app.config['SECRET_KEY'])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.db.utils import IntegrityError
from scrapy import log
import time
from scrapy.exceptions import DropItem
from dynamic_scraper.models import SchedulerRuntime
from tvshow_scraper.models import TVShow
from django.forms.models import model_to_dict

from leancloud import Object, Query

def item_to_model(item):
    model_class = getattr(item, 'django_model')
    if not model_class:
        raise TypeError("Item is not a `DjangoItem` or is misconfigured")

    return item.instance

def get_or_create(model):
    model_class = type(model)
    created = False

    # Normally, we would use `get_or_create`. However, `get_or_create` would
    # match all properties of an object (i.e. create a new object
    # anytime it changed) rather than update an existing object.
    #
    # Instead, we do the two steps separately
    try:
        # We have no unique identifier at the moment; use the name for now.
        obj = model_class.objects.get(douban_id=model.douban_id)
        obj['updated_at'] = time.strftime("%c")
    except model_class.DoesNotExist:
        created = True
        obj = model  # DjangoItem created a model for us.

    return (obj, created)

def update_model(destination, source, commit=True):
    pk = destination.pk

    source_dict = source.__dict__
    for (key, value) in source_dict.items():
        setattr(destination, key, value)

    setattr(destination, 'pk', pk)

    if commit:
        destination.save()

    return destination

class DjangoWriterPipeline(object):

    def process_item(self, item, spider):
        try:

            checker_rt = SchedulerRuntime(runtime_type='C')
            checker_rt.save()
            item['checker_runtime'] = checker_rt
            item['source'] = spider.ref_object

            try:
                item_model = item_to_model(item)
            except TypeError:
                return item

            model, created = get_or_create(item_model)

            update_model(model, item_model)

            if created:
                spider.log('==' + model.name + '== created.', log.INFO)
            else:
                spider.log('==' + model.name + '== updated.', log.INFO)

            spider.action_successful = True

        except IntegrityError, e:
            spider.log(str(e), log.ERROR)
            raise DropItem("Missing attribute.")

        return item

<commit_msg>remove unnecessary lib<commit_after># -*- coding: utf-8 -*-
from django.db.utils import IntegrityError
from scrapy import log
import time
from scrapy.exceptions import DropItem
from dynamic_scraper.models import SchedulerRuntime
from tvshow_scraper.models import TVShow
from django.forms.models import model_to_dict

def item_to_model(item):
    model_class = getattr(item, 'django_model')
    if not model_class:
        raise TypeError("Item is not a `DjangoItem` or is misconfigured")

    return item.instance

def get_or_create(model):
    model_class = type(model)
    created = False

    # Normally, we would use `get_or_create`. However, `get_or_create` would
    # match all properties of an object (i.e. create a new object
    # anytime it changed) rather than update an existing object.
    #
    # Instead, we do the two steps separately
    try:
        # We have no unique identifier at the moment; use the name for now.
        obj = model_class.objects.get(douban_id=model.douban_id)
        obj['updated_at'] = time.strftime("%c")
    except model_class.DoesNotExist:
        created = True
        obj = model  # DjangoItem created a model for us.

    return (obj, created)

def update_model(destination, source, commit=True):
    pk = destination.pk

    source_dict = source.__dict__
    for (key, value) in source_dict.items():
        setattr(destination, key, value)

    setattr(destination, 'pk', pk)

    if commit:
        destination.save()

    return destination

class DjangoWriterPipeline(object):

    def process_item(self, item, spider):
        try:

            checker_rt = SchedulerRuntime(runtime_type='C')
            checker_rt.save()
            item['checker_runtime'] = checker_rt
            item['source'] = spider.ref_object

            try:
                item_model = item_to_model(item)
            except TypeError:
                return item

            model, created = get_or_create(item_model)

            update_model(model, item_model)

            if created:
                spider.log('==' + model.name + '== created.', log.INFO)

            else:
                spider.log('==' + model.name + '== updated.', log.INFO)

            spider.action_successful = True

        except IntegrityError, e:
            spider.log(str(e), log.ERROR)
            raise DropItem("Missing attribute.")

        return item

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import os

import nox

LOCAL_DEPS = (os.path.join("..", "api_core"), os.path.join("..", "core"))

@nox.session(python="3.7")
def blacken(session):
    """Run black.

    Format code to uniform standard.
    """
    session.install("black")
    session.run(
        "black",
        "google",
        "tests",
        "docs",
        "--exclude",
        ".*/proto/.*|.*/gapic/.*|.*/.*_pb2.py",
    )

@nox.session(python="3.7")
def lint(session):
    """Run linters.

    Returns a failure if the linters find linting errors or sufficiently
    serious code quality issues.
    """
    session.install("flake8", "black", *LOCAL_DEPS)
    session.run(
        "black",
        "--check",
        "google",
        "tests",
        "docs",
        "--exclude",
        ".*/proto/.*|.*/gapic/.*|.*/.*_pb2.py",
    )
    session.run("flake8", "google", "tests")

@nox.session(python="3.7")
def lint_setup_py(session):
    """Verify that setup.py is valid (including RST check)."""
    session.install("docutils", "pygments")
    session.run("python", "setup.py", "check", "--restructuredtext", "--strict")

def default(session):
    # Install all test dependencies, then install this package in-place.
    session.install("mock", "pytest", "pytest-cov")
    for local_dep in LOCAL_DEPS:
        session.install("-e", local_dep)
    session.install("-e", ".")

    # Run py.test against the unit tests.
    session.run(
        "py.test",
        "--quiet",
        "--cov=google.cloud",
        "--cov=tests.unit",
        "--cov-append",
        "--cov-config=.coveragerc",
        "--cov-report=",
        "--cov-fail-under=80",
        os.path.join("tests", "unit"),
        *session.posargs,
    )

@nox.session(python=["2.7", "3.5", "3.6", "3.7"])
def unit(session):
    """Run the unit test suite."""
    default(session)

@nox.session(python=["2.7", "3.7"])
def system(session):
    """Run the system test suite."""
    system_test_path = os.path.join("tests", "system.py")
    system_test_folder_path = os.path.join("tests", "system")
    # Sanity check: Only run tests if the environment variable is set.
    if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""):
        session.skip("Credentials must be set via environment variable")

    system_test_exists = os.path.exists(system_test_path)
    system_test_folder_exists = os.path.exists(system_test_folder_path)
    # Sanity check: only run tests if found.
    if not system_test_exists and not system_test_folder_exists:
        session.skip("System tests were not found")

    # Use pre-release gRPC for system tests.
    session.install("--pre", "grpcio")

    # Install all test dependencies, then install this package into the
    # virtualenv's dist-packages.
    session.install("mock", "pytest")
    for local_dep in LOCAL_DEPS:
        session.install("-e", local_dep)
    session.install("-e", "../test_utils/")
    session.install("-e", ".")

    # Run py.test against the system tests.
    if system_test_exists:
        session.run("py.test", "--quiet", system_test_path, *session.posargs)
    if system_test_folder_exists:
        session.run("py.test", "--quiet", system_test_folder_path, *session.posargs)

@nox.session(python="3.7")
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.install("coverage", "pytest-cov")
    session.run("coverage", "report", "--show-missing", "--fail-under=80")

    session.run("coverage", "erase")
<commit_msg>Update noxfile.<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import os

import nox

LOCAL_DEPS = (os.path.join("..", "api_core"), os.path.join("..", "core"))

@nox.session(python="3.7")
def lint(session):
    """Run linters.

    Returns a failure if the linters find linting errors or sufficiently
    serious code quality issues.
    """
    session.install("flake8", "black", *LOCAL_DEPS)
    session.run(
        "black",
        "--check",
        "google",
        "tests",
        "docs",
    )
    session.run("flake8", "google", "tests")

@nox.session(python="3.6")
def blacken(session):
    """Run black.

    Format code to uniform standard.
    """
    session.install("black")
    session.run(
        "black",
        "google",
        "tests",
        "docs",
    )

@nox.session(python="3.7")
def lint_setup_py(session):
    """Verify that setup.py is valid (including RST check)."""
    session.install("docutils", "pygments")
    session.run("python", "setup.py", "check", "--restructuredtext", "--strict")

def default(session):
    # Install all test dependencies, then install this package in-place.
    session.install("mock", "pytest", "pytest-cov")
    for local_dep in LOCAL_DEPS:
        session.install("-e", local_dep)
    session.install("-e", ".")

    # Run py.test against the unit tests.
    session.run(
        "py.test",
        "--quiet",
        "--cov=google.cloud",
        "--cov=tests.unit",
        "--cov-append",
        "--cov-config=.coveragerc",
        "--cov-report=",
        "--cov-fail-under=80",
        os.path.join("tests", "unit"),
        *session.posargs,
    )

@nox.session(python=["2.7", "3.5", "3.6", "3.7"])
def unit(session):
    """Run the unit test suite."""
    default(session)

@nox.session(python=["2.7", "3.7"])
def system(session):
    """Run the system test suite."""
    system_test_path = os.path.join("tests", "system.py")
    system_test_folder_path = os.path.join("tests", "system")
    # Sanity check: Only run tests if the environment variable is set.
    if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""):
        session.skip("Credentials must be set via environment variable")

    system_test_exists = os.path.exists(system_test_path)
    system_test_folder_exists = os.path.exists(system_test_folder_path)
    # Sanity check: only run tests if found.
    if not system_test_exists and not system_test_folder_exists:
        session.skip("System tests were not found")

    # Use pre-release gRPC for system tests.
    session.install("--pre", "grpcio")

    # Install all test dependencies, then install this package into the
    # virtualenv's dist-packages.
    session.install("mock", "pytest")
    for local_dep in LOCAL_DEPS:
        session.install("-e", local_dep)
    session.install("-e", "../test_utils/")
    session.install("-e", ".")

    # Run py.test against the system tests.
    if system_test_exists:
        session.run("py.test", "--quiet", system_test_path, *session.posargs)
    if system_test_folder_exists:
        session.run("py.test", "--quiet", system_test_folder_path, *session.posargs)

@nox.session(python="3.7")
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.install("coverage", "pytest-cov")
    session.run("coverage", "report", "--show-missing", "--fail-under=80")

    session.run("coverage", "erase")
<|endoftext|>


<commit_before>#
# Collective Knowledge (basic gemmbench crowdtuning)
#
# See CK LICENSE.txt for licensing details
# See CK COPYRIGHT.txt for copyright details
#
# Developer: Grigori Fursin, Grigori.Fursin@cTuning.org, http://fursin.net
#

cfg={}  # Will be updated by CK (meta description of this module)
work={} # Will be updated by CK (temporal data)
ck=None # Will be updated by CK (initialized CK kernel)

# Local settings
line='****************************************************************'

rep1='remote-ck'
duoa1='SGEMM_NT-explore-f-n-lws'

##############################################################################
# Initialize module

def init(i):
    """

    Input:  {}

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """
    return {'return':0}

##############################################################################
# start gemmbench crowdtuning

def start(i):
    """
    Input:  {
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    import copy
    import os

    # Setting output
    o=i.get('out','')
    oo=''
    if o=='con': oo='con'

    user=''

    curdir=os.getcwd()

    # Params
    hos=i.get('host_os','')
    tos=i.get('target_os', '')
    tdid=i.get('device_id', '')

    #**************************************************************************************************************
    # Welcome info
    ck.out(line)

    r=ck.load_text_file({'text_file':os.path.join(work['path'],'welcome.txt')})
    if r['return']==0:
       s=r['string']

       ck.out(s)

    user=''

    ck.out(line)

    r=ck.inp({'text':'Please, enter your email if you would like to identify your contributions as well as participate in monthly prize draws: '})
    user=r['string'].strip()

    r=ck.inp({'text':'Enter any integer seed: '})
    seed=r['string'].strip()
    if seed!='': seed=int(seed)

    #**************************************************************************************************************
    # Testing remote platform
    ck.out(line)
    ck.out('Testing experiment crowdsourcing server ...')
    ck.out('')

    ii={'action':'test',
        'module_uoa':cfg['module_deps']['program.optimization'],
        'out':'',
        'email':user,
        'repo_uoa':rep1}
    r=ck.access(ii)
    if r['return']>0: return r

    ck.out('  SUCCESS!')

    # Get local platform type
    r=ck.get_os_ck({})
    if r['return']>0: return r

    plat=r['platform']
    sext='.bat'
    if plat=='linux':
       sext='.sh'

    #**************************************************************************************************************
    # Preparing and starting crowd-tuning
    os.environ["CK_CROWDTUNING_USER"] = user

    # Load pipeline json
    r=ck.load_json_file({'json_file':'_setup_program_pipeline_tmp.json'})
    if r['return']>0:
       if r['return']==16:
          ck.out(line)
          ck.out('Cleaning and setting up pipeline ...')

          os.system('_clean_program_pipeline'+sext)

          os.system('_setup_program_pipeline'+sext)
       else:
          return r

    # Restore path
    os.chdir(curdir)

    # Load autotuning json
    r=ck.load_json_file({'json_file':'explore-f-n-lws-remote.json'})
    if r['return']>0: return r
    d=r['dict']

    d['seed']=seed

    x=os.path.join(curdir, '_setup_program_pipeline_tmp.json')

    # Start crowdtuning
    d['action']='autotune'
    d['module_uoa']='pipeline'
    d['data_uoa']='program'
    d['pipeline_from_file']=x
    d['out']='con'

    r=ck.access(d)
    if r['return']>0: return r

    return {'return':0}

##############################################################################
# clean results

def clean(i):
    """
    Input:  {
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    import os

    # Find experiment
    r=ck.access({'action':'find',
                 'module_uoa':cfg['module_deps']['experiment'],
                 'data_uoa':duoa1})
    if r['return']==0:
       p=r['path']

       rx=ck.list_all_files({'path':p, 'add_path':'yes'})
       if rx['return']>0: return rx

       lst=rx['list']

       for q in lst:
           qq=lst[q]

           px=os.path.join(qq.get('path',''),q)
           if os.path.isfile(px):
              os.remove(px)

       # update entry with empty dictionary
       rx=ck.access({'action':'update',
                     'module_uoa':cfg['module_deps']['experiment'],
                     'data_uoa':duoa1,
                     'dict':{},
                     'substitute':'yes',
                     'ignore_update':'yes'})
       if rx['return']>0: return rx

    return {'return':0}
<commit_msg>note when cleaning<commit_after>#
# Collective Knowledge (basic gemmbench crowdtuning)
#
# See CK LICENSE.txt for licensing details
# See CK COPYRIGHT.txt for copyright details
#
# Developer: Grigori Fursin, Grigori.Fursin@cTuning.org, http://fursin.net
#

cfg={}  # Will be updated by CK (meta description of this module)
work={} # Will be updated by CK (temporal data)
ck=None # Will be updated by CK (initialized CK kernel)

# Local settings
line='****************************************************************'

rep1='remote-ck'
duoa1='SGEMM_NT-explore-f-n-lws'

##############################################################################
# Initialize module

def init(i):
    """

    Input:  {}

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """
    return {'return':0}

##############################################################################
# start gemmbench crowdtuning

def start(i):
    """
    Input:  {
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    import copy
    import os

    # Setting output
    o=i.get('out','')
    oo=''
    if o=='con': oo='con'

    user=''

    curdir=os.getcwd()

    # Params
    hos=i.get('host_os','')
    tos=i.get('target_os', '')
    tdid=i.get('device_id', '')

    #**************************************************************************************************************
    # Welcome info
    ck.out(line)

    r=ck.load_text_file({'text_file':os.path.join(work['path'],'welcome.txt')})
    if r['return']==0:
       s=r['string']

       ck.out(s)

    user=''

    ck.out(line)

    r=ck.inp({'text':'Please, enter your email if you would like to identify your contributions as well as participate in monthly prize draws: '})
    user=r['string'].strip()

    r=ck.inp({'text':'Enter any integer seed: '})
    seed=r['string'].strip()
    if seed!='': seed=int(seed)

    #**************************************************************************************************************
    # Testing remote platform
    ck.out(line)
    ck.out('Testing experiment crowdsourcing server ...')
    ck.out('')

    ii={'action':'test',
        'module_uoa':cfg['module_deps']['program.optimization'],
        'out':'',
        'email':user,
        'repo_uoa':rep1}
    r=ck.access(ii)
    if r['return']>0: return r

    ck.out('  SUCCESS!')

    # Get local platform type
    r=ck.get_os_ck({})
    if r['return']>0: return r

    plat=r['platform']
    sext='.bat'
    if plat=='linux':
       sext='.sh'

    #**************************************************************************************************************
    # Preparing and starting crowd-tuning
    os.environ["CK_CROWDTUNING_USER"] = user

    # Load pipeline json
    r=ck.load_json_file({'json_file':'_setup_program_pipeline_tmp.json'})
    if r['return']>0:
       if r['return']==16:
          ck.out(line)
          ck.out('Cleaning and setting up pipeline ...')

          os.system('_clean_program_pipeline'+sext)

          os.system('_setup_program_pipeline'+sext)
       else:
          return r

    # Restore path
    os.chdir(curdir)

    # Load autotuning json
    r=ck.load_json_file({'json_file':'explore-f-n-lws-remote.json'})
    if r['return']>0: return r
    d=r['dict']

    d['seed']=seed

    x=os.path.join(curdir, '_setup_program_pipeline_tmp.json')

    # Start crowdtuning
    d['action']='autotune'
    d['module_uoa']='pipeline'
    d['data_uoa']='program'
    d['pipeline_from_file']=x
    d['out']='con'

    r=ck.access(d)
    if r['return']>0: return r

    return {'return':0}

##############################################################################
# clean results

def clean(i):
    """
    Input:  {
            }

    Output: {
              return       - return code =  0, if successful
                                         >  0, if error
              (error)      - error text if return > 0
            }

    """

    import os

    # Find experiment
    r=ck.access({'action':'find',
                 'module_uoa':cfg['module_deps']['experiment'],
                 'data_uoa':duoa1})
    if r['return']==0:
       p=r['path']

       rx=ck.list_all_files({'path':p, 'add_path':'yes'})
       if rx['return']>0: return rx

       lst=rx['list']

       for q in lst:
           qq=lst[q]

           px=os.path.join(qq.get('path',''),q)
           if os.path.isfile(px):
              os.remove(px)

       # update entry with empty dictionary
       rx=ck.access({'action':'update',
                     'module_uoa':cfg['module_deps']['experiment'],
                     'data_uoa':duoa1,
                     'dict':{},
                     'substitute':'yes',
                     'ignore_update':'yes'})
       if rx['return']>0: return rx

       ck.out('Entry cleaned successfully!')

    return {'return':0}
<|endoftext|>


<commit_before>import os

import numpy as np

from scipy import interpolate

import math

import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt

from pymatgen.io.vasp.outputs import Vasprun
from pymatgen.core.structure import Structure

import warnings

def plot_gamma_surface(fmt='pdf'):
    """
    Collect the energies along a 10x10 grid of static energy
    calculations to plot the Gamma surface between two layers of the 2D
    material.
    """

    lattice = Structure.from_file('CONTCAR').lattice
    area = np.cross(lattice._matrix[0], lattice._matrix[1])[2]

    ax = plt.figure(figsize=(10, 10)).gca()

    n_divs_x = int(math.ceil(lattice.a * 2.5))
    n_divs_y = int(math.ceil(lattice.b * 2.5))

    ax.set_xlim(0, n_divs_x + 1)
    ax.set_ylim(0, n_divs_y + 1)

    ENERGY_ARRAY = []

    X_VALUES = range(n_divs_x)
    Y_VALUES = range(n_divs_y)

    os.chdir('friction/lateral')

    not_converged = []
    for x in X_VALUES:
        ENERGY_ARRAY.append([])
        for y in Y_VALUES:
            dir = '{}x{}'.format(x, y)
            os.chdir(dir)
            try:
                energy = Vasprun('vasprun.xml').final_energy / area
                ENERGY_ARRAY[x].append(energy)
            except:
                not_converged.append('{}x{}'.format(x, y))
                ENERGY_ARRAY[x].append(0)
            os.chdir('../')
        ENERGY_ARRAY[x].append(ENERGY_ARRAY[x][0])

    ENERGY_ARRAY.append([])
    ENERGY_ARRAY[n_divs_x] = ENERGY_ARRAY[0]

    if not_converged:
        warnings.warn('{} did not converge.'.format(not_converged))
        for coords in not_converged:
            ENERGY_ARRAY[
                int(coords.split('x')[0])][int(coords.split('x')[1])] = energy

    minima = []
    maxima = []
    for x in X_VALUES:
        minima.append(min(ENERGY_ARRAY[x]))
        maxima.append(max(ENERGY_ARRAY[x]))

    abs_minimum = min(minima)
    abs_maximum = max(maxima)

    for x in range(n_divs_x + 1):
        for y in range(n_divs_y + 1):

            # Plot all energies relative to the global minimum.
            scaled_energy = ENERGY_ARRAY[x][y] - abs_minimum
            if '{}x{}'.format(x, y) in not_converged:
                color_code = 'w'
            else:
                color_code = plt.cm.jet(
                    scaled_energy / (abs_maximum - abs_minimum)
                    )

            ax.add_patch(plt.Rectangle((x, y), width=1, height=1,
                                       facecolor=color_code,
                                       linewidth=0))

    # Get rid of annoying ticks.
    ax.axes.get_yaxis().set_ticks([])
    ax.axes.get_xaxis().set_ticks([])

    plt.savefig('gamma_surface.{}'.format(fmt), transparent=True)
    os.chdir('../../')

def get_basin_and_peak_locations():
    """
    Find which directories inside `friction/lateral` represent
    the minimum (basin) and maximum (peak) energy stacking
    configurations. Returns a tuple of the form (basin, peak).
    """

    lattice = Structure.from_file('CONTCAR').lattice

    n_divs_x = int(math.ceil(lattice.a * 2.5))
    n_divs_y = int(math.ceil(lattice.b * 2.5))

    X_VALUES = range(n_divs_x)
    Y_VALUES = range(n_divs_y)

    os.chdir('friction/lateral')

    abs_maximum = -10000
    abs_minimum = 0
    for x in X_VALUES:
        for y in Y_VALUES:
            dir = '{}x{}'.format(x, y)
            os.chdir(dir)
            try:
                energy = Vasprun('vasprun.xml').final_energy
                if energy < abs_minimum:
                    basin = dir
                    abs_minimum = energy
                if energy > abs_maximum:
                    peak = dir
                    abs_minimum = energy
            except:
                pass
            os.chdir('../')
    os.chdir('../../')

    return(basin, peak)

def plot_de_dx(fmt='pdf'):
    """
    Plot the sinusoidal curve of delta E between basin and saddle
    points for each normal spacing dz.
    """

    os.chdir('friction/normal')

    f, (ax1, ax2) = plt.subplots(2, figsize=(16, 16))

    for spacing in sorted([float(spc) for spc in os.listdir(os.getcwd()) if
            os.path.isdir(spc)]):
        os.chdir(str(spacing))
        subdirectories = os.listdir(os.getcwd())

        amplitude = abs(
            Vasprun('{}/vasprun.xml'.format(subdirectories[0])).final_energy
            - Vasprun('{}/vasprun.xml'.format(subdirectories[1])).final_energy
            ) / 2

        start_coords = Structure.from_file(
            '{}/POSCAR'.format(subdirectories[0])
            ).sites[-1].coords
        end_coords = Structure.from_file(
            '{}/POSCAR'.format(subdirectories[1])
            ).sites[-1].coords
        dist = np.sqrt(
            (start_coords[0] - end_coords[0])**2
            + (start_coords[1] - end_coords[1])**2)

        b = (2 * np.pi) / (dist * 2)

        x = np.arange(0, 4, 0.01)
        sinx = [amplitude * np.sin(b * val) + amplitude for val in x]
        cosx = [b * amplitude * np.cos(b * val)
                if np.cos(b * val) > 0 else 0 for val in x]

        ax1.plot(x, sinx, linewidth=8, color=plt.cm.jet(-(spacing - 4) / 2.51),
                 label=spacing)
        ax1.set_xticklabels(ax1.get_xticks(), family='serif', fontsize=18)
        ax1.set_yticklabels(ax1.get_yticks(), family='serif', fontsize=18)
        ax1.set_xlabel(r'$\mathrm{\Delta d\/(\AA)}$', family='serif',
            fontsize=24)
        ax1.set_ylabel(r'$\mathrm{E(z)\/(eV)}$', family='serif', fontsize=24)
        ax2.plot(x, cosx, linewidth=8, color=plt.cm.jet(-(spacing - 4) / 2.51),
                 label=spacing)
        ax2.set_xticklabels(ax2.get_xticks(), family='serif', fontsize=18)
        ax2.set_yticklabels(ax2.get_yticks(), family='serif', fontsize=18)
        ax2.set_xlabel(r'$\mathrm{\Delta d\/(\AA)}$', family='serif',
            fontsize=24)
        ax2.set_ylabel(r'$\mathrm{f_x\/(eV/\AA)}$', family='serif', fontsize=24)
        os.chdir('../')

    ax1.legend(loc='upper right')
    ax2.legend(loc='upper right')
    plt.savefig('de_dx.{}'.format(fmt))
    os.chdir('../../')

def plot_de_dz(basin_dir, fmt='pdf'):
    """
    Plot the LJ-like curve of the energy at the basin point
    as a function of normal spacing dz.
    """

    os.chdir('friction/normal')
    spacings = [float(dir) for dir in os.listdir(os.getcwd()) if
                os.path.isdir(dir)]
    spacings.sort()

    fig = plt.figure(figsize=(16, 10))
    ax = fig.gca()
    ax2 = ax.twinx()

    abs_E = [
        Vasprun('{}/{}/vasprun.xml'.format(spacing, basin_dir)).final_energy
        for spacing in spacings
        ]
    E = [energy - abs_E[-1] for energy in abs_E]

    spline = interpolate.splrep(spacings, E, s=0)
    xnew = np.arange(1.5, 4, 0.001)
    ynew = interpolate.splev(xnew, spline, der=0)
    ynew_slope = interpolate.splev(spacings, spline, der=1)

    ax.set_xlim(spacings[0], spacings[-1])

    ax.plot([spacings[0], spacings[-1]], [0, 0], '--', color=plt.cm.jet(0))
    ax2.plot([spacings[0], spacings[-1]], [0, 0], '--', color=plt.cm.jet(0.9))
    E_z = ax.plot(xnew, ynew, color=plt.cm.jet(0),
                  linewidth=4, label=r'$\mathrm{E(z)}$')
    f_N = ax2.plot(spacings, [-y for y in ynew_slope], color=plt.cm.jet(0.9),
                   linewidth=4, label=r'$\mathrm{f_N}$')

    ax.set_ylim(ax.get_ylim())

    ax.set_xticklabels(ax.get_xticks(), family='serif', fontsize=18)
    ax.set_yticklabels(ax.get_yticks(), family='serif', fontsize=18)
    ax2.set_yticklabels(ax2.get_yticks(), family='serif', fontsize=18)

    ax.set_xlabel(r'$\mathrm{z\/(\AA)}$', fontsize=24)
    ax.set_ylabel(r'$\mathrm{E(z)\/(eV)}$', fontsize=24)
    ax2.set_ylabel(r'$\mathrm{f_N\/(eV/\AA)}$', fontsize=24)

    data = E_z + f_N
    labs = [l.get_label() for l in data]

    ax.legend(data, labs, loc='upper right', fontsize=24)

    ax.plot(spacings, E, linewidth=0, marker='o', color=plt.cm.jet(0),
            markersize=10, markeredgecolor='none')

    plt.savefig('de_dz.pdf')

    os.chdir('../../')
<commit_msg>Add get_mu_vs_F_N.<commit_after>import os

import numpy as np

from scipy import interpolate

import math

import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt

from pymatgen.io.vasp.outputs import Vasprun
from pymatgen.core.structure import Structure

import warnings

def plot_gamma_surface(fmt='pdf'):
    """
    Collect the energies along a 10x10 grid of static energy
    calculations to plot the Gamma surface between two layers of the 2D
    material.
    """

    lattice = Structure.from_file('CONTCAR').lattice
    area = np.cross(lattice._matrix[0], lattice._matrix[1])[2]

    ax = plt.figure(figsize=(10, 10)).gca()

    n_divs_x = int(math.ceil(lattice.a * 2.5))
    n_divs_y = int(math.ceil(lattice.b * 2.5))

    ax.set_xlim(0, n_divs_x + 1)
    ax.set_ylim(0, n_divs_y + 1)

    ENERGY_ARRAY = []

    X_VALUES = range(n_divs_x)
    Y_VALUES = range(n_divs_y)

    os.chdir('friction/lateral')

    not_converged = []
    for x in X_VALUES:
        ENERGY_ARRAY.append([])
        for y in Y_VALUES:
            dir = '{}x{}'.format(x, y)
            os.chdir(dir)
            try:
                energy = Vasprun('vasprun.xml').final_energy / area
                ENERGY_ARRAY[x].append(energy)
            except:
                not_converged.append('{}x{}'.format(x, y))
                ENERGY_ARRAY[x].append(0)
            os.chdir('../')
        ENERGY_ARRAY[x].append(ENERGY_ARRAY[x][0])

    ENERGY_ARRAY.append([])
    ENERGY_ARRAY[n_divs_x] = ENERGY_ARRAY[0]

    if not_converged:
        warnings.warn('{} did not converge.'.format(not_converged))
        for coords in not_converged:
            ENERGY_ARRAY[
                int(coords.split('x')[0])][int(coords.split('x')[1])] = energy

    minima = []
    maxima = []
    for x in X_VALUES:
        minima.append(min(ENERGY_ARRAY[x]))
        maxima.append(max(ENERGY_ARRAY[x]))

    abs_minimum = min(minima)
    abs_maximum = max(maxima)

    for x in range(n_divs_x + 1):
        for y in range(n_divs_y + 1):

            # Plot all energies relative to the global minimum.
            scaled_energy = ENERGY_ARRAY[x][y] - abs_minimum
            if '{}x{}'.format(x, y) in not_converged:
                color_code = 'w'
            else:
                color_code = plt.cm.jet(
                    scaled_energy / (abs_maximum - abs_minimum)
                    )

            ax.add_patch(plt.Rectangle((x, y), width=1, height=1,
                                       facecolor=color_code,
                                       linewidth=0))

    # Get rid of annoying ticks.
    ax.axes.get_yaxis().set_ticks([])
    ax.axes.get_xaxis().set_ticks([])

    plt.savefig('gamma_surface.{}'.format(fmt), transparent=True)
    os.chdir('../../')

def get_basin_and_peak_locations():
    """
    Find which directories inside `friction/lateral` represent
    the minimum (basin) and maximum (peak) energy stacking
    configurations. Returns a tuple of the form (basin, peak).
    """

    lattice = Structure.from_file('CONTCAR').lattice

    n_divs_x = int(math.ceil(lattice.a * 2.5))
    n_divs_y = int(math.ceil(lattice.b * 2.5))

    X_VALUES = range(n_divs_x)
    Y_VALUES = range(n_divs_y)

    os.chdir('friction/lateral')

    abs_maximum = -10000
    abs_minimum = 0
    for x in X_VALUES:
        for y in Y_VALUES:
            dir = '{}x{}'.format(x, y)
            os.chdir(dir)
            try:
                energy = Vasprun('vasprun.xml').final_energy
                if energy < abs_minimum:
                    basin = dir
                    abs_minimum = energy
                if energy > abs_maximum:
                    peak = dir
                    abs_maximum = energy
            except:
                pass
            os.chdir('../')
    os.chdir('../../')

    return(basin, peak)

def plot_friction_force(fmt='pdf'):
    """
    Plot the sinusoidal curve of delta E between basin and saddle
    points for each normal spacing dz.
    """

    os.chdir('friction/normal')

    f, (ax1, ax2) = plt.subplots(2, figsize=(16, 16))

    for spacing in sorted([float(spc) for spc in os.listdir(os.getcwd()) if
            os.path.isdir(spc)]):
        os.chdir(str(spacing))
        subdirectories = os.listdir(os.getcwd())

        amplitude = abs(
            Vasprun('{}/vasprun.xml'.format(subdirectories[0])).final_energy
            - Vasprun('{}/vasprun.xml'.format(subdirectories[1])).final_energy
            ) / 2

        start_coords = Structure.from_file(
            '{}/POSCAR'.format(subdirectories[0])
            ).sites[-1].coords
        end_coords = Structure.from_file(
            '{}/POSCAR'.format(subdirectories[1])
            ).sites[-1].coords
        dist = np.sqrt(
            (start_coords[0] - end_coords[0])**2
            + (start_coords[1] - end_coords[1])**2)

        b = (2 * np.pi) / (dist * 2)

        x = np.arange(0, 4, 0.01)
        sinx = [amplitude * np.sin(b * val) + amplitude for val in x]
        cosx = [b * amplitude * np.cos(b * val)
                if np.cos(b * val) > 0 else 0 for val in x]

        ax1.plot(x, sinx, linewidth=8, color=plt.cm.jet(-(spacing - 4) / 2.51),
                 label=spacing)
        ax1.set_xticklabels(ax1.get_xticks(), family='serif', fontsize=18)
        ax1.set_yticklabels(ax1.get_yticks(), family='serif', fontsize=18)
        ax1.set_xlabel(r'$\mathrm{\Delta d\/(\AA)}$', family='serif',
            fontsize=24)
        ax1.set_ylabel(r'$\mathrm{E(z)\/(eV)}$', family='serif', fontsize=24)
        ax2.plot(x, cosx, linewidth=8, color=plt.cm.jet(-(spacing - 4) / 2.51),
                 label=spacing)
        ax2.set_xticklabels(ax2.get_xticks(), family='serif', fontsize=18)
        ax2.set_yticklabels(ax2.get_yticks(), family='serif', fontsize=18)
        ax2.set_xlabel(r'$\mathrm{\Delta d\/(\AA)}$', family='serif',
            fontsize=24)
        ax2.set_ylabel(r'$\mathrm{F_f\/(eV/\AA)}$', family='serif', fontsize=24)
        os.chdir('../')

    ax1.legend(loc='upper right')
    ax2.legend(loc='upper right')
    plt.savefig('F_f.{}'.format(fmt))
    os.chdir('../../')

def plot_normal_force(basin_dir, fmt='pdf'):
    """
    Plot the LJ-like curve of the energy at the basin point
    as a function of normal spacing dz.
    """

    os.chdir('friction/normal')
    spacings = [float(dir) for dir in os.listdir(os.getcwd()) if
                os.path.isdir(dir)]
    spacings.sort()

    fig = plt.figure(figsize=(16, 10))
    ax = fig.gca()
    ax2 = ax.twinx()

    abs_E = [
        Vasprun('{}/{}/vasprun.xml'.format(spacing, basin_dir)).final_energy
        for spacing in spacings
        ]
    E = [energy - abs_E[-1] for energy in abs_E]

    spline = interpolate.splrep(spacings, E, s=0)
    xnew = np.arange(1.5, 4, 0.001)
    ynew = interpolate.splev(xnew, spline, der=0)
    ynew_slope = interpolate.splev(spacings, spline, der=1)

    ax.set_xlim(spacings[0], spacings[-1])

    ax.plot([spacings[0], spacings[-1]], [0, 0], '--', color=plt.cm.jet(0))
    ax2.plot([spacings[0], spacings[-1]], [0, 0], '--', color=plt.cm.jet(0.9))
    E_z = ax.plot(xnew, ynew, color=plt.cm.jet(0),
                  linewidth=4, label=r'$\mathrm{E(z)}$')
    F_N = ax2.plot(spacings, [-y for y in ynew_slope], color=plt.cm.jet(0.9),
                   linewidth=4, label=r'$\mathrm{F_N}$')

    ax.set_ylim(ax.get_ylim())

    ax.set_xticklabels(ax.get_xticks(), family='serif', fontsize=18)
    ax.set_yticklabels(ax.get_yticks(), family='serif', fontsize=18)
    ax2.set_yticklabels(ax2.get_yticks(), family='serif', fontsize=18)

    ax.set_xlabel(r'$\mathrm{z\/(\AA)}$', fontsize=24)
    ax.set_ylabel(r'$\mathrm{E(z)\/(eV)}$', fontsize=24)
    ax2.set_ylabel(r'$\mathrm{F_N\/(eV/\AA)}$', fontsize=24)

    data = E_z + F_N
    labs = [l.get_label() for l in data]

    ax.legend(data, labs, loc='upper right', fontsize=24)

    ax.plot(spacings, E, linewidth=0, marker='o', color=plt.cm.jet(0),
            markersize=10, markeredgecolor='none')

    plt.savefig('F_N.{}'.format(fmt))

    os.chdir('../../')

def plot_mu_vs_F_N(basin_dir, fmt='pdf'):
    """
    Plot friction coefficient 'mu' vs. F_Normal.
    mu = F_friction / F_Normal
    """

    fig = plt.figure(figsize=(16, 10))
    ax = fig.gca()
    ax2 = ax.twinx()

    os.chdir('friction/normal')
    spacings = [float(dir) for dir in os.listdir(os.getcwd()) if
                os.path.isdir(dir)]
    spacings.sort()

    abs_E = [
        Vasprun('{}/{}/vasprun.xml'.format(spacing, basin_dir)).final_energy
        for spacing in spacings
        ]
    E = [energy - abs_E[-1] for energy in abs_E]

    spline = interpolate.splrep(spacings, E, s=0)
    xnew = np.arange(1.5, 4, 0.001)
    ynew = interpolate.splev(xnew, spline, der=0)
    ynew_slope = interpolate.splev(spacings, spline, der=1)
    F_N = [-y for y in ynew_slope]

    os.chdir('../../friction/normal')

    F_f = []

    for spacing in sorted([float(spc) for spc in os.listdir(os.getcwd()) if
            os.path.isdir(spc)]):
        os.chdir(str(spacing))
        subdirectories = os.listdir(os.getcwd())

        amplitude = abs(
            Vasprun('{}/vasprun.xml'.format(subdirectories[0])).final_energy
            - Vasprun('{}/vasprun.xml'.format(subdirectories[1])).final_energy
            ) / 2

        start_coords = Structure.from_file(
            '{}/POSCAR'.format(subdirectories[0])
            ).sites[-1].coords
        end_coords = Structure.from_file(
            '{}/POSCAR'.format(subdirectories[1])
            ).sites[-1].coords
        dist = np.sqrt(
            (start_coords[0] - end_coords[0])**2
            + (start_coords[1] - end_coords[1])**2)

        b = (2 * np.pi) / (dist * 2)

        x = np.arange(0, 4, 0.01)
        sinx = [amplitude * np.sin(b * val) + amplitude for val in x]
        cosx = [b * amplitude * np.cos(b * val)
                if np.cos(b * val) > 0 else 0 for val in x]
        F_f.append(max(cosx))
        os.chdir('../')

    os.chdir('../../')

    mu = [f / N for f, N in zip(F_f, F_N)]

    remove_indices = []
    for i in range(len(mu)):
        if mu[i] > 1 or mu[i] < 0:
            remove_indices.append(i)
    trimmed_mu = [mu[i] for i in range(len(mu)) if i not in remove_indices]
    trimmed_F_N = [F_N[i] for i in range(len(F_N)) if i not in remove_indices]

    ax = plt.figure().gca()
    ax.plot(F_N, mu, linewidth=2, marker='o', markeredgecolor='none',
            markersize=3, color=plt.cm.jet(0))
    plt.savefig('mu_vs_F_N.{}'.format(fmt))

def get_mu_vs_F_N(basin_dir):
    """
    Essentially the same function as plotting, but without the plot.
    Returns {'F_N': F_N, 'mu': mu}
    """

    os.chdir('friction/normal')
    spacings = [float(dir) for dir in os.listdir(os.getcwd()) if
                os.path.isdir(dir)]
    spacings.sort()

    abs_E = [
        Vasprun('{}/{}/vasprun.xml'.format(spacing, basin_dir)).final_energy
        for spacing in spacings
        ]
    E = [energy - abs_E[-1] for energy in abs_E]

    spline = interpolate.splrep(spacings, E, s=0)
    xnew = np.arange(1.5, 4, 0.001)
    ynew = interpolate.splev(xnew, spline, der=0)
    ynew_slope = interpolate.splev(spacings, spline, der=1)
    F_N = [-y for y in ynew_slope]

    os.chdir('../../friction/normal')

    F_f = []

    for spacing in sorted([float(spc) for spc in os.listdir(os.getcwd()) if
            os.path.isdir(spc)]):
        os.chdir(str(spacing))
        subdirectories = os.listdir(os.getcwd())

        amplitude = abs(
            Vasprun('{}/vasprun.xml'.format(subdirectories[0])).final_energy
            - Vasprun('{}/vasprun.xml'.format(subdirectories[1])).final_energy
            ) / 2

        start_coords = Structure.from_file(
            '{}/POSCAR'.format(subdirectories[0])
            ).sites[-1].coords
        end_coords = Structure.from_file(
            '{}/POSCAR'.format(subdirectories[1])
            ).sites[-1].coords
        dist = np.sqrt(
            (start_coords[0] - end_coords[0])**2
            + (start_coords[1] - end_coords[1])**2)

        b = (2 * np.pi) / (dist * 2)

        x = np.arange(0, 4, 0.01)
        sinx = [amplitude * np.sin(b * val) + amplitude for val in x]
        cosx = [b * amplitude * np.cos(b * val)
                if np.cos(b * val) > 0 else 0 for val in x]
        F_f.append(max(cosx))
        os.chdir('../')

    os.chdir('../../')

    mu = [f / N for f, N in zip(F_f, F_N)]

    remove_indices = []
    for i in range(len(mu)):
        if mu[i] > 1 or mu[i] < 0:
            remove_indices.append(i)
    trimmed_mu = [mu[i] for i in range(len(mu)) if i not in remove_indices]
    trimmed_F_N = [F_N[i] for i in range(len(F_N)) if i not in remove_indices]

    return({'F_N': F_N, 'mu': mu})
<|endoftext|>


<commit_before>#ipc_lista1.5
#Professor: Jucimar Junior
#Any Mendes Carvalho - 1615310044
#
#
#
#

#Faça um programa que converta metros para centímetros.

metros = input("Digite o valor em metros que deseja converter em centímetros: ")
centimetros = metros * 100
print "Esse valor equivale a: %d
<commit_msg>Update ipc_lista1.5.py<commit_after>#ipc_lista1.5
#Professor: Jucimar Junior
#Any Mendes Carvalho - 1615310044
#
#
#
#

#Faça um programa que converta metros para centímetros.

metros = input("Digite o valor em metros que deseja converter em centímetros: ")
centimetros = metros * 100
print "Esse valor equivale a: %d"
<|endoftext|>