<commit_before>#!/usr/bin/python3
'''
Description: Fetches a list of all the Steam ID's and checks the
current pricing of each game
'''
import time
import requests
import json
import datetime
from sqlalchemy import create_engine, ForeignKey
from sqlalchemy import Column, Date, DateTime, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound

#Globals
engine = create_engine('sqlite:///games.db')
Base = declarative_base()
API_URL = "http://store.steampowered.com/api/appdetails/"
LIMIT = 250

#DB Table descriptions
class Blacklist(Base):
    __tablename__ = 'blacklist'
    id = Column(Integer, primary_key=True)

    def __repr__(self):
        return "<Blacklist(id='{}')>".format(self.id)

class Game(Base):
    __tablename__ = 'games'
    id = Column(Integer, primary_key=True)
    name = Column(String(100))
    last_update = Column(DateTime, onupdate=datetime.datetime.utcnow)
    init_price = Column(Integer, default=0)
    final_price = Column(Integer, default=0)
    lowest_price = Column(Integer, default=0)
    highest_price = Column(Integer, default=0)

    def __repr__(self):
        return "<Game(id='{}', name='{}', last_update='{}', initial_price='{}', final_price='{}')>".format(self.id, self.name, self.last_update, self.init_price, self.final_price)

#Dumps the game database
def dump_game_db(session):
    dump = {}
    for g in session.query(Game).all():
        dump[g.id] = g.name
        #print("{} - {}".format(g.name, g.id))
    output = json.dumps(dump)
    return output

#Builds list of all the possible Steam ID's
def build_list():
    URL = "http://api.steampowered.com/ISteamApps/GetAppList/v2"
    response = requests.get(URL)
    game_list = response.json()["applist"]["apps"]
    return game_list

#Maps the Steam ID to an actual name
def name_matcher(appid, master_list):
    for game in master_list:
        if int(appid) == game['appid']:
            return game['name']

#Queries the game DB
def query_db(session, game):
    try:
        result = session.query(Game).filter_by(id=game).one()
        return result
    except MultipleResultsFound as e:
        print("{}".format(e))
        return False
    except NoResultFound as e:
        print("Couldn't find ID {} : {}".format(game, e))
        return False

#Builds a list of all the blacklist ID's (Those that have no price)
def build_blacklist(session):
    blacklist = []
    for black in session.query(Blacklist).all():
        blacklist.append(black.id)
    return blacklist

#Updates the game DB
def update_db(session, game, field, value):
    try:
        session.query(Game).filter_by(id=game).update({field: value})
    except:
        print("Unknown error occured updating the DB!")

#Main routine for fetching the current price per game
def fetchdump(session, appids, master_list):
    blacklist = build_blacklist(session)
    for applist in appids:
        params = {
            "appids": ",".join(applist),
            "filters": "price_overview"
        }
        response = requests.get(API_URL, params=params)
        try:
            data = response.json()
        except:
            print("Error requesting data for the following ids: {} \n continuing.".format(", ".join(applist)))
            continue
        for game in data:
            if int(game) in blacklist:
                print("Skipping {} due to blacklist".format(game))
                continue
            if data[game]["success"] is True and data[game]["data"]:
                init_price = data[game]["data"]["price_overview"]["initial"]
                final_price = data[game]["data"]["price_overview"]["final"]
                name = name_matcher(game, master_list)
                game_obj = Game(id=game, name=name, init_price=init_price, final_price=final_price)
                price_check = query_db(session, game)
                if price_check:
                    if price_check.final_price != final_price:
                        update_db(session, game, "final_price", final_price)
                    if price_check.lowest_price > final_price:
                        update_db(session, game, "lowest_price", final_price)
                    if price_check.highest_price < final_price:
                        update_db(session, game, "highest_price", final_price)
                else:
                    session.add(game_obj)
            else:
                print("ID {} is false for game: {}".format(game, name_matcher(game,master_list)))
                blacklist_obj = Blacklist(id=game)
                session.add(blacklist_obj)
            try:
                session.commit()
            except IntegrityError as err:
                print("Error updating DB! {}".format(err))
        print("Sleeping 30 seconds until the next batch")
        try:
            time.sleep(30)
        except KeyboardInterrupt:
            exit(1)

#Routine for splitting up the queries into chunks of a certain limit
def chunker(l, n):
    """Yield successive n-sized chunks from list l."""
    for i in range(0, len(l), n):
        yield l[i:i+n]

#Main method (starting point)
def main():
    master_list = build_list()
    appids = []
    for game in master_list:
        appids.append(str(game["appid"]))
    appids = list(chunker(appids, LIMIT))
    Base.metadata.create_all(engine)
    Session = sessionmaker(bind=engine)
    session = Session()
    #json_game_db = dump_game_db(session)
    fetchdump(session, appids, master_list)

if __name__ == "__main__":
    main()
<commit_msg>if the request returns null split the list up and try again. (Fixes #1) Also pull appids that are blacklisted while building appids list.<commit_after>#!/usr/bin/python3
'''
Description: Fetches a list of all the Steam ID's and checks the
current pricing of each game
'''
import time
import requests
import json
import datetime
from sqlalchemy import create_engine, ForeignKey
from sqlalchemy import Column, Date, DateTime, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound

#Globals
engine = create_engine('sqlite:///games.db')
Base = declarative_base()
API_URL = "http://store.steampowered.com/api/appdetails/"
LIMIT = 250

#DB Table descriptions
class Blacklist(Base):
    __tablename__ = 'blacklist'
    id = Column(Integer, primary_key=True)

    def __repr__(self):
        return "<Blacklist(id='{}')>".format(self.id)

class Game(Base):
    __tablename__ = 'games'
    id = Column(Integer, primary_key=True)
    name = Column(String(100))
    last_update = Column(DateTime, onupdate=datetime.datetime.utcnow)
    init_price = Column(Integer, default=0)
    final_price = Column(Integer, default=0)
    lowest_price = Column(Integer, default=0)
    highest_price = Column(Integer, default=0)

    def __repr__(self):
        return "<Game(id='{}', name='{}', last_update='{}', initial_price='{}', final_price='{}')>".format(self.id, self.name, self.last_update, self.init_price, self.final_price)

#Dumps the game database
def dump_game_db(session):
    dump = {}
    for g in session.query(Game).all():
        dump[g.id] = g.name
        #print("{} - {}".format(g.name, g.id))
    output = json.dumps(dump)
    return output

#Builds list of all the possible Steam ID's
def build_list():
    URL = "http://api.steampowered.com/ISteamApps/GetAppList/v2"
    response = requests.get(URL)
    game_list = response.json()["applist"]["apps"]
    return game_list

#Maps the Steam ID to an actual name
def name_matcher(appid, master_list):
    for game in master_list:
        if int(appid) == game['appid']:
            return game['name']

#Queries the game DB
def query_db(session, game):
    try:
        result = session.query(Game).filter_by(id=game).one()
        return result
    except MultipleResultsFound as e:
        print("{}".format(e))
        return False
    except NoResultFound as e:
        print("Couldn't find ID {} : {}".format(game, e))
        return False

#Builds a list of all the blacklist ID's (Those that have no price)
def build_blacklist(session):
    blacklist = []
    for black in session.query(Blacklist).all():
        blacklist.append(black.id)
    return blacklist

#Updates the game DB
def update_db(session, game, field, value):
    try:
        session.query(Game).filter_by(id=game).update({field: value})
    except:
        print("Unknown error occured updating the DB!")

# Split a problematic list in half to try and identify the bad ID for blacklisting.
def list_split(session, applist, master_list):
    """Receives a list, splits in half, resends the list of two lists into fetchdump()"""
    newapplist = [applist[::2], applist[1::2]]
    fetchdump(session, newapplist, master_list)
    return

#Main routine for fetching the current price per game
def fetchdump(session, appids, master_list):
    blacklist = build_blacklist(session)
    for applist in appids:
        params = {
            "appids": ",".join(applist),
            "filters": "price_overview"
        }
        response = requests.get(API_URL, params=params)
        try:
            data = response.json()
        except:
            print("Error requesting data for the following ids: {} \n continuing after splitting them up and retrying.".format(", ".join(applist)))
            if len(applist) <= 1:
                print("ID {} is false for game: {}".format(game, name_matcher(game,master_list)))
                blacklist_obj = Blacklist(id=game)
                session.add(blacklist_obj)
            else:
                list_split(session, applist, master_list)
            continue
        for game in data:
            if int(game) in blacklist: #this check should be done prior to the web request.
                print("Skipping {} due to blacklist".format(game))
                continue
            if data[game]["success"] is True and data[game]["data"]:
                init_price = data[game]["data"]["price_overview"]["initial"]
                final_price = data[game]["data"]["price_overview"]["final"]
                name = name_matcher(game, master_list)
                game_obj = Game(id=game, name=name, init_price=init_price, final_price=final_price)
                price_check = query_db(session, game)
                if price_check:
                    if price_check.final_price != final_price:
                        update_db(session, game, "final_price", final_price)
                    if price_check.lowest_price > final_price:
                        update_db(session, game, "lowest_price", final_price)
                    if price_check.highest_price < final_price:
                        update_db(session, game, "highest_price", final_price)
                else:
                    session.add(game_obj)
            else:
                print("ID {} is false for game: {}".format(game, name_matcher(game,master_list)))
                blacklist_obj = Blacklist(id=game)
                session.add(blacklist_obj)
            try:
                session.commit()
            except IntegrityError as err:
                print("Error updating DB! {}".format(err))
        print("Sleeping 30 seconds until the next batch")
        try:
            time.sleep(30)
        except KeyboardInterrupt:
            exit(1)

#Routine for splitting up the queries into chunks of a certain limit
def chunker(l, n):
    """Yield successive n-sized chunks from list l."""
    for i in range(0, len(l), n):
        yield l[i:i+n]

#Main method (starting point)
def main():
    Base.metadata.create_all(engine)
    Session = sessionmaker(bind=engine)
    session = Session()
    blacklist = build_blacklist(session)
    master_list = build_list()
    appids = []
    for game in master_list:
        if game["appid"] not in blacklist:
            appids.append(str(game["appid"]))
        else:
            print("Skipping ID {}:{} because it is blacklisted".format(game["appid"], game["name"]))
    appids = list(chunker(appids, LIMIT))
    #json_game_db = dump_game_db(session)
    fetchdump(session, appids, master_list)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from sqlalchemy import or_

from airflow.jobs import BackfillJob
from airflow.models import DagRun, TaskInstance
from airflow.operators.subdag_operator import SubDagOperator
from airflow.utils import timezone
from airflow.utils.db import provide_session
from airflow.utils.state import State

def _create_dagruns(dag, execution_dates, state, run_id_template):
    """
    Infers from the dates which dag runs need to be created and does so.
    :param dag: the dag to create dag runs for
    :param execution_dates: list of execution dates to evaluate
    :param state: the state to set the dag run to
    :param run_id_template:the template for run id to be with the execution date
    :return: newly created and existing dag runs for the execution dates supplied
    """
    # find out if we need to create any dag runs
    drs = DagRun.find(dag_id=dag.dag_id, execution_date=execution_dates)
    dates_to_create = list(set(execution_dates) - set([dr.execution_date for dr in drs]))

    for date in dates_to_create:
        dr = dag.create_dagrun(
            run_id=run_id_template.format(date.isoformat()),
            execution_date=date,
            start_date=timezone.utcnow(),
            external_trigger=False,
            state=state,
        )
        drs.append(dr)

    return drs

@provide_session
def set_state(task, execution_date, upstream=False, downstream=False,
              future=False, past=False, state=State.SUCCESS, commit=False, session=None):
    """
    Set the state of a task instance and if needed its relatives. Can set state
    for future tasks (calculated from execution_date) and retroactively
    for past tasks. Will verify integrity of past dag runs in order to create
    tasks that did not exist. It will not create dag runs that are missing
    on the schedule (but it will as for subdag dag runs if needed).
    :param task: the task from which to work. task.task.dag needs to be set
    :param execution_date: the execution date from which to start looking
    :param upstream: Mark all parents (upstream tasks)
    :param downstream: Mark all siblings (downstream tasks) of task_id, including SubDags
    :param future: Mark all future tasks on the interval of the dag up until
        last execution date.
    :param past: Retroactively mark all tasks starting from start_date of the DAG
    :param state: State to which the tasks need to be set
    :param commit: Commit tasks to be altered to the database
    :param session: database session
    :return: list of tasks that have been created and updated
    """
    assert timezone.is_localized(execution_date)

    # microseconds are supported by the database, but is not handled
    # correctly by airflow on e.g. the filesystem and in other places
    execution_date = execution_date.replace(microsecond=0)

    assert task.dag is not None
    dag = task.dag

    latest_execution_date = dag.latest_execution_date
    assert latest_execution_date is not None

    # determine date range of dag runs and tasks to consider
    end_date = latest_execution_date if future else execution_date

    if 'start_date' in dag.default_args:
        start_date = dag.default_args['start_date']
    elif dag.start_date:
        start_date = dag.start_date
    else:
        start_date = execution_date

    start_date = execution_date if not past else start_date

    if dag.schedule_interval == '@once':
        dates = [start_date]
    else:
        dates = dag.date_range(start_date=start_date, end_date=end_date)

    # find relatives (siblings = downstream, parents = upstream) if needed
    task_ids = [task.task_id]
    if downstream:
        relatives = task.get_flat_relatives(upstream=False)
        task_ids += [t.task_id for t in relatives]
    if upstream:
        relatives = task.get_flat_relatives(upstream=True)
        task_ids += [t.task_id for t in relatives]

    # verify the integrity of the dag runs in case a task was added or removed
    # set the confirmed execution dates as they might be different
    # from what was provided
    confirmed_dates = []
    drs = DagRun.find(dag_id=dag.dag_id, execution_date=dates)
    for dr in drs:
        dr.dag = dag
        dr.verify_integrity()
        confirmed_dates.append(dr.execution_date)

    # go through subdagoperators and create dag runs. We will only work
    # within the scope of the subdag. We wont propagate to the parent dag,
    # but we will propagate from parent to subdag.
    dags = [dag]
    sub_dag_ids = []
    while len(dags) > 0:
        current_dag = dags.pop()
        for task_id in task_ids:
            if not current_dag.has_task(task_id):
                continue

            current_task = current_dag.get_task(task_id)
            if isinstance(current_task, SubDagOperator):
                # this works as a kind of integrity check
                # it creates missing dag runs for subdagoperators,
                # maybe this should be moved to dagrun.verify_integrity
                drs = _create_dagruns(current_task.subdag,
                                      execution_dates=confirmed_dates,
                                      state=State.RUNNING,
                                      run_id_template=BackfillJob.ID_FORMAT_PREFIX)

                for dr in drs:
                    dr.dag = current_task.subdag
                    dr.verify_integrity()
                    if commit:
                        dr.state = state
                        session.merge(dr)

                dags.append(current_task.subdag)
                sub_dag_ids.append(current_task.subdag.dag_id)

    # now look for the task instances that are affected
    TI = TaskInstance

    # get all tasks of the main dag that will be affected by a state change
    qry_dag = session.query(TI).filter(
        TI.dag_id == dag.dag_id,
        TI.execution_date.in_(confirmed_dates),
        TI.task_id.in_(task_ids)).filter(
        or_(TI.state.is_(None),
            TI.state != state)
    )

    # get *all* tasks of the sub dags
    if len(sub_dag_ids) > 0:
        qry_sub_dag = session.query(TI).filter(
            TI.dag_id.in_(sub_dag_ids),
            TI.execution_date.in_(confirmed_dates)).filter(
            or_(TI.state.is_(None),
                TI.state != state)
        )

    if commit:
        tis_altered = qry_dag.with_for_update().all()
        if len(sub_dag_ids) > 0:
            tis_altered += qry_sub_dag.with_for_update().all()
        for ti in tis_altered:
            ti.state = state
    else:
        tis_altered = qry_dag.all()
        if len(sub_dag_ids) > 0:
            tis_altered += qry_sub_dag.all()

    return tis_altered

@provide_session
def _set_dag_run_state(dag_id, execution_date, state, session=None):
    """
    Helper method that set dag run state in the DB.
    :param dag_id: dag_id of target dag run
    :param execution_date: the execution date from which to start looking
    :param state: target state
    :param session: database session
    """
    DR = DagRun
    dr = session.query(DR).filter(
        DR.dag_id == dag_id,
        DR.execution_date == execution_date
    ).one()
    dr.state = state
    if state == State.RUNNING:
        dr.start_date = timezone.utcnow()
        dr.end_date = None
    else:
        dr.end_date = timezone.utcnow()
    session.merge(dr)

@provide_session
def set_dag_run_state_to_success(dag, execution_date, commit=False, session=None):
    """
    Set the dag run for a specific execution date and its task instances
    to success.
    :param dag: the DAG of which to alter state
    :param execution_date: the execution date from which to start looking
    :param commit: commit DAG and tasks to be altered to the database
    :param session: database session
    :return: If commit is true, list of tasks that have been updated,
             otherwise list of tasks that will be updated
    :raises: AssertionError if dag or execution_date is invalid
    """
    res = []

    if not dag or not execution_date:
        return res

    # Mark the dag run to success.
    if commit:
        _set_dag_run_state(dag.dag_id, execution_date, State.SUCCESS, session)

    # Mark all task instances of the dag run to success.
    for task in dag.tasks:
        task.dag = dag
        new_state = set_state(task=task, execution_date=execution_date,
                              state=State.SUCCESS, commit=commit)
        res.extend(new_state)

    return res

@provide_session
def set_dag_run_state_to_failed(dag, execution_date, commit=False, session=None):
    """
    Set the dag run for a specific execution date and its running task instances
    to failed.
    :param dag: the DAG of which to alter state
    :param execution_date: the execution date from which to start looking
    :param commit: commit DAG and tasks to be altered to the database
    :param session: database session
    :return: If commit is true, list of tasks that have been updated,
             otherwise list of tasks that will be updated
    :raises: AssertionError if dag or execution_date is invalid
    """
    res = []

    if not dag or not execution_date:
        return res

    # Mark the dag run to failed.
    if commit:
        _set_dag_run_state(dag.dag_id, execution_date, State.FAILED, session)

    # Mark only RUNNING task instances.
    TI = TaskInstance
    task_ids = [task.task_id for task in dag.tasks]
    tis = session.query(TI).filter(
        TI.dag_id == dag.dag_id,
        TI.execution_date == execution_date,
        TI.task_id.in_(task_ids)).filter(TI.state == State.RUNNING)
    task_ids_of_running_tis = [ti.task_id for ti in tis]
    for task in dag.tasks:
        if task.task_id not in task_ids_of_running_tis:
            continue
        task.dag = dag
        new_state = set_state(task=task, execution_date=execution_date,
                              state=State.FAILED, commit=commit)
        res.extend(new_state)

    return res

@provide_session
def set_dag_run_state_to_running(dag, execution_date, commit=False, session=None):
    """
    Set the dag run for a specific execution date to running.
    :param dag: the DAG of which to alter state
    :param execution_date: the execution date from which to start looking
    :param commit: commit DAG and tasks to be altered to the database
    :param session: database session
    :return: If commit is true, list of tasks that have been updated,
             otherwise list of tasks that will be updated
    """
    res = []
    if not dag or not execution_date:
        return res

    # Mark the dag run to running.
    if commit:
        _set_dag_run_state(dag.dag_id, execution_date, State.RUNNING, session)

    # To keep the return type consistent with the other similar functions.
    return res
<commit_msg>[AIRFLOW-3690] Fix bug to set state of a task for manually-triggered DAGs (#4504)<commit_after># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from sqlalchemy import or_

from airflow.jobs import BackfillJob
from airflow.models import DagRun, TaskInstance
from airflow.operators.subdag_operator import SubDagOperator
from airflow.utils import timezone
from airflow.utils.db import provide_session
from airflow.utils.state import State

def _create_dagruns(dag, execution_dates, state, run_id_template):
    """
    Infers from the dates which dag runs need to be created and does so.
    :param dag: the dag to create dag runs for
    :param execution_dates: list of execution dates to evaluate
    :param state: the state to set the dag run to
    :param run_id_template:the template for run id to be with the execution date
    :return: newly created and existing dag runs for the execution dates supplied
    """
    # find out if we need to create any dag runs
    drs = DagRun.find(dag_id=dag.dag_id, execution_date=execution_dates)
    dates_to_create = list(set(execution_dates) - set([dr.execution_date for dr in drs]))

    for date in dates_to_create:
        dr = dag.create_dagrun(
            run_id=run_id_template.format(date.isoformat()),
            execution_date=date,
            start_date=timezone.utcnow(),
            external_trigger=False,
            state=state,
        )
        drs.append(dr)

    return drs

@provide_session
def set_state(task, execution_date, upstream=False, downstream=False,
              future=False, past=False, state=State.SUCCESS, commit=False, session=None):
    """
    Set the state of a task instance and if needed its relatives. Can set state
    for future tasks (calculated from execution_date) and retroactively
    for past tasks. Will verify integrity of past dag runs in order to create
    tasks that did not exist. It will not create dag runs that are missing
    on the schedule (but it will as for subdag dag runs if needed).
    :param task: the task from which to work. task.task.dag needs to be set
    :param execution_date: the execution date from which to start looking
    :param upstream: Mark all parents (upstream tasks)
    :param downstream: Mark all siblings (downstream tasks) of task_id, including SubDags
    :param future: Mark all future tasks on the interval of the dag up until
        last execution date.
    :param past: Retroactively mark all tasks starting from start_date of the DAG
    :param state: State to which the tasks need to be set
    :param commit: Commit tasks to be altered to the database
    :param session: database session
    :return: list of tasks that have been created and updated
    """
    assert timezone.is_localized(execution_date)

    assert task.dag is not None
    dag = task.dag

    latest_execution_date = dag.latest_execution_date
    assert latest_execution_date is not None

    # determine date range of dag runs and tasks to consider
    end_date = latest_execution_date if future else execution_date

    if 'start_date' in dag.default_args:
        start_date = dag.default_args['start_date']
    elif dag.start_date:
        start_date = dag.start_date
    else:
        start_date = execution_date

    start_date = execution_date if not past else start_date

    if dag.schedule_interval == '@once':
        dates = [start_date]
    else:
        dates = dag.date_range(start_date=start_date, end_date=end_date)

    # find relatives (siblings = downstream, parents = upstream) if needed
    task_ids = [task.task_id]
    if downstream:
        relatives = task.get_flat_relatives(upstream=False)
        task_ids += [t.task_id for t in relatives]
    if upstream:
        relatives = task.get_flat_relatives(upstream=True)
        task_ids += [t.task_id for t in relatives]

    # verify the integrity of the dag runs in case a task was added or removed
    # set the confirmed execution dates as they might be different
    # from what was provided
    confirmed_dates = []
    drs = DagRun.find(dag_id=dag.dag_id, execution_date=dates)
    for dr in drs:
        dr.dag = dag
        dr.verify_integrity()
        confirmed_dates.append(dr.execution_date)

    # go through subdagoperators and create dag runs. We will only work
    # within the scope of the subdag. We wont propagate to the parent dag,
    # but we will propagate from parent to subdag.
    dags = [dag]
    sub_dag_ids = []
    while len(dags) > 0:
        current_dag = dags.pop()
        for task_id in task_ids:
            if not current_dag.has_task(task_id):
                continue

            current_task = current_dag.get_task(task_id)
            if isinstance(current_task, SubDagOperator):
                # this works as a kind of integrity check
                # it creates missing dag runs for subdagoperators,
                # maybe this should be moved to dagrun.verify_integrity
                drs = _create_dagruns(current_task.subdag,
                                      execution_dates=confirmed_dates,
                                      state=State.RUNNING,
                                      run_id_template=BackfillJob.ID_FORMAT_PREFIX)

                for dr in drs:
                    dr.dag = current_task.subdag
                    dr.verify_integrity()
                    if commit:
                        dr.state = state
                        session.merge(dr)

                dags.append(current_task.subdag)
                sub_dag_ids.append(current_task.subdag.dag_id)

    # now look for the task instances that are affected
    TI = TaskInstance

    # get all tasks of the main dag that will be affected by a state change
    qry_dag = session.query(TI).filter(
        TI.dag_id == dag.dag_id,
        TI.execution_date.in_(confirmed_dates),
        TI.task_id.in_(task_ids)).filter(
        or_(TI.state.is_(None),
            TI.state != state)
    )

    # get *all* tasks of the sub dags
    if len(sub_dag_ids) > 0:
        qry_sub_dag = session.query(TI).filter(
            TI.dag_id.in_(sub_dag_ids),
            TI.execution_date.in_(confirmed_dates)).filter(
            or_(TI.state.is_(None),
                TI.state != state)
        )

    if commit:
        tis_altered = qry_dag.with_for_update().all()
        if len(sub_dag_ids) > 0:
            tis_altered += qry_sub_dag.with_for_update().all()
        for ti in tis_altered:
            ti.state = state
    else:
        tis_altered = qry_dag.all()
        if len(sub_dag_ids) > 0:
            tis_altered += qry_sub_dag.all()

    return tis_altered

@provide_session
def _set_dag_run_state(dag_id, execution_date, state, session=None):
    """
    Helper method that set dag run state in the DB.
    :param dag_id: dag_id of target dag run
    :param execution_date: the execution date from which to start looking
    :param state: target state
    :param session: database session
    """
    DR = DagRun
    dr = session.query(DR).filter(
        DR.dag_id == dag_id,
        DR.execution_date == execution_date
    ).one()
    dr.state = state
    if state == State.RUNNING:
        dr.start_date = timezone.utcnow()
        dr.end_date = None
    else:
        dr.end_date = timezone.utcnow()
    session.merge(dr)

@provide_session
def set_dag_run_state_to_success(dag, execution_date, commit=False, session=None):
    """
    Set the dag run for a specific execution date and its task instances
    to success.
    :param dag: the DAG of which to alter state
    :param execution_date: the execution date from which to start looking
    :param commit: commit DAG and tasks to be altered to the database
    :param session: database session
    :return: If commit is true, list of tasks that have been updated,
             otherwise list of tasks that will be updated
    :raises: AssertionError if dag or execution_date is invalid
    """
    res = []

    if not dag or not execution_date:
        return res

    # Mark the dag run to success.
    if commit:
        _set_dag_run_state(dag.dag_id, execution_date, State.SUCCESS, session)

    # Mark all task instances of the dag run to success.
    for task in dag.tasks:
        task.dag = dag
        new_state = set_state(task=task, execution_date=execution_date,
                              state=State.SUCCESS, commit=commit)
        res.extend(new_state)

    return res

@provide_session
def set_dag_run_state_to_failed(dag, execution_date, commit=False, session=None):
    """
    Set the dag run for a specific execution date and its running task instances
    to failed.
    :param dag: the DAG of which to alter state
    :param execution_date: the execution date from which to start looking
    :param commit: commit DAG and tasks to be altered to the database
    :param session: database session
    :return: If commit is true, list of tasks that have been updated,
             otherwise list of tasks that will be updated
    :raises: AssertionError if dag or execution_date is invalid
    """
    res = []

    if not dag or not execution_date:
        return res

    # Mark the dag run to failed.
    if commit:
        _set_dag_run_state(dag.dag_id, execution_date, State.FAILED, session)

    # Mark only RUNNING task instances.
    TI = TaskInstance
    task_ids = [task.task_id for task in dag.tasks]
    tis = session.query(TI).filter(
        TI.dag_id == dag.dag_id,
        TI.execution_date == execution_date,
        TI.task_id.in_(task_ids)).filter(TI.state == State.RUNNING)
    task_ids_of_running_tis = [ti.task_id for ti in tis]
    for task in dag.tasks:
        if task.task_id not in task_ids_of_running_tis:
            continue
        task.dag = dag
        new_state = set_state(task=task, execution_date=execution_date,
                              state=State.FAILED, commit=commit)
        res.extend(new_state)

    return res

@provide_session
def set_dag_run_state_to_running(dag, execution_date, commit=False, session=None):
    """
    Set the dag run for a specific execution date to running.
    :param dag: the DAG of which to alter state
    :param execution_date: the execution date from which to start looking
    :param commit: commit DAG and tasks to be altered to the database
    :param session: database session
    :return: If commit is true, list of tasks that have been updated,
             otherwise list of tasks that will be updated
    """
    res = []
    if not dag or not execution_date:
        return res

    # Mark the dag run to running.
    if commit:
        _set_dag_run_state(dag.dag_id, execution_date, State.RUNNING, session)

    # To keep the return type consistent with the other similar functions.
    return res
<|endoftext|>


<commit_before>"""
Yahoo OpenId, OAuth1 and OAuth2 backends, docs at:
    https://python-social-auth.readthedocs.io/en/latest/backends/yahoo.html
"""
from requests.auth import HTTPBasicAuth

from ..utils import handle_http_errors
from .open_id import OpenIdAuth
from .oauth import BaseOAuth2, BaseOAuth1

class YahooOpenId(OpenIdAuth):
    """Yahoo OpenID authentication backend"""
    name = 'yahoo'
    URL = 'http://me.yahoo.com'

class YahooOAuth(BaseOAuth1):
    """Yahoo OAuth authentication backend. DEPRECATED"""
    name = 'yahoo-oauth'
    ID_KEY = 'guid'
    AUTHORIZATION_URL = 'https://api.login.yahoo.com/oauth/v2/request_auth'
    REQUEST_TOKEN_URL = \
        'https://api.login.yahoo.com/oauth/v2/get_request_token'
    ACCESS_TOKEN_URL = 'https://api.login.yahoo.com/oauth/v2/get_token'
    EXTRA_DATA = [
        ('guid', 'id'),
        ('access_token', 'access_token'),
        ('expires', 'expires')
    ]

    def get_user_details(self, response):
        """Return user details from Yahoo Profile"""
        fullname, first_name, last_name = self.get_user_names(
            first_name=response.get('givenName'),
            last_name=response.get('familyName')
        )
        emails = [email for email in response.get('emails', [])
                        if email.get('handle')]
        emails.sort(key=lambda e: e.get('primary', False), reverse=True)
        return {'username': response.get('nickname'),
                'email': emails[0]['handle'] if emails else '',
                'fullname': fullname,
                'first_name': first_name,
                'last_name': last_name}

    def user_data(self, access_token, *args, **kwargs):
        """Loads user data from service"""
        url = 'https://social.yahooapis.com/v1/user/{0}/profile?format=json'
        return self.get_json(
            url.format(self._get_guid(access_token)),
            auth=self.oauth_auth(access_token)
        )['profile']

    def _get_guid(self, access_token):
        """
        Beause you have to provide GUID for every API request it's also
        returned during one of OAuth calls
        """
        return self.get_json(
            'https://social.yahooapis.com/v1/me/guid?format=json',
            auth=self.oauth_auth(access_token)
        )['guid']['value']

class YahooOAuth2(BaseOAuth2):
    """Yahoo OAuth2 authentication backend"""
    name = 'yahoo-oauth2'
    ID_KEY = 'guid'
    AUTHORIZATION_URL = 'https://api.login.yahoo.com/oauth2/request_auth'
    ACCESS_TOKEN_URL = 'https://api.login.yahoo.com/oauth2/get_token'
    ACCESS_TOKEN_METHOD = 'POST'
    EXTRA_DATA = [
        ('xoauth_yahoo_guid', 'id'),
        ('access_token', 'access_token'),
        ('expires_in', 'expires'),
        ('refresh_token', 'refresh_token'),
        ('token_type', 'token_type'),
    ]

    def get_user_names(self, first_name, last_name):
        if first_name or last_name:
            return ' '.join((first_name, last_name)), first_name, last_name
        return None, None, None

    def get_user_details(self, response):
        """
        Return user details from Yahoo Profile.
        To Get user email you need the profile private read permission.
        """
        fullname, first_name, last_name = self.get_user_names(
            first_name=response.get('givenName'),
            last_name=response.get('familyName')
        )
        emails = [email for email in response.get('emails', [])
                        if 'handle' in email]
        emails.sort(key=lambda e: e.get('primary', False), reverse=True)
        email = emails[0]['handle'] if emails else response.get('guid', '')
        return {
            'username': response.get('nickname'),
            'email': email,
            'fullname': fullname,
            'first_name': first_name,
            'last_name': last_name
        }

    def user_data(self, access_token, *args, **kwargs):
        """Loads user data from service"""
        url = 'https://social.yahooapis.com/v1/user/{0}/profile?format=json' \
                .format(kwargs['response']['xoauth_yahoo_guid'])
        return self.get_json(url, headers={
            'Authorization': 'Bearer {0}'.format(access_token)
        }, method='GET')['profile']

    @handle_http_errors
    def auth_complete(self, *args, **kwargs):
        """Completes login process, must return user instance"""
        self.process_error(self.data)
        response = self.request_access_token(
            self.ACCESS_TOKEN_URL,
            auth=HTTPBasicAuth(*self.get_key_and_secret()),
            data=self.auth_complete_params(self.validate_state()),
            headers=self.auth_headers(),
            method=self.ACCESS_TOKEN_METHOD
        )
        self.process_error(response)
        return self.do_auth(response['access_token'], response=response,
                            *args, **kwargs)

    def refresh_token_params(self, token, *args, **kwargs):
        return {
            'refresh_token': token,
            'grant_type': 'refresh_token',
            'redirect_uri': 'oob',  # out of bounds
        }

    def refresh_token(self, token, *args, **kwargs):
        params = self.refresh_token_params(token, *args, **kwargs)
        url = self.REFRESH_TOKEN_URL or self.ACCESS_TOKEN_URL
        method = self.REFRESH_TOKEN_METHOD
        key = 'params' if method == 'GET' else 'data'
        request_args = {
            'headers': self.auth_headers(),
            'method': method,
            key: params
        }
        request = self.request(
            url,
            auth=HTTPBasicAuth(*self.get_key_and_secret()),
            **request_args
        )
        return self.process_refresh_token_response(request, *args, **kwargs)

    def auth_complete_params(self, state=None):
        return {
            'grant_type': 'authorization_code',  # request auth code
            'code': self.data.get('code', ''),  # server response code
            'redirect_uri': self.get_redirect_uri(state)
        }
<commit_msg>Drop YahooOpenId<commit_after>"""
Yahoo OpenId, OAuth1 and OAuth2 backends, docs at:
    https://python-social-auth.readthedocs.io/en/latest/backends/yahoo.html
"""
from requests.auth import HTTPBasicAuth

from ..utils import handle_http_errors
from .oauth import BaseOAuth2, BaseOAuth1

class YahooOAuth(BaseOAuth1):
    """Yahoo OAuth authentication backend. DEPRECATED"""
    name = 'yahoo-oauth'
    ID_KEY = 'guid'
    AUTHORIZATION_URL = 'https://api.login.yahoo.com/oauth/v2/request_auth'
    REQUEST_TOKEN_URL = \
        'https://api.login.yahoo.com/oauth/v2/get_request_token'
    ACCESS_TOKEN_URL = 'https://api.login.yahoo.com/oauth/v2/get_token'
    EXTRA_DATA = [
        ('guid', 'id'),
        ('access_token', 'access_token'),
        ('expires', 'expires')
    ]

    def get_user_details(self, response):
        """Return user details from Yahoo Profile"""
        fullname, first_name, last_name = self.get_user_names(
            first_name=response.get('givenName'),
            last_name=response.get('familyName')
        )
        emails = [email for email in response.get('emails', [])
                        if email.get('handle')]
        emails.sort(key=lambda e: e.get('primary', False), reverse=True)
        return {'username': response.get('nickname'),
                'email': emails[0]['handle'] if emails else '',
                'fullname': fullname,
                'first_name': first_name,
                'last_name': last_name}

    def user_data(self, access_token, *args, **kwargs):
        """Loads user data from service"""
        url = 'https://social.yahooapis.com/v1/user/{0}/profile?format=json'
        return self.get_json(
            url.format(self._get_guid(access_token)),
            auth=self.oauth_auth(access_token)
        )['profile']

    def _get_guid(self, access_token):
        """
        Beause you have to provide GUID for every API request it's also
        returned during one of OAuth calls
        """
        return self.get_json(
            'https://social.yahooapis.com/v1/me/guid?format=json',
            auth=self.oauth_auth(access_token)
        )['guid']['value']

class YahooOAuth2(BaseOAuth2):
    """Yahoo OAuth2 authentication backend"""
    name = 'yahoo-oauth2'
    ID_KEY = 'guid'
    AUTHORIZATION_URL = 'https://api.login.yahoo.com/oauth2/request_auth'
    ACCESS_TOKEN_URL = 'https://api.login.yahoo.com/oauth2/get_token'
    ACCESS_TOKEN_METHOD = 'POST'
    EXTRA_DATA = [
        ('xoauth_yahoo_guid', 'id'),
        ('access_token', 'access_token'),
        ('expires_in', 'expires'),
        ('refresh_token', 'refresh_token'),
        ('token_type', 'token_type'),
    ]

    def get_user_names(self, first_name, last_name):
        if first_name or last_name:
            return ' '.join((first_name, last_name)), first_name, last_name
        return None, None, None

    def get_user_details(self, response):
        """
        Return user details from Yahoo Profile.
        To Get user email you need the profile private read permission.
        """
        fullname, first_name, last_name = self.get_user_names(
            first_name=response.get('givenName'),
            last_name=response.get('familyName')
        )
        emails = [email for email in response.get('emails', [])
                        if 'handle' in email]
        emails.sort(key=lambda e: e.get('primary', False), reverse=True)
        email = emails[0]['handle'] if emails else response.get('guid', '')
        return {
            'username': response.get('nickname'),
            'email': email,
            'fullname': fullname,
            'first_name': first_name,
            'last_name': last_name
        }

    def user_data(self, access_token, *args, **kwargs):
        """Loads user data from service"""
        url = 'https://social.yahooapis.com/v1/user/{0}/profile?format=json' \
                .format(kwargs['response']['xoauth_yahoo_guid'])
        return self.get_json(url, headers={
            'Authorization': 'Bearer {0}'.format(access_token)
        }, method='GET')['profile']

    @handle_http_errors
    def auth_complete(self, *args, **kwargs):
        """Completes login process, must return user instance"""
        self.process_error(self.data)
        response = self.request_access_token(
            self.ACCESS_TOKEN_URL,
            auth=HTTPBasicAuth(*self.get_key_and_secret()),
            data=self.auth_complete_params(self.validate_state()),
            headers=self.auth_headers(),
            method=self.ACCESS_TOKEN_METHOD
        )
        self.process_error(response)
        return self.do_auth(response['access_token'], response=response,
                            *args, **kwargs)

    def refresh_token_params(self, token, *args, **kwargs):
        return {
            'refresh_token': token,
            'grant_type': 'refresh_token',
            'redirect_uri': 'oob',  # out of bounds
        }

    def refresh_token(self, token, *args, **kwargs):
        params = self.refresh_token_params(token, *args, **kwargs)
        url = self.REFRESH_TOKEN_URL or self.ACCESS_TOKEN_URL
        method = self.REFRESH_TOKEN_METHOD
        key = 'params' if method == 'GET' else 'data'
        request_args = {
            'headers': self.auth_headers(),
            'method': method,
            key: params
        }
        request = self.request(
            url,
            auth=HTTPBasicAuth(*self.get_key_and_secret()),
            **request_args
        )
        return self.process_refresh_token_response(request, *args, **kwargs)

    def auth_complete_params(self, state=None):
        return {
            'grant_type': 'authorization_code',  # request auth code
            'code': self.data.get('code', ''),  # server response code
            'redirect_uri': self.get_redirect_uri(state)
        }
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    gittime
    ~~~~~~~

    Estimate time spent programming, aided by git commit metadata. More
    documentation and stuff is in the readme.

    Example usage::

        python gittime.py git@github.com:uniphil/commit--blog.git

    For full usage details::

        python gittime.py --help

    :copyright: (c) 2014 by Phil Schleihauf.
    :licence: MPL, see LICENSE for more details.
"""

from __future__ import print_function

import shutil
from tempfile import mkdtemp
from datetime import datetime, timedelta

from pygit2 import clone_repository
from pygit2 import GIT_SORT_TIME, GIT_SORT_REVERSE

class TempRepo(object):
    """Make a bare repository in a directory for safe isolated inspection."""

    def __init__(self, url):
        self.url = url

    def __enter__(self):
        self.path = mkdtemp()
        try:
            print('cloning {}... '.format(self.url), end='\n\n')
            repo = clone_repository(self.url, self.path, bare=True)
        except Exception as e:
            self.__exit__()  # make sure the temp directory gets cleaned up
            raise e
        return repo

    def __exit__(self, *exc_args):
        shutil.rmtree(self.path, onerror=self.on_cleanup_error)

    def on_cleanup_error(self, raising_fn, path, exc):
        Warn('{:.__name__} failed for {}: {}'.format(raising_fn, path, exc))

class T(object):
    """A namespace for templating functions"""

    indentation = 2
    bullet_character = "*"

    @staticmethod
    def indent_line(line):
        template = "{spaces}{line}"
        spaces = " " * T.indentation
        return template.format(spaces=spaces, line=line)

    @staticmethod
    def indent(message):
        return '\n'.join(map(T.indent_line, message.splitlines()))

    @staticmethod
    def bullet(message):
        template = "{bullet} {firstline}\n{otherlines}"
        lines = message.splitlines()
        return template.format(bullet=T.bullet_character,
                               firstline=lines[0],
                               otherlines=T.indent('\n'.join(lines[1:])))

    @staticmethod
    def nice_time(time_obj):
        return time_obj.strftime('%A, %b %d at %X')

    @staticmethod
    def nice_timedelta(delta_obj):
        if delta_obj is None:
            return 'undefined'
        elif delta_obj < timedelta(seconds=48):
            return '{}s'.format(delta_obj.seconds)
        elif delta_obj < timedelta(minutes=48):
            minutes = float(delta_obj.seconds) / timedelta(minutes=1).seconds
            return '{:.0f}m'.format(minutes)
        elif delta_obj < timedelta(hours=22):
            hours = float(delta_obj.seconds) / timedelta(hours=1).seconds
            return '{:.1f}h'.format(hours)
        else:
            return '{}d'.format(delta_obj.days)

    @staticmethod
    def file_change(change):
        filename, plus, minus = change
        template = '+{plus} -{minus} {filename}'
        return template.format(plus=plus, minus=minus, filename=filename)

    @staticmethod
    def commit_summary(sha1, title, when, author, total_plus, total_minus,
                       changes_by_file, time_since):
        template = ("{sha1} {title}\n"
                    "{when} by {author}\n"
                    "Total line changes: +{plus} -{minus}\n"
                      "{changes_by_file}\n"
                    "Time since previous commit: {time_since}")
        changes = '\n'.join(map(T.file_change, changes_by_file))
        return template.format(sha1=sha1[:7],
                               title=title,
                               when=T.nice_time(when),
                               author=author,
                               plus=total_plus,
                               minus=total_minus,
                               changes_by_file=T.indent(changes),
                               time_since=T.nice_timedelta(time_since))

def get_changes(diff):
    total_adds = 0
    total_deletes = 0
    changes_by_file = []
    for patch in diff:
        total_adds += patch.additions
        total_deletes += patch.deletions
        change = (patch.new_file_path, patch.additions, patch.deletions)
        changes_by_file.append(change)
    return total_adds, total_deletes, changes_by_file

def summarize(repo, commit, previous=None):
    when = datetime.fromtimestamp(commit.commit_time)
    if previous is not None:
        prev_when = datetime.fromtimestamp(previous.commit_time)
        since = when - prev_when
        diff = previous.tree.diff_to_tree(commit.tree)
    else:
        since = None
        empty_tree_oid = repo.TreeBuilder().write()
        empty_tree = repo.get(empty_tree_oid)
        diff = empty_tree.diff_to_tree(commit.tree)

    changes = get_changes(diff)
    total_plus, total_minus, changes_by_file = changes

    summary = T.commit_summary(
        sha1=commit.hex,
        title=commit.message.splitlines()[0],
        when=when,
        author=commit.author.email,
        total_plus=total_plus,
        total_minus=total_minus,
        changes_by_file=changes_by_file,
        time_since=since,
    )
    return T.bullet(summary)

def estimate(repo):
    chronological = GIT_SORT_TIME|GIT_SORT_REVERSE
    for commit in repo.walk(repo.head.target, chronological):
        previous_commit = None if commit.parents == [] else commit.parents[0]
        summary = summarize(repo, commit, previous=previous_commit)
        print(summary, end='\n\n')

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='Estimate programming time.')
    parser.add_argument('url', help='repository clone URL or local path')
    args = parser.parse_args()
    with TempRepo(args.url) as repo:
        time_info = estimate(repo)
    print(time_info)
<commit_msg>use more descriptive variable names<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    gittime
    ~~~~~~~

    Estimate time spent programming, aided by git commit metadata. More
    documentation and stuff is in the readme.

    Example usage::

        python gittime.py git@github.com:uniphil/commit--blog.git

    For full usage details::

        python gittime.py --help

    :copyright: (c) 2014 by Phil Schleihauf.
    :licence: MPL, see LICENSE for more details.
"""

from __future__ import print_function

import shutil
from tempfile import mkdtemp
from datetime import datetime, timedelta

from pygit2 import clone_repository
from pygit2 import GIT_SORT_TIME, GIT_SORT_REVERSE

class TempRepo(object):
    """Make a bare repository in a directory for safe isolated inspection."""

    def __init__(self, url):
        self.url = url

    def __enter__(self):
        self.path = mkdtemp()
        try:
            print('cloning {}... '.format(self.url), end='\n\n')
            repo = clone_repository(self.url, self.path, bare=True)
        except Exception as e:
            self.__exit__()  # make sure the temp directory gets cleaned up
            raise e
        return repo

    def __exit__(self, *exc_args):
        shutil.rmtree(self.path, onerror=self.on_cleanup_error)

    def on_cleanup_error(self, raising_fn, path, exc):
        Warn('{:.__name__} failed for {}: {}'.format(raising_fn, path, exc))

class T(object):
    """A namespace for templating functions"""

    indentation = 2
    bullet_character = "*"

    @staticmethod
    def indent_line(line):
        template = "{spaces}{line}"
        spaces = " " * T.indentation
        return template.format(spaces=spaces, line=line)

    @staticmethod
    def indent(message):
        return '\n'.join(map(T.indent_line, message.splitlines()))

    @staticmethod
    def bullet(message):
        template = "{bullet} {firstline}\n{otherlines}"
        lines = message.splitlines()
        return template.format(bullet=T.bullet_character,
                               firstline=lines[0],
                               otherlines=T.indent('\n'.join(lines[1:])))

    @staticmethod
    def nice_time(time_obj):
        return time_obj.strftime('%A, %b %d at %X')

    @staticmethod
    def nice_timedelta(delta_obj):
        if delta_obj is None:
            return 'undefined'
        elif delta_obj < timedelta(seconds=48):
            return '{}s'.format(delta_obj.seconds)
        elif delta_obj < timedelta(minutes=48):
            minutes = float(delta_obj.seconds) / timedelta(minutes=1).seconds
            return '{:.0f}m'.format(minutes)
        elif delta_obj < timedelta(hours=22):
            hours = float(delta_obj.seconds) / timedelta(hours=1).seconds
            return '{:.1f}h'.format(hours)
        else:
            return '{}d'.format(delta_obj.days)

    @staticmethod
    def file_change(change):
        filename, plus, minus = change
        template = '+{plus} -{minus} {filename}'
        return template.format(plus=plus, minus=minus, filename=filename)

    @staticmethod
    def commit_summary(sha1, title, when, author, total_plus, total_minus,
                       changes_by_file, time_since):
        template = ("{sha1} {title}\n"
                    "{when} by {author}\n"
                    "Total line changes: +{plus} -{minus}\n"
                      "{changes_by_file}\n"
                    "Time since previous commit: {time_since}")
        changes = '\n'.join(map(T.file_change, changes_by_file))
        return template.format(sha1=sha1[:7],
                               title=title,
                               when=T.nice_time(when),
                               author=author,
                               plus=total_plus,
                               minus=total_minus,
                               changes_by_file=T.indent(changes),
                               time_since=T.nice_timedelta(time_since))

def get_changes(diff):
    total_adds = 0
    total_deletes = 0
    changes_by_file = []
    for patch in diff:
        total_adds += patch.additions
        total_deletes += patch.deletions
        change = (patch.new_file_path, patch.additions, patch.deletions)
        changes_by_file.append(change)
    return total_adds, total_deletes, changes_by_file

def summarize(repo, commit, previous=None):
    commit_time = datetime.fromtimestamp(commit.commit_time)
    if previous is not None:
        prev_commit_time = datetime.fromtimestamp(previous.commit_time)
        time_since_last_commit = commit_time - prev_commit_time
        diff = previous.tree.diff_to_tree(commit.tree)
    else:
        time_since_last_commit = None
        empty_tree_oid = repo.TreeBuilder().write()
        empty_tree = repo.get(empty_tree_oid)
        diff = empty_tree.diff_to_tree(commit.tree)

    changes = get_changes(diff)
    total_plus, total_minus, changes_by_file = changes

    summary = T.commit_summary(
        sha1=commit.hex,
        title=commit.message.splitlines()[0],
        when=commit_time,
        author=commit.author.email,
        total_plus=total_plus,
        total_minus=total_minus,
        changes_by_file=changes_by_file,
        time_since=time_since_last_commit,
    )
    return T.bullet(summary)

def estimate(repo):
    chronological = GIT_SORT_TIME|GIT_SORT_REVERSE
    for commit in repo.walk(repo.head.target, chronological):
        previous_commit = None if commit.parents == [] else commit.parents[0]
        summary = summarize(repo, commit, previous=previous_commit)
        print(summary, end='\n\n')

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='Estimate programming time.')
    parser.add_argument('url', help='repository clone URL or local path')
    args = parser.parse_args()
    with TempRepo(args.url) as repo:
        time_info = estimate(repo)
    print(time_info)
<|endoftext|>


<commit_before><commit_msg>Update client.py<commit_after>import socket
import time
import uuid
from _thread import *

HOST = '192.168.42.2'
PORT = 5555
UUID = uuid.uuid4().hex

client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

connected = False
registered = False
alarm_tripped = False
alarm_enabled = False
alarm_interval_seconds = 5

def parse_data(data: str):
    if data == "REG_COMPLETE":
        registered = True
    elif data == "ALRM_TRIP":
        pass
    elif data == "ALRM_STOP":
        pass
    elif data == "ALRM_ON":
        pass
    elif data == "ALRM_OFF":
        pass
    elif data == "CHNG_INTERVAL":
        pass
    elif data == "IS_ALIVE":
        socket_write("ACK", "")
    elif data == "UUID_REQ":
        socket_write(str(UUID), "UUID")

def socket_write(message: str, data_header: str):
    """
        return[0] = Client node UUID
        return[1] = data_header
        return[2] = data
    """
    message = str(UUID) + "," + data_header + "," + message
    client_socket.send(message.encode('ascii'))

def socket_is_alive():
    global connected
    try:
        while True:
            socket_write("IS_ALIVE", "")
            data = client_socket.recv(2048).decode('utf-8').strip()
            if data == "ACK":
                connected = True
            else:
                connected = False
            time.sleep(1)
    finally:
        client_socket.close()

def socket_read():
    try:
        while True:
            data = client_socket.recv(2048)
            data = data.decode('utf-8').strip().split(',')
            if (data[0] == UUID) or (data[0] == "BROADCAST"):
                return parse_data(data[1])
    finally:
        client_socket.close()

if __name__ == '__main__':
    try:
        client_socket.connect((HOST, PORT))
        connected = True
    except socket.error as e:
        print("Socket error {}".format(e))
        exit()
    finally:
        print("Successfully connect to IP:{}, PORT:{}".format(HOST, PORT))

    while connected:
        start_new_thread(socket_is_alive, ())
        start_new_thread(socket_read, ())
    else:
        print("Not connected anymore!")
<|endoftext|>


<commit_before>import requests
from urllib.parse import urlparse, urlencode

import json
import time
from jwt.jwt import JWT
from jwt.jwk import JWKSet

default_id_classes = {}

class OAuthException(RuntimeError):
    pass

class OAuthIdentity:
    name = "default"
    config = {}

    def __init__(self, config, status_code, data):
        """Constructor

        :param OAuthProviderConfig config: the OAuthProviderConfig which constructs this object
        :param dict data: the token returned by OAuth provider
        """

        if status_code not in [200, 400]:
            raise OAuthException('#http error', 'status code is {}'.format(status_code))

        if "error" in data:
            raise OAuthException(data["error"], data["error_description"])

        self.data = data
        self.config = config
        for attr in ["access_token", "expires_in", "token_type"]:
            if attr in self.data:
                setattr(self, attr, self.data[attr])
            else:
                setattr(self, attr, None)

        if "refresh_token" in self.data:
            self.refresh_token = self.data["refresh_token"]
        else:
            self.refresh_token = None

    @classmethod
    def register(cls, name=None, defconf=None, id_classes=default_id_classes):
        if name is None:
            name = cls.name
        if defconf is None:
            defconf = cls.config
        id_classes[name] = (cls, defconf)

OAuthIdentity.register()

class JWTBasedOAuthIdentity(OAuthIdentity):
    @classmethod
    def __refresh_keyset(cls):
        """Refresh __cached_keset with new value if the old one is expired.

        __cached_keset is JWT public keyset from the OAuth provider. The JWT
        keyset content is the return value of _fetch_keyset_cert(), which
        should be fetched through a secure channel.
        """

        try:
            cls.__cached_keyset
        except AttributeError:
            cls.__cached_keyset = None

        if cls.__cached_keyset is None or cls.__cached_keyset_exp < time.time():
            cert, exp = cls._fetch_keyset_cert()
            cls.__cached_keyset_exp = exp
            cls.__cached_keyset = JWKSet.from_dict(json.loads(cert))
            cls.jwt = JWT(cls.__cached_keyset)

    @staticmethod
    def _fetch_keyset_cert():
        raise NotImplementedError()

    def __init__(self, config, status_code, data):
        super().__init__(config, status_code, data)
        self.__refresh_keyset()
        self.id_token = json.loads(self.jwt.decode(data["id_token"]).decode("utf-8"))
        if self.id_token["aud"] != self.config.client_id:
            raise OAuthException("#audience not match",
                                 "Audience of the issued token is not this application")

class OAuthProviderConfig:
    def __init__(self, name, callback_url_prefix):
        self.name = name
        self.callback_url_prefix = callback_url_prefix
        self.authentication_endpoint = None
        self.token_endpoint = None
        self.client_id = None
        self.client_secret = None
        self.id_class = OAuthIdentity

    def authorization_url(self, scope, state):
        param = {'client_id': self.client_id,
                 'response_type': 'code',
                 'redirect_uri': self.callback_url_prefix + self.name,
                 'scope': scope,
                 'state': state}
        ret = self.authentication_endpoint
        if ret.find('?') == -1:
            ret += '?'
        else:
            ret += '&'
        ret += urlencode(param)
        return ret

    def get_token_by_code(self, authorization_code):
        """Fetch the OAuth token, the token is returned as an instance of
        self.id_class.

        Request the OAuth provider with code to get the token, then parse the
        token and use it to construct the id to return.

        :returns: an instance of self.id_class
        """

        data = {
            "code": authorization_code,
            "client_id": self.client_id,
            "client_secret": self.client_secret,
            "redirect_uri": self.callback_url_prefix + self.name,
            "grant_type": "authorization_code",
        }
        resp = requests.post(self.token_endpoint, data)
        return self.id_class(self, resp.status_code, json.loads(resp.text) if resp.status_code in [200, 400] else None)

    def get_token(self, request):
        """Fetch the OAuth token with the code in Pyramid request object.

        Conceptually: return self.get_token_by_code(request.params["code"]).
        It's the caller's responsibility to verify CSRF token.

        :returns: an instance of self.id_class
        """

        if "error" in request.params:
            raise Exception(request.params["error"])
        return self.get_token_by_code(request.params["code"])

    @staticmethod
    def directory_factory(request):
        return getattr(request.registry, '_oauth2_config', None)

def configure_oauth2(config, settings, id_classes=default_id_classes):
    """Configure Pyramid config object with settings

    Setting strings starting with ``oauth2.'' will be analyzed. And
    registry._oauth2_config will be generated as a dictionary. The part
    following ``oauth2.'' is the provider name, which is the key of
    registry._oauth2_config. The values in the dictionary are instances of
    OAuthProviderConfig. ``oauth2.<provider>.id_class'' will be looked up in id_classes.

    OAuthProviderConfig.factory is a simple factory returns registry._oauth2_config.

    Example:
        Settings::
            oauth2callback = http://mysite/oauth2callback/
            oauth2.google.id_class = google
            oauth2.google.client_id = XXX.apps.googleusercontent.com
            oauth2.google.client_secret = XXX

        Configuring code::
            config = Configurator(settings=settings)
            config.include('pyramid_bricks.oauth')

        Callback View code::
            @view_config(route_name = 'oauth2callback', context = OAuthProviderConfig)
            def oauth2callback(request):
                state = json.loads(request.params["state"])
                if request.session.get_csrf_token() != state['csrf_token']:
                    raise BadCSRFToken()
                requests.session.new_csrf_token()
                # context is a OAuthProviderConfig instance
                identity = request.context.get_token(request)
                # identity is a GoogleOAuthIdentity object
                ...
    """
    ocs = {}
    for name in settings:
        if name.startswith("oauth2."):
            comps = name.split('.')
            provider = comps[1]
            attribute = comps[2]
            value = settings[name]
            if attribute == "id_class":
                value, defconf = id_classes[value]
                conf = defconf.copy()
                if provider in ocs:
                    conf.update(ocs[provider])
                ocs[provider] = conf
            if provider not in ocs:
                ocs[provider] = {}
            ocs[provider][attribute] = value
    for provider in ocs:
        provider_config = OAuthProviderConfig(provider, settings['oauth2callback'])
        for attribute, value in ocs[provider].items():
            setattr(provider_config, attribute, value)
        ocs[provider] = provider_config
    setattr(config.registry, '_oauth2_config', ocs)
    oauth2callback = urlparse(settings['oauth2callback'])
    config.add_route('oauth2callback', oauth2callback.path + '*traverse',
                     factory=OAuthProviderConfig.directory_factory)

def oauth2_config(request):
    return getattr(request.registry, '_oauth2_config', None)

def includeme(config):
    configure_oauth2(config, config.registry.settings)

# vim: ts=4 sw=4 et nu
<commit_msg>get oauth registry by ZCA<commit_after>from urllib.parse import urlparse, urlencode
from jwt.jwt import JWT
from jwt.jwk import JWKSet
from zope.interface import implementer, Interface
import json
import time
import requests

default_id_classes = {}

class OAuthException(RuntimeError):
    pass

class OAuthIdentity:
    name = "default"
    config = {}

    def __init__(self, config, status_code, data):
        """Constructor

        :param OAuthProviderConfig config: the OAuthProviderConfig which constructs this object
        :param dict data: the token returned by OAuth provider
        """

        if status_code not in [200, 400]:
            raise OAuthException('#http error', 'status code is {}'.format(status_code))

        if "error" in data:
            raise OAuthException(data["error"], data["error_description"])

        self.data = data
        self.config = config
        for attr in ["access_token", "expires_in", "token_type"]:
            if attr in self.data:
                setattr(self, attr, self.data[attr])
            else:
                setattr(self, attr, None)

        if "refresh_token" in self.data:
            self.refresh_token = self.data["refresh_token"]
        else:
            self.refresh_token = None

    @classmethod
    def register(cls, name=None, defconf=None, id_classes=default_id_classes):
        if name is None:
            name = cls.name
        if defconf is None:
            defconf = cls.config
        id_classes[name] = (cls, defconf)

OAuthIdentity.register()

class JWTBasedOAuthIdentity(OAuthIdentity):
    @classmethod
    def __refresh_keyset(cls):
        """Refresh __cached_keset with new value if the old one is expired.

        __cached_keset is JWT public keyset from the OAuth provider. The JWT
        keyset content is the return value of _fetch_keyset_cert(), which
        should be fetched through a secure channel.
        """

        try:
            cls.__cached_keyset
        except AttributeError:
            cls.__cached_keyset = None

        if cls.__cached_keyset is None or cls.__cached_keyset_exp < time.time():
            cert, exp = cls._fetch_keyset_cert()
            cls.__cached_keyset_exp = exp
            cls.__cached_keyset = JWKSet.from_dict(json.loads(cert))
            cls.jwt = JWT(cls.__cached_keyset)

    @staticmethod
    def _fetch_keyset_cert():
        raise NotImplementedError()

    def __init__(self, config, status_code, data):
        super().__init__(config, status_code, data)
        self.__refresh_keyset()
        self.id_token = json.loads(self.jwt.decode(data["id_token"]).decode("utf-8"))
        if self.id_token["aud"] != self.config.client_id:
            raise OAuthException("#audience not match",
                                 "Audience of the issued token is not this application")

class OAuthProviderConfig:
    def __init__(self, name, callback_url_prefix):
        self.name = name
        self.callback_url_prefix = callback_url_prefix
        self.authentication_endpoint = None
        self.token_endpoint = None
        self.client_id = None
        self.client_secret = None
        self.id_class = OAuthIdentity

    def authorization_url(self, scope, state):
        param = {'client_id': self.client_id,
                 'response_type': 'code',
                 'redirect_uri': self.callback_url_prefix + self.name,
                 'scope': scope,
                 'state': state}
        ret = self.authentication_endpoint
        if ret.find('?') == -1:
            ret += '?'
        else:
            ret += '&'
        ret += urlencode(param)
        return ret

    def get_token_by_code(self, authorization_code):
        """Fetch the OAuth token, the token is returned as an instance of
        self.id_class.

        Request the OAuth provider with code to get the token, then parse the
        token and use it to construct the id to return.

        :returns: an instance of self.id_class
        """

        data = {
            "code": authorization_code,
            "client_id": self.client_id,
            "client_secret": self.client_secret,
            "redirect_uri": self.callback_url_prefix + self.name,
            "grant_type": "authorization_code",
        }
        resp = requests.post(self.token_endpoint, data)
        return self.id_class(self, resp.status_code, json.loads(resp.text) if resp.status_code in [200, 400] else None)

    def get_token(self, request):
        """Fetch the OAuth token with the code in Pyramid request object.

        Conceptually: return self.get_token_by_code(request.params["code"]).
        It's the caller's responsibility to verify CSRF token.

        :returns: an instance of self.id_class
        """

        if "error" in request.params:
            raise Exception(request.params["error"])
        return self.get_token_by_code(request.params["code"])

class IOAuthConfig(Interface):
    def add(self, provider_config: OAuthProviderConfig):
        pass

    def __setitem__(self, provider: str, provider_config: OAuthProviderConfig):
        pass

    def __getitem__(self, provider: str):
        pass

    def __contains__(self, provider: str):
        pass

@implementer(IOAuthConfig)
class OAuthConfig:
    def __init__(self):
        self._data = {}

    def add(self, provider_config: OAuthProviderConfig):
        self[provider_config.name] = provider_config

    def __setitem__(self, provider: str, provider_config: OAuthProviderConfig):
        self._data[provider] = provider_config

    def __getitem__(self, provider: str):
        return self._data[provider]

    def __contains__(self, provider: str):
        return provider in self._data

def configure_oauth2(config, settings, id_classes=default_id_classes):
    """Configure Pyramid config object with settings

    Example:
        Settings::
            oauth2callback = http://mysite/oauth2callback/
            oauth2.google.id_class = google
            oauth2.google.client_id = XXX.apps.googleusercontent.com
            oauth2.google.client_secret = XXX

        Configuring code::
            config = Configurator(settings=settings)
            config.include('pyramid_bricks.oauth')

        Callback View code::
            @view_config(route_name = 'oauth2callback', context = OAuthProviderConfig)
            def oauth2callback(request):
                state = json.loads(request.params["state"])
                if request.session.get_csrf_token() != state['csrf_token']:
                    raise BadCSRFToken()
                requests.session.new_csrf_token()
                # context is a OAuthProviderConfig instance
                identity = request.context.get_token(request)
                # identity is a GoogleOAuthIdentity object
                ...
    """
    ocs = {}
    for name in settings:
        if name.startswith("oauth2."):
            comps = name.split('.')
            provider = comps[1]
            attribute = comps[2]
            value = settings[name]
            if attribute == "id_class":
                value, defconf = id_classes[value]
                conf = defconf.copy()
                if provider in ocs:
                    conf.update(ocs[provider])
                ocs[provider] = conf
            if provider not in ocs:
                ocs[provider] = {}
            ocs[provider][attribute] = value

    oauth_config = OAuthConfig()
    for provider in ocs:
        provider_config = OAuthProviderConfig(provider, settings['oauth2callback'])
        for attribute, value in ocs[provider].items():
            setattr(provider_config, attribute, value)
        oauth_config.add(provider_config)

    def register_oauth_config():
        config.registry.registerUtility(oauth_config, IOAuthConfig)
    config.action('register_oauth_config', register_oauth_config)
    return oauth_config

def oauth2_config(request):
    return request.registry.getUtility(IOAuthConfig)

def includeme(config):
    settings = config.get_settings()
    oauth_config = configure_oauth2(config, settings)

    oauth2callback = urlparse(settings['oauth2callback'])
    config.add_route('oauth2callback', oauth2callback.path + '*traverse',
                     factory=lambda request: oauth_config)

# vim: ts=4 sw=4 et nu
<|endoftext|>


<commit_before>from django.conf.urls import patterns, url
from social_friends_finder.views import FriendListView
from django.contrib.auth.decorators import login_required

urlpatterns = patterns('social_friends_finder.views',
    url(r'^list/$', login_required(FriendListView.as_view()), name='friend_list'),
    url(r'^list/(?P<provider>[\w.@+-]+)/$', login_required(FriendListView.as_view()), name='friend_list'),
)
<commit_msg>urls import support for django 1.3<commit_after>from django.conf.urls import *
from social_friends_finder.views import FriendListView
from django.contrib.auth.decorators import login_required

urlpatterns = patterns('social_friends_finder.views',
    url(r'^list/$', login_required(FriendListView.as_view()), name='friend_list'),
    url(r'^list/(?P<provider>[\w.@+-]+)/$', login_required(FriendListView.as_view()), name='friend_list'),
)
<|endoftext|>


<commit_before>import os
import pytest
from pyroSAR.auxdata import dem_autoload, DEMHandler

from spatialist import bbox

def test_handler(auxdata_dem_cases):
    with bbox({'xmin': 11.5, 'xmax': 11.9, 'ymin': 51.1, 'ymax': 51.5}, crs=4326) as box:
        with DEMHandler([box]) as handler:
            for demType, reference in auxdata_dem_cases:
                result = handler.remote_ids(demType=demType, extent=box.extent)
                assert result == reference

    with bbox({'xmin': -11.9, 'xmax': -11.5, 'ymin': -51.5, 'ymax': -51.1}, crs=4326) as box:
        with DEMHandler([box]) as handler:
            cases = [('AW3D30', ['S055W015/S052W012.tar.gz']),
                     ('SRTM 1Sec HGT', ['S52W012.SRTMGL1.hgt.zip']),
                     ('SRTM 3Sec', ['srtm_34_23.zip']),
                     ('TDX90m', ['90mdem/DEM/S52/W010/TDM1_DEM__30_S52W012.zip'])]
            for demType, reference in cases:
                result = handler.remote_ids(demType=demType, extent=box.extent)
                assert result == reference

def test_autoload(auxdata_dem_cases, travis):
    # delete all target files to test downloading them again
    home = os.path.expanduser('~')
    demdir = os.path.join(home, '.snap', 'auxdata', 'dem')
    locals = [os.path.join(demdir, x, os.path.basename(y[0])) for x, y in auxdata_dem_cases]
    for item in locals:
        if os.path.isfile(item):
            os.remove(item)
    with bbox({'xmin': 11.5, 'xmax': 11.9, 'ymin': 51, 'ymax': 51.5}, crs=4326) as box:
        # if the following is run in a loop, it is not possible to see which demType failed
        # Travis CI does not support ftp access;
        # see https://blog.travis-ci.com/2018-07-23-the-tale-of-ftp-at-travis-ci
        if not travis:
            files = dem_autoload([box], 'AW3D30')
            assert len(files) == 1
            files = dem_autoload([box], 'AW3D30', product='stk')
            assert len(files) == 1
        files = dem_autoload([box], 'SRTM 1Sec HGT')
        assert len(files) == 1
        files = dem_autoload([box], 'SRTM 3Sec')
        assert len(files) == 1
        with pytest.raises(RuntimeError):
            files = dem_autoload([box], 'TDX90m')
        with pytest.raises(RuntimeError):
            dem_autoload([box], 'AW3D30', product='foobar')
<commit_msg>[test_auxdata] additional tests<commit_after>import os
import pytest
from pyroSAR.auxdata import dem_autoload, DEMHandler, dem_create

from spatialist import bbox

def test_handler(auxdata_dem_cases):
    with bbox({'xmin': 11.5, 'xmax': 11.9, 'ymin': 51.1, 'ymax': 51.5}, crs=4326) as box:
        with DEMHandler([box]) as handler:
            for demType, reference in auxdata_dem_cases:
                result = handler.remote_ids(demType=demType, extent=box.extent)
                assert result == reference

    with bbox({'xmin': -11.9, 'xmax': -11.5, 'ymin': -51.5, 'ymax': -51.1}, crs=4326) as box:
        with DEMHandler([box]) as handler:
            cases = [('AW3D30', ['S055W015/S052W012.tar.gz']),
                     ('SRTM 1Sec HGT', ['S52W012.SRTMGL1.hgt.zip']),
                     ('SRTM 3Sec', ['srtm_34_23.zip']),
                     ('TDX90m', ['90mdem/DEM/S52/W010/TDM1_DEM__30_S52W012.zip'])]
            for demType, reference in cases:
                result = handler.remote_ids(demType=demType, extent=box.extent)
                assert result == reference
    with pytest.raises(RuntimeError):
        test = DEMHandler('foobar')
    ext_utm = {'xmin': -955867, 'xmax': -915536, 'ymin': -5915518, 'ymax': -5863678}
    with bbox(ext_utm, crs=32632) as box:
        with pytest.raises(RuntimeError):
            test = DEMHandler([box])

def test_autoload(auxdata_dem_cases, travis):
    # delete all target files to test downloading them again
    home = os.path.expanduser('~')
    demdir = os.path.join(home, '.snap', 'auxdata', 'dem')
    locals = [os.path.join(demdir, x, os.path.basename(y[0])) for x, y in auxdata_dem_cases]
    for item in locals:
        if os.path.isfile(item):
            os.remove(item)
    with bbox({'xmin': 11.5, 'xmax': 11.9, 'ymin': 51, 'ymax': 51.5}, crs=4326) as box:
        # if the following is run in a loop, it is not possible to see which demType failed
        # Travis CI does not support ftp access;
        # see https://blog.travis-ci.com/2018-07-23-the-tale-of-ftp-at-travis-ci
        if not travis:
            files = dem_autoload([box], 'AW3D30')
            assert len(files) == 1
            files = dem_autoload([box], 'AW3D30', product='stk')
            assert len(files) == 1
        files = dem_autoload([box], 'SRTM 1Sec HGT')
        assert len(files) == 1
        files = dem_autoload([box], 'SRTM 3Sec')
        assert len(files) == 1
        with pytest.raises(RuntimeError):
            files = dem_autoload([box], 'TDX90m')
        with pytest.raises(RuntimeError):
            dem_autoload([box], 'AW3D30', product='foobar')

def test_dem_create(tmpdir):
    with bbox({'xmin': 11.5, 'xmax': 11.9, 'ymin': 51, 'ymax': 51.5}, crs=4326) as box:
        with pytest.raises(RuntimeError):
            files = dem_autoload([box], 'foobar')
        vrt = dem_autoload([box], 'SRTM 3Sec', vrt='/vsimem/test.vrt')
    out = os.path.join(str(tmpdir), 'srtm.tif')
    dem_create(src=vrt, dst=out, t_srs=32632, tr=(90, 90))
    assert os.path.isfile(out)
<|endoftext|>


<commit_before>from django.contrib import admin

from fo2.admin import intr_adm_site
from .models import RecordTracking, Dispositivos, RoloBipado, Painel, \
    PainelModulo, UsuarioPainelModulo, InformacaoModulo, Pop
from .forms import PainelModelForm, InformacaoModuloModelForm

class RecordTrackingAdmin(admin.ModelAdmin):
    list_display = ['time', 'table', 'record_id', 'iud', 'log', 'user']
    search_fields = ['table', 'log', 'user']
    ordering = ['time', 'table', 'record_id']
    fields = ['time', 'table', 'record_id', 'iud', 'log', 'user']
    readonly_fields = ['time', 'table', 'record_id', 'iud', 'log', 'user']

class DispositivosAdmin(admin.ModelAdmin):
    list_display = ['key', 'nome']
    search_fields = ['key', 'nome']
    ordering = ['nome']
    fields = ['key', 'nome']
    readonly_fields = ['key']

class RoloBipadoAdmin(admin.ModelAdmin):
    list_display = [
        'dispositivo', 'rolo', 'date', 'referencia', 'tamanho', 'cor']
    search_fields = ['rolo', 'date', 'referencia', 'tamanho', 'cor']
    ordering = ['referencia', 'tamanho', 'cor', 'rolo']
    fields = ['dispositivo', 'rolo', 'date', 'referencia', 'tamanho', 'cor']
    readonly_fields = [
        'dispositivo', 'rolo', 'date', 'referencia', 'tamanho', 'cor']

intr_adm_site.register(RecordTracking, RecordTrackingAdmin)

intr_adm_site.register(Dispositivos, DispositivosAdmin)

intr_adm_site.register(RoloBipado, RoloBipadoAdmin)

class PainelAdmin(admin.ModelAdmin):
    form = PainelModelForm
    list_display = ['nome', 'slug']
    search_fields = ['nome']
    ordering = ['nome']
    fields = ['nome', 'slug', 'layout']
    readonly_fields = ['slug']

class PainelModuloAdmin(admin.ModelAdmin):
    list_display = ['nome', 'slug', 'tipo']
    search_fields = ['nome']
    ordering = ['nome']
    fields = ['nome', 'slug', 'tipo']
    readonly_fields = ['slug']

class UsuarioPainelModuloAdmin(admin.ModelAdmin):
    list_display = ['usuario', 'painel_modulo']
    fields = ['usuario', 'painel_modulo']

class InformacaoModuloAdmin(admin.ModelAdmin):
    form = InformacaoModuloModelForm
    list_display = [
        'usuario', 'painel_modulo', 'data', 'chamada', 'habilitado']
    search_fields = ['chamada']
    ordering = ['data']
    fields = [
        'usuario', 'painel_modulo', 'data', 'chamada', 'texto', 'habilitado']
    readonly_fields = ['data']

intr_adm_site.register(Painel, PainelAdmin)
intr_adm_site.register(PainelModulo, PainelModuloAdmin)
intr_adm_site.register(UsuarioPainelModulo, UsuarioPainelModuloAdmin)
intr_adm_site.register(InformacaoModulo, InformacaoModuloAdmin)

class PopAdmin(admin.ModelAdmin):
    list_display = ['descricao']
    search_fields = ['descricao']
    ordering = ['descricao']
    fields = ['descricao', 'pop', 'habilitado', 'uploaded_at']
    readonly_fields = ['uploaded_at']

intr_adm_site.register(Pop, PopAdmin)
<commit_msg>reverse order<commit_after>from django.contrib import admin

from fo2.admin import intr_adm_site
from .models import RecordTracking, Dispositivos, RoloBipado, Painel, \
    PainelModulo, UsuarioPainelModulo, InformacaoModulo, Pop
from .forms import PainelModelForm, InformacaoModuloModelForm

class RecordTrackingAdmin(admin.ModelAdmin):
    list_display = ['time', 'table', 'record_id', 'iud', 'log', 'user']
    search_fields = ['table', 'log', 'user']
    ordering = ['-time', '-table', '-record_id']
    fields = ['time', 'table', 'record_id', 'iud', 'log', 'user']
    readonly_fields = ['time', 'table', 'record_id', 'iud', 'log', 'user']

class DispositivosAdmin(admin.ModelAdmin):
    list_display = ['key', 'nome']
    search_fields = ['key', 'nome']
    ordering = ['nome']
    fields = ['key', 'nome']
    readonly_fields = ['key']

class RoloBipadoAdmin(admin.ModelAdmin):
    list_display = [
        'dispositivo', 'rolo', 'date', 'referencia', 'tamanho', 'cor']
    search_fields = ['rolo', 'date', 'referencia', 'tamanho', 'cor']
    ordering = ['referencia', 'tamanho', 'cor', 'rolo']
    fields = ['dispositivo', 'rolo', 'date', 'referencia', 'tamanho', 'cor']
    readonly_fields = [
        'dispositivo', 'rolo', 'date', 'referencia', 'tamanho', 'cor']

intr_adm_site.register(RecordTracking, RecordTrackingAdmin)

intr_adm_site.register(Dispositivos, DispositivosAdmin)

intr_adm_site.register(RoloBipado, RoloBipadoAdmin)

class PainelAdmin(admin.ModelAdmin):
    form = PainelModelForm
    list_display = ['nome', 'slug']
    search_fields = ['nome']
    ordering = ['nome']
    fields = ['nome', 'slug', 'layout']
    readonly_fields = ['slug']

class PainelModuloAdmin(admin.ModelAdmin):
    list_display = ['nome', 'slug', 'tipo']
    search_fields = ['nome']
    ordering = ['nome']
    fields = ['nome', 'slug', 'tipo']
    readonly_fields = ['slug']

class UsuarioPainelModuloAdmin(admin.ModelAdmin):
    list_display = ['usuario', 'painel_modulo']
    fields = ['usuario', 'painel_modulo']

class InformacaoModuloAdmin(admin.ModelAdmin):
    form = InformacaoModuloModelForm
    list_display = [
        'usuario', 'painel_modulo', 'data', 'chamada', 'habilitado']
    search_fields = ['chamada']
    ordering = ['data']
    fields = [
        'usuario', 'painel_modulo', 'data', 'chamada', 'texto', 'habilitado']
    readonly_fields = ['data']

intr_adm_site.register(Painel, PainelAdmin)
intr_adm_site.register(PainelModulo, PainelModuloAdmin)
intr_adm_site.register(UsuarioPainelModulo, UsuarioPainelModuloAdmin)
intr_adm_site.register(InformacaoModulo, InformacaoModuloAdmin)

class PopAdmin(admin.ModelAdmin):
    list_display = ['descricao']
    search_fields = ['descricao']
    ordering = ['descricao']
    fields = ['descricao', 'pop', 'habilitado', 'uploaded_at']
    readonly_fields = ['uploaded_at']

intr_adm_site.register(Pop, PopAdmin)
<|endoftext|>