<commit_before>from maproulette import app
from flask.ext.restful import reqparse, fields, marshal, \
    marshal_with, Api, Resource
from flask.ext.restful.fields import Raw
from flask import session, make_response, request
from maproulette.helpers import *
from maproulette.models import Challenge, Task, TaskGeometry, Action, db
from geoalchemy2.functions import ST_Buffer
from shapely.geometry.base import BaseGeometry
from shapely.geometry import asShape
import geojson
import json
import markdown

class ProtectedResource(Resource):
    """A Resource that requires the caller to be authenticated against OSM"""
    method_decorators = [require_signedin]

class PointField(Raw):
    """An encoded point"""

    def format(self, geometry):
        # if we get a linestring, take the first point,
        # else, just get the point.
        point = geometry.coords[0]
        return '%f|%f' % point

class MarkdownField(Raw):
    """Markdown text"""

    def format(self, text):
        return markdown.markdown(text)

challenge_summary = {
    'slug': fields.String,
    'title': fields.String,
    'difficulty': fields.Integer,
    'description': fields.String,
    'help': MarkdownField,
    'blurb': fields.String,
    'islocal': fields.Boolean
}

task_fields = {
    'identifier': fields.String(attribute='identifier'),
    'instruction': fields.String(attribute='instruction'),
    'location': PointField
}

me_fields = {
    'username': fields.String(attribute='display_name'),
    'osm_id': fields.String()
}

action_fields = {
    'task': fields.String(attribute='task_id'),
    'timestamp': fields.DateTime,
    'status': fields.String,
    'user': fields.String(attribute='user_id')
}

api = Api(app)

# override the default JSON representation to support the geo objects

@api.representation('application/json')
def output_json(data, code, headers=None):
    """Automatic JSON / GeoJSON output"""
    app.logger.debug(data)
    # return empty result if data contains nothing
    if not data:
        resp = make_response(geojson.dumps({}), code)
    # if this is a Shapely object, dump it as geojson
    elif isinstance(data, BaseGeometry):
        resp = make_response(geojson.dumps(data), code)
    # if this is a list of task geometries, we need to unpack it
    elif not isinstance(data, dict) and isinstance(data[0], TaskGeometry):
        # unpack the geometries FIXME can this be done in the model?
        geometries = [geojson.Feature(
            geometry=g.geometry,
            properties={
                'selected': True,
                'osmid': g.osmid}) for g in data]
        resp = make_response(
            geojson.dumps(geojson.FeatureCollection(geometries)),
            code)
    # otherwise perform default json representation
    else:
        resp = make_response(json.dumps(data), code)
    # finish and return the response object
    resp.headers.extend(headers or {})
    return resp

class ApiPing(Resource):
    """a simple ping endpoint"""
    def get(self):
        return "I am alive"

class ApiGetAChallenge(ProtectedResource):
    @marshal_with(challenge_summary)
    def get(self):
        """Return a single challenge"""
        challenge = get_challenge_or_404(app.config["DEFAULT_CHALLENGE"], True)
        return challenge

class ApiChallengeList(ProtectedResource):
    """Challenge list endpoint"""

    @marshal_with(challenge_summary)
    def get(self):
        """returns a list of challenges.
        Optional URL parameters are:
        difficulty: the desired difficulty to filter on
        (1=easy, 2=medium, 3=hard)
        lon/lat: the coordinate to filter on (returns only
        challenges whose bounding polygons contain this point)
        example: /api/challenges?lon=-100.22&lat=40.45&difficulty=2
        all: if true, return all challenges regardless of
        OSM user home location
        """
        # initialize the parser
        parser = reqparse.RequestParser()
        parser.add_argument('difficulty', type=int, choices=["1", "2", "3"],
                            help='difficulty cannot be parsed')
        parser.add_argument('lon', type=float,
                            help="lon cannot be parsed")
        parser.add_argument('lat', type=float,
                            help="lat cannot be parsed")
        parser.add_argument('all', type=bool,
                            help="all cannot be parsed")
        args = parser.parse_args()

        difficulty = None
        contains = None

        # Try to get difficulty from argument, or users preference
        difficulty = args['difficulty'] or session.get('difficulty')

        # for local challenges, first look at lon / lat passed in
        if args.lon and args.lat:
            contains = 'POINT(%s %s)' % (args.lon, args.lat)
        # if there is none, look at the user's home location from OSM
        elif 'home_location' in session:
            contains = 'POINT(%s %s)' % tuple(session['home_location'])

        # get the list of challenges meeting the criteria
        query = db.session.query(Challenge).filter(Challenge.active is True)

        if difficulty:
            query = query.filter(Challenge.difficulty == difficulty)
        if contains and not args.all:
            query = query.filter(Challenge.polygon.ST_Contains(contains))

        challenges = query.all()

        return challenges

class ApiChallengeDetail(ProtectedResource):
    """Single Challenge endpoint"""

    def get(self, slug):
        """Return a single challenge by slug"""
        challenge = get_challenge_or_404(slug, True)
        return marshal(challenge, challenge.marshal_fields)

class ApiSelfInfo(ProtectedResource):
    """Information about the currently logged in user"""

    def get(self):
        """Return information about the logged in user"""
        return marshal(session, me_fields)

class ApiChallengePolygon(ProtectedResource):
    """Challenge geometry endpoint"""

    def get(self, slug):
        """Return the geometry (spatial extent)
        for the challenge identified by 'slug'"""
        challenge = get_challenge_or_404(slug, True)
        return challenge.polygon

class ApiChallengeStats(ProtectedResource):
    """Challenge Statistics endpoint"""

    def get(self, slug):
        """Return statistics for the challenge identified by 'slug'"""
        challenge = get_challenge_or_404(slug, True)
        total = len(challenge.tasks)
        # for task in Task.query.filter(Task.challenge_slug == slug):
        #    app.logger.debug(task.available)
        available = challenge.tasks_available
        return {'total': total, 'available': available}

class ApiChallengeTask(ProtectedResource):
    """Random Task endpoint"""

    def get(self, slug):
        """Returns a task for specified challenge"""
        challenge = get_challenge_or_404(slug, True)
        parser = reqparse.RequestParser()
        parser.add_argument('lon', type=float,
                            help='longitude could not be parsed')
        parser.add_argument('lat', type=float,
                            help='longitude could not be parsed')
        parser.add_argument('assign', type=int, default=1,
                            help='Assign could not be parsed')
        args = parser.parse_args()
        osmid = session.get('osm_id')
        assign = args['assign']
        lon = args['lon']
        lat = args['lat']

        app.logger.info(
            "{user} requesting task from {challenge} near\
             ({lon}, {lat}) assiging: {assign}".format(
            user=osmid,
            challenge=slug,
            lon=lon,
            lat=lat,
            assign=assign))

        task = None
        if lon and lat:
            coordWKT = 'POINT(%s %s)' % (lat, lon)
            task = Task.query.filter(Task.location.ST_Intersects(
                ST_Buffer(coordWKT, app.config["NEARBUFFER"]))).first()
        if task is None:  # we did not get a lon/lat or there was no task close
            # If no location is specified, or no tasks were found, gather
            # random tasks
            task = get_random_task(challenge)
            # If no tasks are found with this method, then this challenge
            # is complete
        if task is None:
            # Is this the right error?
            return osmerror("ChallengeComplete",
                            "Challenge {} is complete".format(slug))
        if assign:
            task.append_action(Action("assigned", osmid))
            db.session.add(task)

        db.session.commit()
        return marshal(task, task_fields)

class ApiChallengeTaskDetails(ProtectedResource):
    """Task details endpoint"""

    def get(self, slug, identifier):
        """Returns non-geo details for the task identified by
        'identifier' from the challenge identified by 'slug'"""
        task = get_task_or_404(slug, identifier)
        return marshal(task, task_fields)

    def post(self, slug, identifier):
        """Update the task identified by 'identifier' from
        the challenge identified by 'slug'"""
        app.logger.debug('updating task %s' % (identifier, ))
        # initialize the parser
        parser = reqparse.RequestParser()
        parser.add_argument('action', type=str,
                            help='action cannot be parsed')
        parser.add_argument('editor', type=str,
                            help="editor cannot be parsed")
        args = parser.parse_args()

        # get the task
        task = get_task_or_404(slug, identifier)
        # append the latest action to it.
        task.append_action(Action(args.action,
                                  session.get('osm_id'),
                                  args.editor))
        db.session.add(task)
        db.session.commit()
        return {}

class ApiChallengeTaskStatus(ProtectedResource):
    """Task status endpoint"""

    def get(self, slug, identifier):
        """Returns current status for the task identified by
        'identifier' from the challenge identified by 'slug'"""
        task = get_task_or_404(slug, identifier)
        app.logger.debug(task.currentaction)
        return {'status': task.currentaction}

class ApiChallengeTaskGeometries(ProtectedResource):
    """Task geometry endpoint"""

    def get(self, slug, identifier):
        """Returns the geometries for the task identified by
        'identifier' from the challenge identified by 'slug'"""
        task = get_task_or_404(slug, identifier)
        return task.geometries

# Add all resources to the RESTful API
api.add_resource(ApiPing, '/api/ping')
api.add_resource(ApiChallengeList, '/api/challenges')
api.add_resource(ApiChallengeDetail, '/api/challenge/<string:slug>')
api.add_resource(ApiChallengePolygon, '/api/challenge/<string:slug>/polygon')
api.add_resource(ApiChallengeStats, '/api/challenge/<string:slug>/stats')
api.add_resource(ApiChallengeTask, '/api/challenge/<slug>/task')
api.add_resource(ApiSelfInfo, '/api/me')
api.add_resource(ApiGetAChallenge, '/api/challenge')
api.add_resource(
    ApiChallengeTaskDetails,
    '/api/challenge/<slug>/task/<identifier>/')
api.add_resource(
    ApiChallengeTaskGeometries,
    '/api/challenge/<slug>/task/<identifier>/geometries/')
api.add_resource(
    ApiChallengeTaskStatus,
    '/api/challenge/<slug>/task/<identifier>/status/')

################################
# The Admin API ################
################################

class AdminApiChallenge(ProtectedResource):
    """Admin challenge creation endpoint"""
    def put(self, slug):
        if challenge_exists(slug):
            app.logger.debug('challenge exists')
            abort(403)
        try:
            payload = json.loads(request.data)
        except Exception:
            app.logger.debug('payload invalid, no json')
            abort(400)
        if not 'title' in payload:
            app.logger.debug('payload invalid, no title')
            abort(400)
        c = Challenge(
            slug,
            payload.get('title'),
            payload.get('geometry'),
            payload.get('description'),
            payload.get('blurb'),
            payload.get('help'),
            payload.get('instruction'),
            payload.get('active'),
            payload.get('difficulty'))
        db.session.add(c)
        db.session.commit()

    def delete(self, slug):
        """delete a challenge"""
        challenge = get_challenge_or_404(slug)
        db.session.delete(challenge)
        db.session.commit()

class AdminApiTaskStatuses(ProtectedResource):
    """Admin Task status endpoint"""

    def get(self, slug):
        """Return task statuses for the challenge identified by 'slug'"""
        challenge = get_challenge_or_404(slug, True)
        return [{
            'identifier': task.identifier,
            'status': task.currentaction} for task in challenge.tasks]

class AdminApiUpdateTask(ProtectedResource):
    """Challenge Task Statuses endpoint"""

    def put(self, slug, identifier):
        """Create or update one task. By default, the
        geometry must be supplied as WKB, but this can
        be overridden by adding ?geoformat=geojson to
        the URL"""

        task_geometries = []

        # Get the posted data
        taskdata = json.loads(request.data)

        exists = task_exists(slug, identifier)

        app.logger.debug("taskdata: %s" % (taskdata,))

        # abort if the taskdata does not contain geometries and it's a new task
        if not 'geometries' in taskdata:
            if not exists:
                abort(400)
        else:
            # extract the geometries
            geometries = taskdata.pop('geometries')
            app.logger.debug("geometries: %s" % (geometries,))
            app.logger.debug("features: %s" % (geometries['features'],))

            # parse the geometries
            for feature in geometries['features']:
                app.logger.debug(feature)
                osmid = feature['properties'].get('osmid')
                shape = asShape(feature['geometry'])
                t = TaskGeometry(osmid, shape)
                task_geometries.append(t)

        # there's two possible scenarios:
        # 1.    An existing task gets an update, in that case
        #       we only need the identifier
        # 2.    A new task is inserted, in this case we need at
        #       least an identifier and encoded geometries.

        # now we check if the task exists
        if exists:
            # if it does, update it
            app.logger.debug('existing task')
            task = get_task_or_404(slug, identifier)
            if not task.update(taskdata, task_geometries):
                abort(400)
        else:
            # if it does not, create it
            app.logger.debug('new task')
            new_task = Task(slug, identifier)
            new_task.update(taskdata, task_geometries)
        return {}

    def delete(self, slug, identifier):
        """Delete a task"""

        task = get_task_or_404(slug, identifier)
        task.append_action(Action('deleted'))
        db.session.add(task)
        db.session.commit()

api.add_resource(AdminApiChallenge, '/api/admin/challenge/<string:slug>')
api.add_resource(AdminApiTaskStatuses,
                 '/api/admin/challenge/<string:slug>/tasks')
api.add_resource(AdminApiUpdateTask,
                 '/api/admin/challenge/<string:slug>/task/<string:identifier>')
<commit_msg>cleaning up imports and removing trailing slashes from endpoints<commit_after>from maproulette import app
from flask.ext.restful import reqparse, fields, marshal, \
    marshal_with, Api, Resource
from flask.ext.restful.fields import Raw
from flask import session, make_response, request, abort
from maproulette.helpers import get_random_task,\
    get_challenge_or_404, get_task_or_404,\
    require_signedin, osmerror, challenge_exists,\
    task_exists
from maproulette.models import Challenge, Task, TaskGeometry, Action, db
from geoalchemy2.functions import ST_Buffer
from shapely.geometry.base import BaseGeometry
from shapely.geometry import asShape
import geojson
import json
import markdown

class ProtectedResource(Resource):
    """A Resource that requires the caller to be authenticated against OSM"""
    method_decorators = [require_signedin]

class PointField(Raw):
    """An encoded point"""

    def format(self, geometry):
        # if we get a linestring, take the first point,
        # else, just get the point.
        point = geometry.coords[0]
        return '%f|%f' % point

class MarkdownField(Raw):
    """Markdown text"""

    def format(self, text):
        return markdown.markdown(text)

challenge_summary = {
    'slug': fields.String,
    'title': fields.String,
    'difficulty': fields.Integer,
    'description': fields.String,
    'help': MarkdownField,
    'blurb': fields.String,
    'islocal': fields.Boolean
}

task_fields = {
    'identifier': fields.String(attribute='identifier'),
    'instruction': fields.String(attribute='instruction'),
    'location': PointField
}

me_fields = {
    'username': fields.String(attribute='display_name'),
    'osm_id': fields.String()
}

action_fields = {
    'task': fields.String(attribute='task_id'),
    'timestamp': fields.DateTime,
    'status': fields.String,
    'user': fields.String(attribute='user_id')
}

api = Api(app)

# override the default JSON representation to support the geo objects

@api.representation('application/json')
def output_json(data, code, headers=None):
    """Automatic JSON / GeoJSON output"""
    app.logger.debug(data)
    # return empty result if data contains nothing
    if not data:
        resp = make_response(geojson.dumps({}), code)
    # if this is a Shapely object, dump it as geojson
    elif isinstance(data, BaseGeometry):
        resp = make_response(geojson.dumps(data), code)
    # if this is a list of task geometries, we need to unpack it
    elif not isinstance(data, dict) and isinstance(data[0], TaskGeometry):
        # unpack the geometries FIXME can this be done in the model?
        geometries = [geojson.Feature(
            geometry=g.geometry,
            properties={
                'selected': True,
                'osmid': g.osmid}) for g in data]
        resp = make_response(
            geojson.dumps(geojson.FeatureCollection(geometries)),
            code)
    # otherwise perform default json representation
    else:
        resp = make_response(json.dumps(data), code)
    # finish and return the response object
    resp.headers.extend(headers or {})
    return resp

class ApiPing(Resource):
    """a simple ping endpoint"""
    def get(self):
        return "I am alive"

class ApiGetAChallenge(ProtectedResource):
    @marshal_with(challenge_summary)
    def get(self):
        """Return a single challenge"""
        challenge = get_challenge_or_404(app.config["DEFAULT_CHALLENGE"], True)
        return challenge

class ApiChallengeList(ProtectedResource):
    """Challenge list endpoint"""

    @marshal_with(challenge_summary)
    def get(self):
        """returns a list of challenges.
        Optional URL parameters are:
        difficulty: the desired difficulty to filter on
        (1=easy, 2=medium, 3=hard)
        lon/lat: the coordinate to filter on (returns only
        challenges whose bounding polygons contain this point)
        example: /api/challenges?lon=-100.22&lat=40.45&difficulty=2
        all: if true, return all challenges regardless of
        OSM user home location
        """
        # initialize the parser
        parser = reqparse.RequestParser()
        parser.add_argument('difficulty', type=int, choices=["1", "2", "3"],
                            help='difficulty cannot be parsed')
        parser.add_argument('lon', type=float,
                            help="lon cannot be parsed")
        parser.add_argument('lat', type=float,
                            help="lat cannot be parsed")
        parser.add_argument('all', type=bool,
                            help="all cannot be parsed")
        args = parser.parse_args()

        difficulty = None
        contains = None

        # Try to get difficulty from argument, or users preference
        difficulty = args['difficulty'] or session.get('difficulty')

        # for local challenges, first look at lon / lat passed in
        if args.lon and args.lat:
            contains = 'POINT(%s %s)' % (args.lon, args.lat)
        # if there is none, look at the user's home location from OSM
        elif 'home_location' in session:
            contains = 'POINT(%s %s)' % tuple(session['home_location'])

        # get the list of challenges meeting the criteria
        query = db.session.query(Challenge).filter(Challenge.active is True)

        if difficulty:
            query = query.filter(Challenge.difficulty == difficulty)
        if contains and not args.all:
            query = query.filter(Challenge.polygon.ST_Contains(contains))

        challenges = query.all()

        return challenges

class ApiChallengeDetail(ProtectedResource):
    """Single Challenge endpoint"""

    def get(self, slug):
        """Return a single challenge by slug"""
        challenge = get_challenge_or_404(slug, True)
        return marshal(challenge, challenge.marshal_fields)

class ApiSelfInfo(ProtectedResource):
    """Information about the currently logged in user"""

    def get(self):
        """Return information about the logged in user"""
        return marshal(session, me_fields)

class ApiChallengePolygon(ProtectedResource):
    """Challenge geometry endpoint"""

    def get(self, slug):
        """Return the geometry (spatial extent)
        for the challenge identified by 'slug'"""
        challenge = get_challenge_or_404(slug, True)
        return challenge.polygon

class ApiChallengeStats(ProtectedResource):
    """Challenge Statistics endpoint"""

    def get(self, slug):
        """Return statistics for the challenge identified by 'slug'"""
        challenge = get_challenge_or_404(slug, True)
        total = len(challenge.tasks)
        # for task in Task.query.filter(Task.challenge_slug == slug):
        #    app.logger.debug(task.available)
        available = challenge.tasks_available
        return {'total': total, 'available': available}

class ApiChallengeTask(ProtectedResource):
    """Random Task endpoint"""

    def get(self, slug):
        """Returns a task for specified challenge"""
        challenge = get_challenge_or_404(slug, True)
        parser = reqparse.RequestParser()
        parser.add_argument('lon', type=float,
                            help='longitude could not be parsed')
        parser.add_argument('lat', type=float,
                            help='longitude could not be parsed')
        parser.add_argument('assign', type=int, default=1,
                            help='Assign could not be parsed')
        args = parser.parse_args()
        osmid = session.get('osm_id')
        assign = args['assign']
        lon = args['lon']
        lat = args['lat']

        app.logger.info(
            "{user} requesting task from {challenge} near\
             ({lon}, {lat}) assiging: {assign}".format(
            user=osmid,
            challenge=slug,
            lon=lon,
            lat=lat,
            assign=assign))

        task = None
        if lon and lat:
            coordWKT = 'POINT(%s %s)' % (lat, lon)
            task = Task.query.filter(Task.location.ST_Intersects(
                ST_Buffer(coordWKT, app.config["NEARBUFFER"]))).first()
        if task is None:  # we did not get a lon/lat or there was no task close
            # If no location is specified, or no tasks were found, gather
            # random tasks
            task = get_random_task(challenge)
            # If no tasks are found with this method, then this challenge
            # is complete
        if task is None:
            # Is this the right error?
            return osmerror("ChallengeComplete",
                            "Challenge {} is complete".format(slug))
        if assign:
            task.append_action(Action("assigned", osmid))
            db.session.add(task)

        db.session.commit()
        return marshal(task, task_fields)

class ApiChallengeTaskDetails(ProtectedResource):
    """Task details endpoint"""

    def get(self, slug, identifier):
        """Returns non-geo details for the task identified by
        'identifier' from the challenge identified by 'slug'"""
        task = get_task_or_404(slug, identifier)
        return marshal(task, task_fields)

    def post(self, slug, identifier):
        """Update the task identified by 'identifier' from
        the challenge identified by 'slug'"""
        app.logger.debug('updating task %s' % (identifier, ))
        # initialize the parser
        parser = reqparse.RequestParser()
        parser.add_argument('action', type=str,
                            help='action cannot be parsed')
        parser.add_argument('editor', type=str,
                            help="editor cannot be parsed")
        args = parser.parse_args()

        # get the task
        task = get_task_or_404(slug, identifier)
        # append the latest action to it.
        task.append_action(Action(args.action,
                                  session.get('osm_id'),
                                  args.editor))
        db.session.add(task)
        db.session.commit()
        return {}

class ApiChallengeTaskStatus(ProtectedResource):
    """Task status endpoint"""

    def get(self, slug, identifier):
        """Returns current status for the task identified by
        'identifier' from the challenge identified by 'slug'"""
        task = get_task_or_404(slug, identifier)
        app.logger.debug(task.currentaction)
        return {'status': task.currentaction}

class ApiChallengeTaskGeometries(ProtectedResource):
    """Task geometry endpoint"""

    def get(self, slug, identifier):
        """Returns the geometries for the task identified by
        'identifier' from the challenge identified by 'slug'"""
        task = get_task_or_404(slug, identifier)
        return task.geometries

# Add all resources to the RESTful API
api.add_resource(ApiPing, '/api/ping')
api.add_resource(ApiChallengeList, '/api/challenges')
api.add_resource(ApiChallengeDetail, '/api/challenge/<string:slug>')
api.add_resource(ApiChallengePolygon, '/api/challenge/<string:slug>/polygon')
api.add_resource(ApiChallengeStats, '/api/challenge/<string:slug>/stats')
api.add_resource(ApiChallengeTask, '/api/challenge/<slug>/task')
api.add_resource(ApiSelfInfo, '/api/me')
api.add_resource(ApiGetAChallenge, '/api/challenge')
api.add_resource(
    ApiChallengeTaskDetails,
    '/api/challenge/<slug>/task/<identifier>')
api.add_resource(
    ApiChallengeTaskGeometries,
    '/api/challenge/<slug>/task/<identifier>/geometries')
api.add_resource(
    ApiChallengeTaskStatus,
    '/api/challenge/<slug>/task/<identifier>/status')

################################
# The Admin API ################
################################

class AdminApiChallenge(ProtectedResource):
    """Admin challenge creation endpoint"""
    def put(self, slug):
        if challenge_exists(slug):
            app.logger.debug('challenge exists')
            abort(403)
        try:
            payload = json.loads(request.data)
        except Exception:
            app.logger.debug('payload invalid, no json')
            abort(400)
        if not 'title' in payload:
            app.logger.debug('payload invalid, no title')
            abort(400)
        c = Challenge(
            slug,
            payload.get('title'),
            payload.get('geometry'),
            payload.get('description'),
            payload.get('blurb'),
            payload.get('help'),
            payload.get('instruction'),
            payload.get('active'),
            payload.get('difficulty'))
        db.session.add(c)
        db.session.commit()

    def delete(self, slug):
        """delete a challenge"""
        challenge = get_challenge_or_404(slug)
        db.session.delete(challenge)
        db.session.commit()

class AdminApiTaskStatuses(ProtectedResource):
    """Admin Task status endpoint"""

    def get(self, slug):
        """Return task statuses for the challenge identified by 'slug'"""
        challenge = get_challenge_or_404(slug, True)
        return [{
            'identifier': task.identifier,
            'status': task.currentaction} for task in challenge.tasks]

class AdminApiUpdateTask(ProtectedResource):
    """Challenge Task Statuses endpoint"""

    def put(self, slug, identifier):
        """Create or update one task. By default, the
        geometry must be supplied as WKB, but this can
        be overridden by adding ?geoformat=geojson to
        the URL"""

        task_geometries = []

        # Get the posted data
        taskdata = json.loads(request.data)

        exists = task_exists(slug, identifier)

        app.logger.debug("taskdata: %s" % (taskdata,))

        # abort if the taskdata does not contain geometries and it's a new task
        if not 'geometries' in taskdata:
            if not exists:
                abort(400)
        else:
            # extract the geometries
            geometries = taskdata.pop('geometries')
            app.logger.debug("geometries: %s" % (geometries,))
            app.logger.debug("features: %s" % (geometries['features'],))

            # parse the geometries
            for feature in geometries['features']:
                app.logger.debug(feature)
                osmid = feature['properties'].get('osmid')
                shape = asShape(feature['geometry'])
                t = TaskGeometry(osmid, shape)
                task_geometries.append(t)

        # there's two possible scenarios:
        # 1.    An existing task gets an update, in that case
        #       we only need the identifier
        # 2.    A new task is inserted, in this case we need at
        #       least an identifier and encoded geometries.

        # now we check if the task exists
        if exists:
            # if it does, update it
            app.logger.debug('existing task')
            task = get_task_or_404(slug, identifier)
            if not task.update(taskdata, task_geometries):
                abort(400)
        else:
            # if it does not, create it
            app.logger.debug('new task')
            new_task = Task(slug, identifier)
            new_task.update(taskdata, task_geometries)
        return {}

    def delete(self, slug, identifier):
        """Delete a task"""

        task = get_task_or_404(slug, identifier)
        task.append_action(Action('deleted'))
        db.session.add(task)
        db.session.commit()

api.add_resource(AdminApiChallenge, '/api/admin/challenge/<string:slug>')
api.add_resource(AdminApiTaskStatuses,
                 '/api/admin/challenge/<string:slug>/tasks')
api.add_resource(AdminApiUpdateTask,
                 '/api/admin/challenge/<string:slug>/task/<string:identifier>')
<|endoftext|>


<commit_before>import unittest
import os
import subprocess

from tempfile import mkstemp
from time import sleep

class DaemonizeTest(unittest.TestCase):
    def setUp(self):
        self.pidfile = mkstemp()[1]
        os.system("python tests/daemon_sigterm.py %s" % self.pidfile)
        sleep(.1)

    def tearDown(self):
        os.system("kill `cat %s`" % self.pidfile)
        sleep(.1)

    def test_is_working(self):
        sleep(10)
        proc = subprocess.Popen("ps ax | awk '{print $1}' | grep `cat %s`" \
                                % self.pidfile, shell=True,
                                stdout=subprocess.PIPE)
        ps_pid = proc.communicate()[0]
        pidfile = open(self.pidfile, "r")
        pid = pidfile.read()
        pidfile.close()
        self.assertEqual("%s\n" % pid, ps_pid)

    def test_pidfile_presense(self):
        sleep(10)
        self.assertTrue(os.path.isfile(self.pidfile))

class KeepFDsTest(unittest.TestCase):
    def setUp(self):
        self.pidfile = mkstemp()[1]
        self.logfile = mkstemp()[1]
        os.system("python tests/daemon_keep_fds.py %s %s" % (self.pidfile, self.logfile))
        sleep(.1)

    def tearDown(self):
        os.system("kill `cat %s`" % self.pidfile)
        os.remove(self.logfile)
        os.remove(self.pidfile)
        sleep(.1)

    def test_keep_fds(self):
        log = open(self.logfile, "r").read()
        self.assertEqual(log, "Test\n")

if __name__ == '__main__':
    unittest.main()
<commit_msg>Maybe travis ci is not enough fast<commit_after>import unittest
import os
import subprocess

from tempfile import mkstemp
from time import sleep

class DaemonizeTest(unittest.TestCase):
    def setUp(self):
        self.pidfile = mkstemp()[1]
        os.system("python tests/daemon_sigterm.py %s" % self.pidfile)
        sleep(.1)

    def tearDown(self):
        os.system("kill `cat %s`" % self.pidfile)
        sleep(.1)

    def test_is_working(self):
        sleep(10)
        proc = subprocess.Popen("ps ax | awk '{print $1}' | grep `cat %s`" \
                                % self.pidfile, shell=True,
                                stdout=subprocess.PIPE)
        ps_pid = proc.communicate()[0]
        pidfile = open(self.pidfile, "r")
        pid = pidfile.read()
        pidfile.close()
        self.assertEqual("%s\n" % pid, ps_pid)

    def test_pidfile_presense(self):
        sleep(10)
        self.assertTrue(os.path.isfile(self.pidfile))

class KeepFDsTest(unittest.TestCase):
    def setUp(self):
        self.pidfile = mkstemp()[1]
        self.logfile = mkstemp()[1]
        os.system("python tests/daemon_keep_fds.py %s %s" % (self.pidfile, self.logfile))
        sleep(1)

    def tearDown(self):
        os.system("kill `cat %s`" % self.pidfile)
        os.remove(self.logfile)
        os.remove(self.pidfile)
        sleep(.1)

    def test_keep_fds(self):
        log = open(self.logfile, "r").read()
        self.assertEqual(log, "Test\n")

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import socket
import struct
import select
import os
import hashlib

# we import PollingObserver instead of Observer because the deleted event
# is not capturing https://github.com/gorakhargosh/watchdog/issues/46
from watchdog.observers.polling import PollingObserver as Observer
from watchdog.events import FileSystemEventHandler

import connection_manager

class DirectoryMonitor(FileSystemEventHandler):
    """
    The DirectoryMonitor for file system events
    like: moved, deleted, created, modified
    """
    def __init__(self, folder_path, event_dispatcher, client_state):
        FileSystemEventHandler.__init__(self)
        self.client_state = client_state
        self.event_dispatcher = event_dispatcher
        self.folder_path = folder_path
        self.folder_watched = self.folder_path.split(os.sep)[-1]
        self.observer = Observer()
        self.observer.schedule(self, path=folder_path, recursive=True)

    def on_any_event(self, event):
        """
        it catpures any filesytem event and redirects it to the event_dispatcher
        """
        def build_data(cmd,e):
            data = {}
            data['cmd'] = cmd
            data['file'] = {
                "filepath": self.relativize_path(e.src_path),
                "mtime": os.path.getmtime(e.src_path),
                "md5": hashlib.md5(e.src_path).hexdigest()
            } # TODO update struct with new implemantation data = {<md5> : <filepath>}
            return data

        if event.is_directory is False:
            e = event

            if e.event_type == 'modified':
                data = build_data('modify', e)
                self.event_dispatcher(data['cmd'], data['file'])

            elif e.event_type == 'created':
                data = build_data('upload', e)
                if data['file']['filepath'] not in self.client_state:
                    self.event_dispatcher(data['cmd'], data['file'])
                else:
                    print 'FILE ESISTENTE, TODO VERIFICA md5 INVECE DEL path'

            elif e.event_type == 'moved':
                data = {'cmd':'move'}
                data['file'] = {
                    'src_path': self.relativize_path(e.src_path),
                    'dest_path': self.relativize_path(e.dest_path)}
                self.event_dispatcher(data['cmd'], data['file'])

            elif e.event_type == 'deleted':
                data = {'cmd':'delete'}
                data['file'] = {
                    "filepath": self.relativize_path(e.src_path)}
                self.event_dispatcher(data['cmd'], data['file'])

    def relativize_path(self,path_to_clean):
        """
        This function relativize the path watched by watchdog:
        for example: /home/user/watched/subfolder/ will be subfolder/
        """
        cleaned_path = path_to_clean.split(self.folder_watched)[-1]
        # cleaned from first slash character
        return cleaned_path[1:]

    def start(self):
        """
        starts the observer thread
        """
        self.observer.start()

    def stop(self):
        """
        stops the observer thread
        """
        self.observer.stop()

    def join(self):
        """
        waits for observer execution finalize
        """
        self.observer.join()

class Daemon(object):
    """
    Root of all evil:
    it loads program configurations,
    it starts directory events observations
    it serves command manager
    """

    TIMEOUT = 0.5

    def __init__(self):
        self.cfg = load_json('config.json')
        if not self.cfg:
            print "No config File!"
            exit()
        self.client_state = {}
        self.update_client_state()
        self.dir_manager = DirectoryMonitor(self.cfg['sharing_path'], self.event_dispatcher, self.client_state)
        self.conn_mng = connection_manager.ConnectionManager(self.cfg)
        self.sync_with_server()
        self.running = 0

    def relativize_path(self,path_to_clean):
        """
        This function relativize the path watched by watchdog:
        for example: /home/user/watched/subfolder/ will be subfolder/
        """
        folder_watched = self.cfg['sharing_path'].split(os.sep)[-1]
        cleaned_path = path_to_clean.split(folder_watched)[-1]
        # cleaned from first slash character
        return cleaned_path[1:]

    def update_client_state(self):
            for dirpath, dirs, files in os.walk(self.cfg['sharing_path']):
                for filename in files:
                    file_path = os.path.join(dirpath, filename)
                    relative_file_path = self.relativize_path(file_path)
                    with open(file_path, 'rb') as f:
                        self.client_state[relative_file_path] = hashlib.md5(f.read()).hexdigest()

    def sync_with_server(self):
        """
        download from server the files state and find the difference from actual state
        """
        def download_files_state():
            """download from server the files state"""
            server_state = self.event_dispatcher('get_server_state')
            return server_state['files']

        server_state = download_files_state()
        for file_path in server_state:
            if file_path not in self.client_state:
                self.client_state[file_path] = server_state[file_path]
                data = {'filepath' : file_path }
                self.event_dispatcher('download', data )
            else:
                if server_state[file_path] != self.client_state[file_path]:
                    print 'aggiora file'
                    pass # TODO files/aggiorna

    def cmd_dispatcher(self, data):
        """
        It dispatch cmd and args to the api manager object
        """
        cmd = data.keys()[0]  # it will be always one key
        args = data[cmd]  # it will be a dict specifying the args

        print cmd, args

        if cmd == 'shutdown':
            self.shutdown()
        else:
            self.conn_mng.dispatch_request(cmd, args)

    def event_dispatcher(self, cmd, data_file):
        """
        It dispatch the captured events to the api manager object
        """
        self.conn_mng.dispatch_request(cmd, data_file)

    def serve_forever(self):
        """
        it handles the dir manager thread and the server socket together
        """
        backlog = 5
        int_size = struct.calcsize('!i')

        listener_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        listener_socket.bind((self.cfg['host'], self.cfg['port']))
        listener_socket.listen(backlog)
        r_list = [listener_socket]

        self.dir_manager.start()

        self.running = 1
        try:
            while self.running:
                r_ready, w_ready, e_ready = select.select(r_list, [], [], self.TIMEOUT)

                for s in r_ready:

                    if s == listener_socket:
                        # handle the server socket
                        client_socket, client_address = listener_socket.accept()
                        r_list.append(client_socket)
                    else:
                        # handle all other sockets
                        length = s.recv(int_size)
                        if length:
                            length = int(struct.unpack('!i', length)[0])
                            data = json.loads(s.recv(length))
                            self.cmd_dispatcher(data)
                        else:
                            s.close()
                            r_list.remove(s)
        except KeyboardInterrupt:
            self.shutdown()

        self.dir_manager.join()
        listener_socket.close()

    def shutdown(self):
        self.dir_manager.stop()
        self.running = 0

def load_json(conf_path):
    if os.path.isfile(conf_path):
        with open(conf_path,"r") as fo:
            config = json.load(fo)
        return config
    else:
        return False

if __name__ == '__main__':

    daemon = Daemon()
    daemon.serve_forever()
<commit_msg>Add reuse to the socket so i can use the same port after crash of daemon<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import socket
import struct
import select
import os
import hashlib

# we import PollingObserver instead of Observer because the deleted event
# is not capturing https://github.com/gorakhargosh/watchdog/issues/46
from watchdog.observers.polling import PollingObserver as Observer
from watchdog.events import FileSystemEventHandler

import connection_manager

class DirectoryMonitor(FileSystemEventHandler):
    """
    The DirectoryMonitor for file system events
    like: moved, deleted, created, modified
    """
    def __init__(self, folder_path, event_dispatcher, client_state):
        FileSystemEventHandler.__init__(self)
        self.client_state = client_state
        self.event_dispatcher = event_dispatcher
        self.folder_path = folder_path
        self.folder_watched = self.folder_path.split(os.sep)[-1]
        self.observer = Observer()
        self.observer.schedule(self, path=folder_path, recursive=True)

    def on_any_event(self, event):
        """
        it catpures any filesytem event and redirects it to the event_dispatcher
        """
        def build_data(cmd,e):
            data = {}
            data['cmd'] = cmd
            data['file'] = {
                "filepath": self.relativize_path(e.src_path),
                "mtime": os.path.getmtime(e.src_path),
                "md5": hashlib.md5(e.src_path).hexdigest()
            } # TODO update struct with new implemantation data = {<md5> : <filepath>}
            return data

        if event.is_directory is False:
            e = event

            if e.event_type == 'modified':
                data = build_data('modify', e)
                self.event_dispatcher(data['cmd'], data['file'])

            elif e.event_type == 'created':
                data = build_data('upload', e)
                if data['file']['filepath'] not in self.client_state:
                    self.event_dispatcher(data['cmd'], data['file'])
                else:
                    print 'FILE ESISTENTE, TODO VERIFICA md5 INVECE DEL path'

            elif e.event_type == 'moved':
                data = {'cmd':'move'}
                data['file'] = {
                    'src_path': self.relativize_path(e.src_path),
                    'dest_path': self.relativize_path(e.dest_path)}
                self.event_dispatcher(data['cmd'], data['file'])

            elif e.event_type == 'deleted':
                data = {'cmd':'delete'}
                data['file'] = {
                    "filepath": self.relativize_path(e.src_path)}
                self.event_dispatcher(data['cmd'], data['file'])

    def relativize_path(self,path_to_clean):
        """
        This function relativize the path watched by watchdog:
        for example: /home/user/watched/subfolder/ will be subfolder/
        """
        cleaned_path = path_to_clean.split(self.folder_watched)[-1]
        # cleaned from first slash character
        return cleaned_path[1:]

    def start(self):
        """
        starts the observer thread
        """
        self.observer.start()

    def stop(self):
        """
        stops the observer thread
        """
        self.observer.stop()

    def join(self):
        """
        waits for observer execution finalize
        """
        self.observer.join()

class Daemon(object):
    """
    Root of all evil:
    it loads program configurations,
    it starts directory events observations
    it serves command manager
    """

    TIMEOUT = 0.5

    def __init__(self):
        self.cfg = load_json('config.json')
        if not self.cfg:
            print "No config File!"
            exit()
        self.client_state = {}
        self.update_client_state()
        self.dir_manager = DirectoryMonitor(self.cfg['sharing_path'], self.event_dispatcher, self.client_state)
        self.conn_mng = connection_manager.ConnectionManager(self.cfg)
        self.sync_with_server()
        self.running = 0

    def relativize_path(self,path_to_clean):
        """
        This function relativize the path watched by watchdog:
        for example: /home/user/watched/subfolder/ will be subfolder/
        """
        folder_watched = self.cfg['sharing_path'].split(os.sep)[-1]
        cleaned_path = path_to_clean.split(folder_watched)[-1]
        # cleaned from first slash character
        return cleaned_path[1:]

    def update_client_state(self):
            for dirpath, dirs, files in os.walk(self.cfg['sharing_path']):
                for filename in files:
                    file_path = os.path.join(dirpath, filename)
                    relative_file_path = self.relativize_path(file_path)
                    with open(file_path, 'rb') as f:
                        self.client_state[relative_file_path] = hashlib.md5(f.read()).hexdigest()

    def sync_with_server(self):
        """
        download from server the files state and find the difference from actual state
        """
        def download_files_state():
            """download from server the files state"""
            server_state = self.event_dispatcher('get_server_state')
            return server_state['files']

        server_state = download_files_state()
        for file_path in server_state:
            if file_path not in self.client_state:
                self.client_state[file_path] = server_state[file_path]
                data = {'filepath' : file_path }
                self.event_dispatcher('download', data )
            else:
                if server_state[file_path] != self.client_state[file_path]:
                    print 'aggiora file'
                    pass # TODO files/aggiorna

    def cmd_dispatcher(self, data):
        """
        It dispatch cmd and args to the api manager object
        """
        cmd = data.keys()[0]  # it will be always one key
        args = data[cmd]  # it will be a dict specifying the args

        print cmd, args

        if cmd == 'shutdown':
            self.shutdown()
        else:
            self.conn_mng.dispatch_request(cmd, args)

    def event_dispatcher(self, cmd, data_file):
        """
        It dispatch the captured events to the api manager object
        """
        self.conn_mng.dispatch_request(cmd, data_file)

    def serve_forever(self):
        """
        it handles the dir manager thread and the server socket together
        """
        backlog = 5
        int_size = struct.calcsize('!i')

        listener_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        listener_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        listener_socket.bind((self.cfg['host'], self.cfg['port']))
        listener_socket.listen(backlog)
        r_list = [listener_socket]

        self.dir_manager.start()

        self.running = 1
        try:
            while self.running:
                r_ready, w_ready, e_ready = select.select(r_list, [], [], self.TIMEOUT)

                for s in r_ready:

                    if s == listener_socket:
                        # handle the server socket
                        client_socket, client_address = listener_socket.accept()
                        r_list.append(client_socket)
                    else:
                        # handle all other sockets
                        length = s.recv(int_size)
                        if length:
                            length = int(struct.unpack('!i', length)[0])
                            data = json.loads(s.recv(length))
                            self.cmd_dispatcher(data)
                        else:
                            s.close()
                            r_list.remove(s)
        except KeyboardInterrupt:
            self.shutdown()

        self.dir_manager.join()
        listener_socket.close()

    def shutdown(self):
        self.dir_manager.stop()
        self.running = 0

def load_json(conf_path):
    if os.path.isfile(conf_path):
        with open(conf_path,"r") as fo:
            config = json.load(fo)
        return config
    else:
        return False

if __name__ == '__main__':

    daemon = Daemon()
    daemon.serve_forever()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import socket
import struct
import select
import os
import hashlib
import re
from sys import exit as exit

# we import PollingObserver instead of Observer because the deleted event
# is not capturing https://github.com/gorakhargosh/watchdog/issues/46
from watchdog.observers.polling import PollingObserver as Observer
from watchdog.events import RegexMatchingEventHandler

from connection_manager import ConnectionManager

class Daemon(RegexMatchingEventHandler):
    # TODO : if conf_path doesn't exist create it
    DEFAULT_CONFIG = {
        "sharing_path": "./sharing_folder",
        "cmd_address": "localhost",
        "cmd_port": 50001,
        "api_suffix": "/API/V1/",
        "server_address":"http://localhost:5000",
        "user":"default_user",
        "pass":"default_pass",
        "timeout_listener_sock" : 0.5,
        "backlog_listener_sock" : 5
    }

    IGNORE_REGEXES = [
                        '.*\.[a-zA-z]+?#', # Libreoffice suite temporary file ignored
                        '.*\.[a-zA-Z]+?~' ,# gedit issue solved ignoring this pattern: gedit first delete file, create, and move to dest_path *.txt~
                        '.*\/(\..*)' # hidden files TODO: improve
                    ]

    PATH_CONFIG = 'config.json'
    INT_SIZE = struct.calcsize('!i')

    def __init__(self):
        RegexMatchingEventHandler.__init__(self,  ignore_regexes=Daemon.IGNORE_REGEXES, ignore_directories=True)
        # Initialize variable
        self.daemon_state = 'down' # TODO implement the daemon state( disconnected, connected, syncronizing, ready...)
        self.running = 0
        self.client_snapshot = {}
        self.listener_socket = None
        self.observer = None
        self.cfg = self.load_json(Daemon.PATH_CONFIG)
        # We call os.path.abspath to unrelativize the sharing path(now is relative for development purpose)
        # TODO: Allow the setting of sharing path by user
        self.cfg['sharing_path'] = os.path.abspath(self.cfg['sharing_path'])

        self.conn_mng = ConnectionManager(self.cfg)
        # Operations necessary to start the daemon
        self.connect_to_server()
        self.build_client_snapshot()
        self.sync_with_server()
        self.create_observer()

    def load_json(self, conf_path):
        if os.path.isfile(conf_path):
            with open(conf_path,"r") as fo:
                config = json.load(fo)
            return config
        else:
            return self.DEFAULT_CONFIG

    def connect_to_server(self):
        # self.cfg['server_address']
        pass

    def build_client_snapshot(self):
        for dirpath, dirs, files in os.walk(self.cfg['sharing_path']):
                for filename in files:
                    file_path = os.path.join(dirpath, filename)
                    matched_regex = False
                    for r in Daemon.IGNORE_REGEXES:
                        if re.match(r,file_path) != None:
                            matched_regex = True
                            print "Ignored Path:", file_path
                            break
                    if not matched_regex:
                        relative_path = self.relativize_path(file_path)
                        with open(file_path, 'rb') as f:
                            self.client_snapshot[relative_path] = hashlib.md5(f.read()).hexdigest()

    def sync_with_server(self):
        """
        Download from server the files state and find the difference from actual state.
        """

        server_snapshot = self.conn_mng.dispatch_request('get_server_snapshot')
        if server_snapshot is None:
            self.stop("\nReceived bad snapshot. Server down?\n")
        else:
            server_snapshot = server_snapshot['files']

        for file_path in server_snapshot:
            if file_path not in self.client_snapshot:
                # TODO : check if download succeed, if so update client_snapshot with the new file
                self.conn_mng.dispatch_request('download', {'filepath' : file_path } )
                self.client_snapshot[file_path] = server_snapshot[file_path]
            elif server_snapshot[file_path] != self.client_snapshot[file_path]:
                self.conn_mng.dispatch_request('modify', {'filepath' : file_path } )
        for file_path in self.client_snapshot:
            if file_path not in server_snapshot:
                self.conn_mng.dispatch_request('upload', {'filepath': file_path})

    def relativize_path(self, abs_path):
        """
        This function relativize the path watched by daemon:
        for example: /home/user/watched/subfolder/ will be subfolder/
        """
        folder_watched_abs = os.path.abspath(self.cfg['sharing_path'])
        relative_path = abs_path.split(folder_watched_abs)[-1]
        return relative_path[1:]

    def create_observer(self):
        """
        Create an instance of the watchdog Observer thread class.
        """
        self.observer = Observer()
        self.observer.schedule(self, path=self.cfg['sharing_path'], recursive=True)

    def build_data(self, cmd, e, new_md5 = None):
        """
        Prepares the data from event handler to be delivered to connection_manager.
        """

        data = {'cmd' : cmd}
        if cmd == 'copy':
            for path in self.client_snapshot:
                if new_md5 in self.client_snapshot[path]: path_with_same_md5 = path
            data['file'] = {
            'src' : path_with_same_md5,
            'dst' : self.relativize_path(e.src_path),
            'md5' : self.client_snapshot[path_with_same_md5]
            }
        elif cmd == 'move':
                with open(e.dest_path, 'rb') as f:
                    dest_md5 = hashlib.md5(f.read()).hexdigest()
                data['file'] = {
                'src' : self.relativize_path(e.src_path),
                'dst' : self.relativize_path(e.dest_path),
                'md5' : dest_md5
                }
        elif cmd == 'delete':
            data['file'] = {
                'filepath': self.relativize_path(e.src_path),
                }
        else:
            f = open(e.src_path, 'rb')
            data['file'] = {
                'filepath': self.relativize_path(e.src_path),
                'md5' : hashlib.md5(f.read()).hexdigest()
                }
        return data

    def on_any_event(self, event):
        """
        Catpure any filesytem event and redirects it to the connection_manager.
        """

        # TODO update struct with new more performance data structure

        ####################################
        #### In client_snapshot the structure are {'<filepath>' : '<md5>'} so you have to convert!!!!
        ####################################

        if event.is_directory is False:
            e = event

            if e.event_type == 'modified':
                print 'start modified'
                data = self.build_data('modify', e)
                self.client_snapshot[data['file']['filepath']] = data['file']['md5']

            elif e.event_type == 'created':
                with open(e.src_path, 'rb') as f:
                        new_md5 = hashlib.md5(f.read()).hexdigest()
                # this check that the md5 doesn't exist, in that case is not a creation but a copy
                relative_path = self.relativize_path(e.src_path)

                if new_md5 in self.client_snapshot.values():
                    print 'start copy'
                    data = self.build_data('copy', e, new_md5)
                    self.client_snapshot[data['file']['dst']] = data['file']['md5']
                # this elif relative_path deny to overwrite file with the same path but md5 different
                # TODO FIX this case that happen with gedit
                elif relative_path in self.client_snapshot:
                    print 'start corrected created'
                    data = self.build_data('modify', e)
                    self.client_snapshot[data['file']['filepath']] = data['file']['md5']
                else:
                    print 'start create'
                    data = self.build_data('upload', e)
                    self.client_snapshot[data['file']['filepath']] = data['file']['md5']

            elif e.event_type == 'moved':
                print 'start move'
                data = self.build_data('move', e)
                self.client_snapshot[data['file']['dst']] = data['file']['md5']
                self.client_snapshot.pop(data['file']['src'],"NOTHING TO POP")
            elif e.event_type == 'deleted':
                print 'start delete'
                data = self.build_data('delete', e)
                self.client_snapshot.pop(data['file']['filepath'],"NOTHING TO POP")
            # TODO verify what happen if the server return a error message

            self.conn_mng.dispatch_request(data['cmd'], data['file'])

    def start(self):
        """
        Starts the communication with the command_manager.
        """

        self.listener_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.listener_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.listener_socket.bind((self.cfg['cmd_address'], self.cfg['cmd_port']))
        self.listener_socket.listen(self.cfg['backlog_listener_sock'])
        r_list = [self.listener_socket]
        self.observer.start()
        self.daemon_state = 'started'
        self.running = 1
        try:
            while self.running:
                r_ready, w_ready, e_ready = select.select(r_list, [], [], self.cfg['timeout_listener_sock'])

                for s in r_ready:

                    if s == self.listener_socket:
                        # handle the server socket
                        client_socket, client_address = self.listener_socket.accept()
                        r_list.append(client_socket)
                    else:
                        # handle all other sockets
                        length = s.recv(Daemon.INT_SIZE)
                        if length:
                            length = int(struct.unpack('!i', length)[0])
                            message = json.loads(s.recv(length))
                            for cmd, data in message.items():
                                if cmd == 'shutdown' : raise KeyboardInterrupt
                                self.conn_mng.dispatch_request(cmd, data)
                        else:
                            s.close()
                            r_list.remove(s)
        except KeyboardInterrupt:
            self.stop()

    def stop(self, exit_message = 0):
        """
        Stop the Daemon components (observer and communication with command_manager).
        """
        if self.daemon_state == 'started':
            self.observer.stop()
            self.observer.join()
            self.listener_socket.close()
        self.running = 0
        exit(exit_message)

if __name__ == '__main__':

    daemon = Daemon()
    daemon.start()<commit_msg>changed stop module to return integer to be more coherent with standard sys.exit()<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import socket
import struct
import select
import os
import hashlib
import re
from sys import exit as exit

# we import PollingObserver instead of Observer because the deleted event
# is not capturing https://github.com/gorakhargosh/watchdog/issues/46
from watchdog.observers.polling import PollingObserver as Observer
from watchdog.events import RegexMatchingEventHandler

from connection_manager import ConnectionManager

class Daemon(RegexMatchingEventHandler):
    # TODO : if conf_path doesn't exist create it
    DEFAULT_CONFIG = {
        "sharing_path": "./sharing_folder",
        "cmd_address": "localhost",
        "cmd_port": 50001,
        "api_suffix": "/API/V1/",
        "server_address":"http://localhost:5000",
        "user":"default_user",
        "pass":"default_pass",
        "timeout_listener_sock" : 0.5,
        "backlog_listener_sock" : 5
    }

    IGNORE_REGEXES = [
                        '.*\.[a-zA-z]+?#', # Libreoffice suite temporary file ignored
                        '.*\.[a-zA-Z]+?~' ,# gedit issue solved ignoring this pattern: gedit first delete file, create, and move to dest_path *.txt~
                        '.*\/(\..*)' # hidden files TODO: improve
                    ]

    PATH_CONFIG = 'config.json'
    INT_SIZE = struct.calcsize('!i')

    def __init__(self):
        RegexMatchingEventHandler.__init__(self,  ignore_regexes=Daemon.IGNORE_REGEXES, ignore_directories=True)
        # Initialize variable
        self.daemon_state = 'down' # TODO implement the daemon state( disconnected, connected, syncronizing, ready...)
        self.running = 0
        self.client_snapshot = {}
        self.listener_socket = None
        self.observer = None
        self.cfg = self.load_json(Daemon.PATH_CONFIG)
        # We call os.path.abspath to unrelativize the sharing path(now is relative for development purpose)
        # TODO: Allow the setting of sharing path by user
        self.cfg['sharing_path'] = os.path.abspath(self.cfg['sharing_path'])

        self.conn_mng = ConnectionManager(self.cfg)
        # Operations necessary to start the daemon
        self.connect_to_server()
        self.build_client_snapshot()
        self.sync_with_server()
        self.create_observer()

    def load_json(self, conf_path):
        if os.path.isfile(conf_path):
            with open(conf_path,"r") as fo:
                config = json.load(fo)
            return config
        else:
            return self.DEFAULT_CONFIG

    def connect_to_server(self):
        # self.cfg['server_address']
        pass

    def build_client_snapshot(self):
        for dirpath, dirs, files in os.walk(self.cfg['sharing_path']):
                for filename in files:
                    file_path = os.path.join(dirpath, filename)
                    matched_regex = False
                    for r in Daemon.IGNORE_REGEXES:
                        if re.match(r,file_path) != None:
                            matched_regex = True
                            print "Ignored Path:", file_path
                            break
                    if not matched_regex:
                        relative_path = self.relativize_path(file_path)
                        with open(file_path, 'rb') as f:
                            self.client_snapshot[relative_path] = hashlib.md5(f.read()).hexdigest()

    def sync_with_server(self):
        """
        Download from server the files state and find the difference from actual state.
        """

        server_snapshot = self.conn_mng.dispatch_request('get_server_snapshot')
        if server_snapshot is None:
            self.stop(1, "\nReceived bad snapshot. Server down?\n")
        else:
            server_snapshot = server_snapshot['files']

        for file_path in server_snapshot:
            if file_path not in self.client_snapshot:
                # TODO : check if download succeed, if so update client_snapshot with the new file
                self.conn_mng.dispatch_request('download', {'filepath' : file_path } )
                self.client_snapshot[file_path] = server_snapshot[file_path]
            elif server_snapshot[file_path] != self.client_snapshot[file_path]:
                self.conn_mng.dispatch_request('modify', {'filepath' : file_path } )
        for file_path in self.client_snapshot:
            if file_path not in server_snapshot:
                self.conn_mng.dispatch_request('upload', {'filepath': file_path})

    def relativize_path(self, abs_path):
        """
        This function relativize the path watched by daemon:
        for example: /home/user/watched/subfolder/ will be subfolder/
        """
        folder_watched_abs = os.path.abspath(self.cfg['sharing_path'])
        relative_path = abs_path.split(folder_watched_abs)[-1]
        return relative_path[1:]

    def create_observer(self):
        """
        Create an instance of the watchdog Observer thread class.
        """
        self.observer = Observer()
        self.observer.schedule(self, path=self.cfg['sharing_path'], recursive=True)

    def build_data(self, cmd, e, new_md5 = None):
        """
        Prepares the data from event handler to be delivered to connection_manager.
        """

        data = {'cmd' : cmd}
        if cmd == 'copy':
            for path in self.client_snapshot:
                if new_md5 in self.client_snapshot[path]: path_with_same_md5 = path
            data['file'] = {
            'src' : path_with_same_md5,
            'dst' : self.relativize_path(e.src_path),
            'md5' : self.client_snapshot[path_with_same_md5]
            }
        elif cmd == 'move':
                with open(e.dest_path, 'rb') as f:
                    dest_md5 = hashlib.md5(f.read()).hexdigest()
                data['file'] = {
                'src' : self.relativize_path(e.src_path),
                'dst' : self.relativize_path(e.dest_path),
                'md5' : dest_md5
                }
        elif cmd == 'delete':
            data['file'] = {
                'filepath': self.relativize_path(e.src_path),
                }
        else:
            f = open(e.src_path, 'rb')
            data['file'] = {
                'filepath': self.relativize_path(e.src_path),
                'md5' : hashlib.md5(f.read()).hexdigest()
                }
        return data

    def on_any_event(self, event):
        """
        Catpure any filesytem event and redirects it to the connection_manager.
        """

        # TODO update struct with new more performance data structure

        ####################################
        #### In client_snapshot the structure are {'<filepath>' : '<md5>'} so you have to convert!!!!
        ####################################

        if event.is_directory is False:
            e = event

            if e.event_type == 'modified':
                print 'start modified'
                data = self.build_data('modify', e)
                self.client_snapshot[data['file']['filepath']] = data['file']['md5']

            elif e.event_type == 'created':
                with open(e.src_path, 'rb') as f:
                        new_md5 = hashlib.md5(f.read()).hexdigest()
                # this check that the md5 doesn't exist, in that case is not a creation but a copy
                relative_path = self.relativize_path(e.src_path)

                if new_md5 in self.client_snapshot.values():
                    print 'start copy'
                    data = self.build_data('copy', e, new_md5)
                    self.client_snapshot[data['file']['dst']] = data['file']['md5']
                # this elif relative_path deny to overwrite file with the same path but md5 different
                # TODO FIX this case that happen with gedit
                elif relative_path in self.client_snapshot:
                    print 'start corrected created'
                    data = self.build_data('modify', e)
                    self.client_snapshot[data['file']['filepath']] = data['file']['md5']
                else:
                    print 'start create'
                    data = self.build_data('upload', e)
                    self.client_snapshot[data['file']['filepath']] = data['file']['md5']

            elif e.event_type == 'moved':
                print 'start move'
                data = self.build_data('move', e)
                self.client_snapshot[data['file']['dst']] = data['file']['md5']
                self.client_snapshot.pop(data['file']['src'],"NOTHING TO POP")
            elif e.event_type == 'deleted':
                print 'start delete'
                data = self.build_data('delete', e)
                self.client_snapshot.pop(data['file']['filepath'],"NOTHING TO POP")
            # TODO verify what happen if the server return a error message

            self.conn_mng.dispatch_request(data['cmd'], data['file'])

    def start(self):
        """
        Starts the communication with the command_manager.
        """

        self.listener_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.listener_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.listener_socket.bind((self.cfg['cmd_address'], self.cfg['cmd_port']))
        self.listener_socket.listen(self.cfg['backlog_listener_sock'])
        r_list = [self.listener_socket]
        self.observer.start()
        self.daemon_state = 'started'
        self.running = 1
        try:
            while self.running:
                r_ready, w_ready, e_ready = select.select(r_list, [], [], self.cfg['timeout_listener_sock'])

                for s in r_ready:

                    if s == self.listener_socket:
                        # handle the server socket
                        client_socket, client_address = self.listener_socket.accept()
                        r_list.append(client_socket)
                    else:
                        # handle all other sockets
                        length = s.recv(Daemon.INT_SIZE)
                        if length:
                            length = int(struct.unpack('!i', length)[0])
                            message = json.loads(s.recv(length))
                            for cmd, data in message.items():
                                if cmd == 'shutdown' : raise KeyboardInterrupt
                                self.conn_mng.dispatch_request(cmd, data)
                        else:
                            s.close()
                            r_list.remove(s)
        except KeyboardInterrupt:
            self.stop(0)

    def stop(self, exit_status, exit_message = None):
        """
        Stop the Daemon components (observer and communication with command_manager).
        """
        if self.daemon_state == 'started':
            self.observer.stop()
            self.observer.join()
            self.listener_socket.close()
        self.running = 0
        if exit_message:
            print exit_message
        exit(exit_status)

if __name__ == '__main__':

    daemon = Daemon()
    daemon.start()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2019-10-08 00:35
from __future__ import unicode_literals

from django.db import migrations
from users.models import DEFAULT_GROUP

def create_default_group(apps, schema_editor):
    Group = apps.get_model('auth', 'Group')
    User = apps.get_model('auth', 'User')
    Permission = apps.get_model('auth', 'Permission')
    Audio = apps.get_model('audio', "Audio")
    ContentType = apps.get_model('contenttypes', "ContentType")

    default_group, created = Group.objects.get_or_create(name=DEFAULT_GROUP)
    if created:
        audio_content_type = ContentType.objects.get_for_model(Audio)
        add_audio, created = Permission.objects.get_or_create(codename='add_audio', content_type=audio_content_type)
        default_group.permissions.add(add_audio)
        default_group.save()

    users = apps.get_model('auth', 'User')
    for user in User.objects.all():
        user.groups.add(default_group)
        user.save()

class Migration(migrations.Migration):

    dependencies = [
        ('auth', '__latest__'),
    ]

    operations = [
         migrations.RunPython(create_default_group),
    ]
<commit_msg>User explicit migration dependency to fix InconsistentMigrationHistory<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2019-10-08 00:35
from __future__ import unicode_literals

from django.db import migrations
from users.models import DEFAULT_GROUP

def create_default_group(apps, schema_editor):
    Group = apps.get_model('auth', 'Group')
    User = apps.get_model('auth', 'User')
    Permission = apps.get_model('auth', 'Permission')
    Audio = apps.get_model('audio', "Audio")
    ContentType = apps.get_model('contenttypes', "ContentType")

    default_group, created = Group.objects.get_or_create(name=DEFAULT_GROUP)
    if created:
        audio_content_type = ContentType.objects.get_for_model(Audio)
        add_audio, created = Permission.objects.get_or_create(codename='add_audio', content_type=audio_content_type)
        default_group.permissions.add(add_audio)
        default_group.save()

    users = apps.get_model('auth', 'User')
    for user in User.objects.all():
        user.groups.add(default_group)
        user.save()

class Migration(migrations.Migration):

    dependencies = [
        ('auth', '0007_alter_validators_add_error_messages'),
    ]

    operations = [
         migrations.RunPython(create_default_group),
    ]
<|endoftext|>


<commit_before>import os
import tempfile
import subprocess
import datetime
import pytz
import dateutil.parser
from ruamel.yaml import YAML
from ruamel.yaml.compat import StringIO
import requests
import json
import logging

import lxml.html
import cachetools

LOGGER = logging.getLogger("conda_forge_webservices.status_monitor")

APP_DATA = {
    'azure-pipelines': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
    'travis-ci': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
    'github-actions': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
    'appveyor': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
    'circleci': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
    'drone': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
}

STATUS_UPDATE_DELAY = 60
NOSTATUS = 'No Status Available'
WEBS_STATUS_UPDATED = None
WEBS_STATUS_DATA = {
    'status': NOSTATUS,
    'updated_at': None,
}
START_TIME = datetime.datetime.fromisoformat("2020-01-01T00:00:00+00:00")
TIME_INTERVAL = 60*5  # five minutes

def _make_time_key(uptime):
    dt = uptime.timestamp() - START_TIME.timestamp()
    return int(dt // TIME_INTERVAL)

# reload the cache
RELOAD_CACHE = True

def _reload_cache():
    print(" ", flush=True)
    print("!!!!!!!!!!!!!! RELOADING THE CACHE !!!!!!!!!!!!!!", flush=True)

    global APP_DATA

    try:
        data = requests.get(
            ("https://raw.githubusercontent.com/conda-forge/"
             "conda-forge-status-monitor/"
             "master/data/latest.json")).json()
    except Exception as e:
        print(e, flush=True)
        data = None

    if data is not None:
        for slug in APP_DATA:
            print('reloading data for %s' % slug, flush=True)

            if slug not in data:
                continue
            else:
                _data = data[slug]

            for repo in _data['repos']:
                APP_DATA[slug]['repos'][repo] = _data['repos'][repo]

            for ts in _data['rates']:
                t = datetime.datetime.fromisoformat(ts).astimezone(pytz.UTC)
                key = _make_time_key(t)
                APP_DATA[slug]['rates'][key] = _data['rates'][ts]

            print("    reloaded %d repos" % len(APP_DATA[slug]['repos']), flush=True)
            print("    reloaded %d rates" % len(APP_DATA[slug]['rates']), flush=True)
    else:
        print("could not get app cache!", flush=True)
    print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", flush=True)
    print(" ", flush=True)

if RELOAD_CACHE:
    _reload_cache()
    RELOAD_CACHE = False

class MyYAML(YAML):
    """dump yaml as string rippd from docs"""
    def dump(self, data, stream=None, **kw):
        inefficient = False
        if stream is None:
            inefficient = True
            stream = StringIO()
        YAML.dump(self, data, stream, **kw)
        if inefficient:
            return stream.getvalue()

def _make_est_from_time_key(key, iso=False):
    est = pytz.timezone('US/Eastern')
    fmt = '%Y-%m-%d %H:%M:%S %Z%z'
    dt = datetime.timedelta(seconds=key * TIME_INTERVAL)
    t = dt + START_TIME
    t = t.astimezone(est)
    if iso:
        return t.isoformat()
    else:
        return t.strftime(fmt)

def _make_report_data(iso=False):
    now = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
    know = _make_time_key(now)

    report = {}
    for key in APP_DATA:
        rates = {}
        for k in range(know, know-96, -1):
            tstr = _make_est_from_time_key(k, iso=iso)
            rates[tstr] = APP_DATA[key]['rates'].get(k, 0)

        total = sum(v for v in rates.values())

        report[key] = {
            'total': total,
            'rates': rates,
            'repos': {
                k: v
                for k, v in
                sorted(APP_DATA[key]['repos'].items(), key=lambda x: x[1])[::-1]
            },
        }

    return report

def render_status_index():
    yaml = MyYAML()
    shell = """\
<!DOCTYPE html>

        <title>conda-forge status data</title>

conda-forge status data

%s

""" % yaml.dump(_make_report_data(iso=False))
    return shell

def dump_report_data(name=None):
    data = _make_report_data(iso=True)
    if name is None:
        return json.dumps(data)
    else:
        return json.dumps(data[name])

def get_azure_status():
    status_data = {}

    # always update azure
    try:
        r = requests.get('https://status.dev.azure.com', timeout=2)
        if r.status_code != 200:
            status_data['azure'] = NOSTATUS
        else:
            s = json.loads(
                lxml
                .html
                .fromstring(r.content)
                .get_element_by_id('dataProviders')
                .text
            )

            def _rec_search(d):
                if isinstance(d, dict):
                    if 'health' in d and 'message' in d:
                        return d['message']
                    else:
                        for v in d.values():
                            if isinstance(v, dict):
                                val = _rec_search(v)
                                if val is not None:
                                    return val
                        return None
                else:
                    return None

            stat = _rec_search(s)

            if stat is None:
                stat = NOSTATUS

            status_data['status'] = stat
    except requests.exceptions.RequestException:
        status_data['status'] = NOSTATUS

    fmt = '%Y-%m-%d %H:%M:%S %Z%z'
    status_data['updated_at'] = (
        datetime.datetime.now().astimezone(pytz.UTC).strftime(fmt)
    )

    return json.dumps(status_data)

def update_data_status(event_data):
    global APP_DATA

    repo = event_data['repository']['full_name']

    if 'circleci' in event_data['context']:
        slug = 'circleci'
    elif 'appveyor' in event_data['context']:
        slug = 'appveyor'
    elif 'travis' in event_data['context']:
        slug = 'travis-ci'
    elif 'drone' in event_data['context']:
        slug = 'drone'
    else:
        LOGGER.warning("    context not found: %s", event_data['context'])
        return

    LOGGER.debug("    repo: %s", repo)
    LOGGER.debug("    app: %s", slug)
    LOGGER.debug("    state: %s", event_data['state'])

    if event_data['state'] in ['success', 'failure', 'error']:

        LOGGER.debug("    updated_at: %s", event_data['updated_at'])

        uptime = dateutil.parser.isoparse(event_data['updated_at'])
        interval = _make_time_key(uptime)
        if interval not in APP_DATA[slug]['rates']:
            APP_DATA[slug]['rates'][interval] = 0
        APP_DATA[slug]['rates'][interval] = (
            APP_DATA[slug]['rates'][interval] + 1)

        if repo not in APP_DATA[slug]['repos']:
            APP_DATA[slug]['repos'][repo] = 0
        APP_DATA[slug]['repos'][repo] = APP_DATA[slug]['repos'][repo] + 1

def update_data_check_run(event_data):
    global APP_DATA

    repo = event_data['repository']['full_name']
    cs = event_data['check_run']

    LOGGER.debug("    repo: %s", repo)
    LOGGER.debug("    app: %s", cs['app']['slug'])
    LOGGER.debug("    action: %s", event_data['action'])
    LOGGER.debug("    status: %s", cs['status'])
    LOGGER.debug("    conclusion: %s", cs['conclusion'])

    if (
        cs['app']['slug'] in APP_DATA and
        cs['status'] == 'completed'
    ):
        LOGGER.debug("    completed_at: %s", cs['completed_at'])
        key = cs['app']['slug']

        uptime = dateutil.parser.isoparse(cs['completed_at'])
        interval = _make_time_key(uptime)
        if interval not in APP_DATA[key]['rates']:
            APP_DATA[key]['rates'][interval] = 0
        APP_DATA[key]['rates'][interval] = (
            APP_DATA[key]['rates'][interval]
            + 1
        )

        if repo not in APP_DATA[key]['repos']:
            APP_DATA[key]['repos'][repo] = 0
        APP_DATA[key]['repos'][repo] = (
            APP_DATA[key]['repos'][repo]
            + 1
        )

def cache_status_data():
    try:
        # first pull down the data
        latest_data = requests.get(
            "https://services.conda-forge.org/status-monitor/db"
        ).json()

        # now update the repo
        with tempfile.TemporaryDirectory() as tmpdir:
            subprocess.run(
                ("cd %s && git clone --depth=1 "
                 "https://github.com/conda-forge/conda-forge-status-monitor.git" %
                 tmpdir
                 ),
                shell=True,
                check=True,
                capture_output=True,
            )

            pth = os.path.join(tmpdir, "conda-forge-status-monitor")

            subprocess.run(
                "cd %s && git remote set-url --push origin "
                "https://x-access-token:${GH_TOKEN}@github.com/conda-forge/conda-forge-status-monitor.git" % pth,  # noqa
                shell=True,
                check=True,
                capture_output=True,
            )

            os.makedirs(os.path.join(pth, "data"), exist_ok=True)

            with open(os.path.join(pth, "data", "latest.json"), "w") as fp:
                json.dump(latest_data, fp, indent=2)

            subprocess.run(
                "cd %s && git add data/latest.json" % pth,
                shell=True,
                check=True,
                capture_output=True,
            )

            stat = subprocess.run(
                "cd %s && git status" % pth,
                shell=True,
                check=True,
                capture_output=True,
            )
            status = stat.stdout.decode('utf-8')
            LOGGER.debug("    cache git status: %s", status)

            if "nothing to commit" not in status:
                LOGGER.info("    making status data commit")
                subprocess.run(
                    "cd %s && git commit -m '[ci skip] "
                    "[skip ci] [cf admin skip] ***NO_CI*** "
                    "status data update %s'" % (
                        pth, datetime.datetime.utcnow().isoformat()
                    ),
                    shell=True,
                    check=True,
                    capture_output=True,
                )

                subprocess.run(
                    "cd %s && git push" % pth,
                    shell=True,
                    check=True,
                    capture_output=True,
                )
            else:
                LOGGER.info("    no status data to commit")
    except Exception as e:
        LOGGER.warning("    caching status data failed: %s" % repr(e))
<commit_msg>ENH move status from master to main<commit_after>import os
import tempfile
import subprocess
import datetime
import pytz
import dateutil.parser
from ruamel.yaml import YAML
from ruamel.yaml.compat import StringIO
import requests
import json
import logging

import lxml.html
import cachetools

LOGGER = logging.getLogger("conda_forge_webservices.status_monitor")

APP_DATA = {
    'azure-pipelines': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
    'travis-ci': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
    'github-actions': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
    'appveyor': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
    'circleci': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
    'drone': {
        'repos': cachetools.LRUCache(maxsize=128),
        'rates': cachetools.LRUCache(maxsize=96),
    },
}

STATUS_UPDATE_DELAY = 60
NOSTATUS = 'No Status Available'
WEBS_STATUS_UPDATED = None
WEBS_STATUS_DATA = {
    'status': NOSTATUS,
    'updated_at': None,
}
START_TIME = datetime.datetime.fromisoformat("2020-01-01T00:00:00+00:00")
TIME_INTERVAL = 60*5  # five minutes

def _make_time_key(uptime):
    dt = uptime.timestamp() - START_TIME.timestamp()
    return int(dt // TIME_INTERVAL)

# reload the cache
RELOAD_CACHE = True

def _reload_cache():
    print(" ", flush=True)
    print("!!!!!!!!!!!!!! RELOADING THE CACHE !!!!!!!!!!!!!!", flush=True)

    global APP_DATA

    try:
        data = requests.get(
            ("https://raw.githubusercontent.com/conda-forge/"
             "conda-forge-status-monitor/"
             "main/data/latest.json")).json()
    except Exception as e:
        print(e, flush=True)
        data = None

    if data is not None:
        for slug in APP_DATA:
            print('reloading data for %s' % slug, flush=True)

            if slug not in data:
                continue
            else:
                _data = data[slug]

            for repo in _data['repos']:
                APP_DATA[slug]['repos'][repo] = _data['repos'][repo]

            for ts in _data['rates']:
                t = datetime.datetime.fromisoformat(ts).astimezone(pytz.UTC)
                key = _make_time_key(t)
                APP_DATA[slug]['rates'][key] = _data['rates'][ts]

            print("    reloaded %d repos" % len(APP_DATA[slug]['repos']), flush=True)
            print("    reloaded %d rates" % len(APP_DATA[slug]['rates']), flush=True)
    else:
        print("could not get app cache!", flush=True)
    print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", flush=True)
    print(" ", flush=True)

if RELOAD_CACHE:
    _reload_cache()
    RELOAD_CACHE = False

class MyYAML(YAML):
    """dump yaml as string rippd from docs"""
    def dump(self, data, stream=None, **kw):
        inefficient = False
        if stream is None:
            inefficient = True
            stream = StringIO()
        YAML.dump(self, data, stream, **kw)
        if inefficient:
            return stream.getvalue()

def _make_est_from_time_key(key, iso=False):
    est = pytz.timezone('US/Eastern')
    fmt = '%Y-%m-%d %H:%M:%S %Z%z'
    dt = datetime.timedelta(seconds=key * TIME_INTERVAL)
    t = dt + START_TIME
    t = t.astimezone(est)
    if iso:
        return t.isoformat()
    else:
        return t.strftime(fmt)

def _make_report_data(iso=False):
    now = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
    know = _make_time_key(now)

    report = {}
    for key in APP_DATA:
        rates = {}
        for k in range(know, know-96, -1):
            tstr = _make_est_from_time_key(k, iso=iso)
            rates[tstr] = APP_DATA[key]['rates'].get(k, 0)

        total = sum(v for v in rates.values())

        report[key] = {
            'total': total,
            'rates': rates,
            'repos': {
                k: v
                for k, v in
                sorted(APP_DATA[key]['repos'].items(), key=lambda x: x[1])[::-1]
            },
        }

    return report

def render_status_index():
    yaml = MyYAML()
    shell = """\
<!DOCTYPE html>

        <title>conda-forge status data</title>

conda-forge status data

%s

""" % yaml.dump(_make_report_data(iso=False))
    return shell

def dump_report_data(name=None):
    data = _make_report_data(iso=True)
    if name is None:
        return json.dumps(data)
    else:
        return json.dumps(data[name])

def get_azure_status():
    status_data = {}

    # always update azure
    try:
        r = requests.get('https://status.dev.azure.com', timeout=2)
        if r.status_code != 200:
            status_data['azure'] = NOSTATUS
        else:
            s = json.loads(
                lxml
                .html
                .fromstring(r.content)
                .get_element_by_id('dataProviders')
                .text
            )

            def _rec_search(d):
                if isinstance(d, dict):
                    if 'health' in d and 'message' in d:
                        return d['message']
                    else:
                        for v in d.values():
                            if isinstance(v, dict):
                                val = _rec_search(v)
                                if val is not None:
                                    return val
                        return None
                else:
                    return None

            stat = _rec_search(s)

            if stat is None:
                stat = NOSTATUS

            status_data['status'] = stat
    except requests.exceptions.RequestException:
        status_data['status'] = NOSTATUS

    fmt = '%Y-%m-%d %H:%M:%S %Z%z'
    status_data['updated_at'] = (
        datetime.datetime.now().astimezone(pytz.UTC).strftime(fmt)
    )

    return json.dumps(status_data)

def update_data_status(event_data):
    global APP_DATA

    repo = event_data['repository']['full_name']

    if 'circleci' in event_data['context']:
        slug = 'circleci'
    elif 'appveyor' in event_data['context']:
        slug = 'appveyor'
    elif 'travis' in event_data['context']:
        slug = 'travis-ci'
    elif 'drone' in event_data['context']:
        slug = 'drone'
    else:
        LOGGER.warning("    context not found: %s", event_data['context'])
        return

    LOGGER.debug("    repo: %s", repo)
    LOGGER.debug("    app: %s", slug)
    LOGGER.debug("    state: %s", event_data['state'])

    if event_data['state'] in ['success', 'failure', 'error']:

        LOGGER.debug("    updated_at: %s", event_data['updated_at'])

        uptime = dateutil.parser.isoparse(event_data['updated_at'])
        interval = _make_time_key(uptime)
        if interval not in APP_DATA[slug]['rates']:
            APP_DATA[slug]['rates'][interval] = 0
        APP_DATA[slug]['rates'][interval] = (
            APP_DATA[slug]['rates'][interval] + 1)

        if repo not in APP_DATA[slug]['repos']:
            APP_DATA[slug]['repos'][repo] = 0
        APP_DATA[slug]['repos'][repo] = APP_DATA[slug]['repos'][repo] + 1

def update_data_check_run(event_data):
    global APP_DATA

    repo = event_data['repository']['full_name']
    cs = event_data['check_run']

    LOGGER.debug("    repo: %s", repo)
    LOGGER.debug("    app: %s", cs['app']['slug'])
    LOGGER.debug("    action: %s", event_data['action'])
    LOGGER.debug("    status: %s", cs['status'])
    LOGGER.debug("    conclusion: %s", cs['conclusion'])

    if (
        cs['app']['slug'] in APP_DATA and
        cs['status'] == 'completed'
    ):
        LOGGER.debug("    completed_at: %s", cs['completed_at'])
        key = cs['app']['slug']

        uptime = dateutil.parser.isoparse(cs['completed_at'])
        interval = _make_time_key(uptime)
        if interval not in APP_DATA[key]['rates']:
            APP_DATA[key]['rates'][interval] = 0
        APP_DATA[key]['rates'][interval] = (
            APP_DATA[key]['rates'][interval]
            + 1
        )

        if repo not in APP_DATA[key]['repos']:
            APP_DATA[key]['repos'][repo] = 0
        APP_DATA[key]['repos'][repo] = (
            APP_DATA[key]['repos'][repo]
            + 1
        )

def cache_status_data():
    try:
        # first pull down the data
        latest_data = requests.get(
            "https://services.conda-forge.org/status-monitor/db"
        ).json()

        # now update the repo
        with tempfile.TemporaryDirectory() as tmpdir:
            subprocess.run(
                ("cd %s && git clone --depth=1 "
                 "https://github.com/conda-forge/conda-forge-status-monitor.git" %
                 tmpdir
                 ),
                shell=True,
                check=True,
                capture_output=True,
            )

            pth = os.path.join(tmpdir, "conda-forge-status-monitor")

            subprocess.run(
                "cd %s && git remote set-url --push origin "
                "https://x-access-token:${GH_TOKEN}@github.com/conda-forge/conda-forge-status-monitor.git" % pth,  # noqa
                shell=True,
                check=True,
                capture_output=True,
            )

            os.makedirs(os.path.join(pth, "data"), exist_ok=True)

            with open(os.path.join(pth, "data", "latest.json"), "w") as fp:
                json.dump(latest_data, fp, indent=2)

            subprocess.run(
                "cd %s && git add data/latest.json" % pth,
                shell=True,
                check=True,
                capture_output=True,
            )

            stat = subprocess.run(
                "cd %s && git status" % pth,
                shell=True,
                check=True,
                capture_output=True,
            )
            status = stat.stdout.decode('utf-8')
            LOGGER.debug("    cache git status: %s", status)

            if "nothing to commit" not in status:
                LOGGER.info("    making status data commit")
                subprocess.run(
                    "cd %s && git commit -m '[ci skip] "
                    "[skip ci] [cf admin skip] ***NO_CI*** "
                    "status data update %s'" % (
                        pth, datetime.datetime.utcnow().isoformat()
                    ),
                    shell=True,
                    check=True,
                    capture_output=True,
                )

                subprocess.run(
                    "cd %s && git push" % pth,
                    shell=True,
                    check=True,
                    capture_output=True,
                )
            else:
                LOGGER.info("    no status data to commit")
    except Exception as e:
        LOGGER.warning("    caching status data failed: %s" % repr(e))
<|endoftext|>