<commit_before>from __future__ import print_function
from __future__ import unicode_literals
import utilities
import nltk.data
import datetime
import logging
import codecs

def query_all(collection, lt_date, gt_date, sources, write_file=False):
    """
    Function to query the MongoDB instance and obtain results for the desired
    date range. The query constructed is: greater_than_date > results
    < less_than_date.

    Parameters
    ----------

    collection: pymongo.collection.Collection.
                Collection within MongoDB that holds the scraped news stories.

    lt_date: Datetime object.
                    Date for which results should be older than. For example,
                    if the date running is the 25th, and the desired date is
                    the 24th, then the `lt_date` is the 25th.

    gt_date: Datetime object.
                        Date for which results should be older than. For
                        example, if the date running is the 25th, and the
                        desired date is the 24th, then the `gt_date`
                        is the 23rd.

    sources: List.
                Sources to pull from the MongoDB instance.

    write_file: Boolean.
                Option indicating whether to write the results from the web
                scraper to an intermediate file. Defaults to false.

    Returns
    -------

    posts: List.
            List of dictionaries of results from the MongoDB query.

    final_out: String.
                If `write_file` is True, this contains a string representation
                of the query results. Otherwise, contains an empty string.

    """

    logger = logging.getLogger('pipeline_log')
    final_out = ''
    if write_file:
        output = []
        sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
        posts = collection.find({"$and": [{"date_added": {"$lt": lt_date}},
                                          {"date_added": {"$gt": gt_date}},
                                          {"source": {"$in": sources}}]})
        for num, post in enumerate(posts):
            try:
                #print 'Processing entry {}...'.format(num)
                content = post['content'].encode('utf-8')
                if post['source'] == 'aljazeera':
                    content = content.replace("""Caution iconAttention The browser or device you are using is out of date.  It has known security flaws and a limited feature set.  You will not see all the features of some websites.  Please update your browser.""", '')
                header = '  '.join(sent_detector.tokenize(content.strip())[:4])
                string = '{}\t{}\t{}\n{}\n'.format(num, post['date'],
                                                   post['url'], header)
                output.append(string)
            except Exception as e:
                print('Error on entry {}: {}.'.format(num, e))
        final_out = '\n'.join(output)

    posts = collection.find({"$and": [{"date_added": {"$lte": lt_date}},
                                      {"date_added": {"$gt": gt_date}},
                                      {"source": {"$in": sources}}]})

    print('Total number of stories: {}'.format(posts.count()))
    logger.info('Total number of stories: {}'.format(posts.count()))

    posts = list(posts)

    return posts, final_out

def _get_sources(filepath):
    """
    Function to create a list of sources that will be used in the query to the
    MongoDB instance.

    Parameters
    ----------

    filepath: String.
                Path to file containing the source keys. File should have a
                single key on each line.

    Returns
    -------

    sources: List.
                List of sources, basically.

    """
    with open(filepath, 'r') as f:
        sources = [key.replace('\n', '') for key in f.readlines()]
    return sources

def main(current_date, file_details, write_file=False, file_stem=None):
    """
    Function to create a connection to a MongoDB instance, query for a given
    day's results, optionally write the results to a file, and return the
    results.

    Parameters
    ----------

    current_date: datetime object.
                    Date for which records are pulled. Normally this is
                    $date_running - 1. For example, if the script is running on
                    the 25th, the current_date will be the 24th.

    file_details: Named tuple.
                    Tuple containing config information.

    write_file: Boolean.
                Option indicating whether to write the results from the web
                scraper to an intermediate file. Defaults to false.

    file_stem: String. Optional.
                Optional string defining the file stem for the intermediate
                file for the scraper results.

    Returns
    -------

    posts: Dictionary.
            Dictionary of results from the MongoDB query.

    filename: String.
                If `write_file` is True, contains the filename to which the
                scraper results are writen. Otherwise is an empty string.

    """
    sources = get_sources('source_keys.txt')
    conn = utilities.make_conn(file_details.auth_db, file_details.auth_user,
                               file_details.auth_pass)

    less_than = datetime.datetime(current_date.year, current_date.month,
                                  current_date.day)
    greater_than = less_than - datetime.timedelta(days=1)
    less_than = less_than + datetime.timedelta(days=1)

    results, text = query_all(conn, less_than, greater_than, sources,
                              write_file=write_file)

    filename = ''
    if text:
        text = text.decode('utf-8')

        if file_stem:
            filename = '{}{:02d}{:02d}{:02d}.txt'.format(file_stem,
                                                         current_date.year,
                                                         current_date.month,
                                                         current_date.day)
            with codecs.open(filename, 'w', encoding='utf-8') as f:
                f.write(text)
        else:
            print('Need filestem to write results to file.')

    return results, filename

if __name__ == '__main__':
    print('Running...')
    main('temp_stem.', 'YYMMDD')
<commit_msg>Bug squashing.<commit_after>from __future__ import print_function
from __future__ import unicode_literals
import utilities
import nltk.data
import datetime
import logging
import codecs

def query_all(collection, lt_date, gt_date, sources, write_file=False):
    """
    Function to query the MongoDB instance and obtain results for the desired
    date range. The query constructed is: greater_than_date > results
    < less_than_date.

    Parameters
    ----------

    collection: pymongo.collection.Collection.
                Collection within MongoDB that holds the scraped news stories.

    lt_date: Datetime object.
                    Date for which results should be older than. For example,
                    if the date running is the 25th, and the desired date is
                    the 24th, then the `lt_date` is the 25th.

    gt_date: Datetime object.
                        Date for which results should be older than. For
                        example, if the date running is the 25th, and the
                        desired date is the 24th, then the `gt_date`
                        is the 23rd.

    sources: List.
                Sources to pull from the MongoDB instance.

    write_file: Boolean.
                Option indicating whether to write the results from the web
                scraper to an intermediate file. Defaults to false.

    Returns
    -------

    posts: List.
            List of dictionaries of results from the MongoDB query.

    final_out: String.
                If `write_file` is True, this contains a string representation
                of the query results. Otherwise, contains an empty string.

    """

    logger = logging.getLogger('pipeline_log')
    final_out = ''
    if write_file:
        output = []
        sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
        posts = collection.find({"$and": [{"date_added": {"$lt": lt_date}},
                                          {"date_added": {"$gt": gt_date}},
                                          {"source": {"$in": sources}}]})
        for num, post in enumerate(posts):
            try:
                #print 'Processing entry {}...'.format(num)
                content = post['content'].encode('utf-8')
                if post['source'] == 'aljazeera':
                    content = content.replace("""Caution iconAttention The browser or device you are using is out of date.  It has known security flaws and a limited feature set.  You will not see all the features of some websites.  Please update your browser.""", '')
                header = '  '.join(sent_detector.tokenize(content.strip())[:4])
                string = '{}\t{}\t{}\n{}\n'.format(num, post['date'],
                                                   post['url'], header)
                output.append(string)
            except Exception as e:
                print('Error on entry {}: {}.'.format(num, e))
        final_out = '\n'.join(output)

    posts = collection.find({"$and": [{"date_added": {"$lte": lt_date}},
                                      {"date_added": {"$gt": gt_date}},
                                      {"source": {"$in": sources}}]})

    print('Total number of stories: {}'.format(posts.count()))
    logger.info('Total number of stories: {}'.format(posts.count()))

    posts = list(posts)

    return posts, final_out

def _get_sources(filepath):
    """
    Function to create a list of sources that will be used in the query to the
    MongoDB instance.

    Parameters
    ----------

    filepath: String.
                Path to file containing the source keys. File should have a
                single key on each line.

    Returns
    -------

    sources: List.
                List of sources, basically.

    """
    with open(filepath, 'r') as f:
        sources = [key.replace('\n', '') for key in f.readlines()]
    return sources

def main(current_date, file_details, write_file=False, file_stem=None):
    """
    Function to create a connection to a MongoDB instance, query for a given
    day's results, optionally write the results to a file, and return the
    results.

    Parameters
    ----------

    current_date: datetime object.
                    Date for which records are pulled. Normally this is
                    $date_running - 1. For example, if the script is running on
                    the 25th, the current_date will be the 24th.

    file_details: Named tuple.
                    Tuple containing config information.

    write_file: Boolean.
                Option indicating whether to write the results from the web
                scraper to an intermediate file. Defaults to false.

    file_stem: String. Optional.
                Optional string defining the file stem for the intermediate
                file for the scraper results.

    Returns
    -------

    posts: Dictionary.
            Dictionary of results from the MongoDB query.

    filename: String.
                If `write_file` is True, contains the filename to which the
                scraper results are writen. Otherwise is an empty string.

    """
    sources = _get_sources('source_keys.txt')
    conn = utilities.make_conn(file_details.auth_db, file_details.auth_user,
                               file_details.auth_pass)

    less_than = datetime.datetime(current_date.year, current_date.month,
                                  current_date.day)
    greater_than = less_than - datetime.timedelta(days=1)
    less_than = less_than + datetime.timedelta(days=1)

    results, text = query_all(conn, less_than, greater_than, sources,
                              write_file=write_file)

    filename = ''
    if text:
        text = text.decode('utf-8')

        if file_stem:
            filename = '{}{:02d}{:02d}{:02d}.txt'.format(file_stem,
                                                         current_date.year,
                                                         current_date.month,
                                                         current_date.day)
            with codecs.open(filename, 'w', encoding='utf-8') as f:
                f.write(text)
        else:
            print('Need filestem to write results to file.')

    return results, filename

if __name__ == '__main__':
    print('Running...')
    main('temp_stem.', 'YYMMDD')
<|endoftext|>


<commit_before>from dumptruck import DumpTruck
import datetime
import re
import os

def _connect(dbname = 'scraperwiki.sqlite'):
  'Initialize the database (again). This is mainly for testing'
  global dt
  dt = DumpTruck(dbname = dbname,  adapt_and_convert = False)

_connect()

def execute(sqlquery, data=[], verbose=1):
    """ Emulate scraperwiki as much as possible by mangling dumptruck result """
    # Allow for a non-list to be passed as data.
    if type(data) != list and type(data) != tuple:
        data = [data]

    result = dt.execute(sqlquery, data, commit=False)
    # None (non-select) and empty list (select) results
    if not result:
        return {u'data': [], u'keys': []}
    dtos = lambda d: str(d) if isinstance(d, datetime.date) else d
    # Select statement with results
    return {u'data': map(lambda row: map(dtos, row.values()), result),
            u'keys': result[0].keys()}

def save(unique_keys, data, table_name="swdata", verbose=2, date=None):
    if not data:
        return
    dt.create_table(data, table_name = table_name, error_if_exists = False)
    if unique_keys != []:
        dt.create_index(unique_keys, table_name, unique = True, if_not_exists = True)
    return dt.upsert(data, table_name = table_name)

def commit(verbose=1):
    dt.commit()

def select(sqlquery, data=[], verbose=1):
    sqlquery = "select %s" % sqlquery   # maybe check if select or another command is there already?
    result = dt.execute(sqlquery, data, commit = False)
    # Convert dates to strings to conform to scraperwiki classic
    if result != []:
      keys = result[0].keys()
      for row in result:
        for key in keys:
          if isinstance(row[key], datetime.date):
            row[key] = str(row[key])
    return result

def show_tables(dbname=""):
    name = "sqlite_master"
    if dbname:
        name = "`%s`.%s" % (dbname, name)
    response = select('name, sql from %s where type = "table";' % name)
    return {row['name']: row['sql'] for row in response}

def save_var(name, value, verbose=2):
    data = dt.save_var(name, value)
    dt.execute(u"CREATE TABLE IF NOT EXISTS swvariables (`value_blob` blob, `type` text, `name` text PRIMARY KEY)", commit = False)
    dt.execute(u'INSERT OR REPLACE INTO swvariables SELECT `value`, `type`, `key` FROM `%s`' % dt._DumpTruck__vars_table, commit = False)
    dt.execute(u'DROP TABLE `%s`' % dt._DumpTruck__vars_table, commit = False)
    dt.commit()
    return data

def get_var(name, default=None, verbose=2):
    if 'swvariables' not in show_tables(): # this should be unecessary
        return default
    dt.execute(u"CREATE TABLE IF NOT EXISTS swvariables (`value_blob` blob, `type` text, `name` text PRIMARY KEY)", commit = False)
    dt.execute(u"CREATE TEMPORARY TABLE IF NOT EXISTS %s (`value` blob, `type` text, `key` text PRIMARY KEY)" % dt._DumpTruck__vars_table, commit = False)

    sql = u'INSERT INTO `%s` (value, type, key) SELECT `value_blob`, `type`, `name` FROM `swvariables`' % dt._DumpTruck__vars_table
    dt.execute(sql, commit = False)
    try:
        value = dt.get_var(name)
    except NameError:
        dt.connection.rollback()
        return default
    dt.execute(u'DROP TABLE `%s`' % dt._DumpTruck__vars_table, commit = False)
    dt.commit()
    return value
<commit_msg>Add SCRAPERWIKI_DATABASE_NAME env var<commit_after>from dumptruck import DumpTruck
import datetime
import re
import os

DATABASE_NAME = os.environ.get("SCRAPERWIKI_DATABASE_NAME", "scraperwiki.sqlite")

def _connect(dbname = DATABASE_NAME):
  'Initialize the database (again). This is mainly for testing'
  global dt
  dt = DumpTruck(dbname = dbname,  adapt_and_convert = False)

_connect()

def execute(sqlquery, data=[], verbose=1):
    """ Emulate scraperwiki as much as possible by mangling dumptruck result """
    # Allow for a non-list to be passed as data.
    if type(data) != list and type(data) != tuple:
        data = [data]

    result = dt.execute(sqlquery, data, commit=False)
    # None (non-select) and empty list (select) results
    if not result:
        return {u'data': [], u'keys': []}
    dtos = lambda d: str(d) if isinstance(d, datetime.date) else d
    # Select statement with results
    return {u'data': map(lambda row: map(dtos, row.values()), result),
            u'keys': result[0].keys()}

def save(unique_keys, data, table_name="swdata", verbose=2, date=None):
    if not data:
        return
    dt.create_table(data, table_name = table_name, error_if_exists = False)
    if unique_keys != []:
        dt.create_index(unique_keys, table_name, unique = True, if_not_exists = True)
    return dt.upsert(data, table_name = table_name)

def commit(verbose=1):
    dt.commit()

def select(sqlquery, data=[], verbose=1):
    sqlquery = "select %s" % sqlquery   # maybe check if select or another command is there already?
    result = dt.execute(sqlquery, data, commit = False)
    # Convert dates to strings to conform to scraperwiki classic
    if result != []:
      keys = result[0].keys()
      for row in result:
        for key in keys:
          if isinstance(row[key], datetime.date):
            row[key] = str(row[key])
    return result

def show_tables(dbname=""):
    name = "sqlite_master"
    if dbname:
        name = "`%s`.%s" % (dbname, name)
    response = select('name, sql from %s where type = "table";' % name)
    return {row['name']: row['sql'] for row in response}

def save_var(name, value, verbose=2):
    data = dt.save_var(name, value)
    dt.execute(u"CREATE TABLE IF NOT EXISTS swvariables (`value_blob` blob, `type` text, `name` text PRIMARY KEY)", commit = False)
    dt.execute(u'INSERT OR REPLACE INTO swvariables SELECT `value`, `type`, `key` FROM `%s`' % dt._DumpTruck__vars_table, commit = False)
    dt.execute(u'DROP TABLE `%s`' % dt._DumpTruck__vars_table, commit = False)
    dt.commit()
    return data

def get_var(name, default=None, verbose=2):
    if 'swvariables' not in show_tables(): # this should be unecessary
        return default
    dt.execute(u"CREATE TABLE IF NOT EXISTS swvariables (`value_blob` blob, `type` text, `name` text PRIMARY KEY)", commit = False)
    dt.execute(u"CREATE TEMPORARY TABLE IF NOT EXISTS %s (`value` blob, `type` text, `key` text PRIMARY KEY)" % dt._DumpTruck__vars_table, commit = False)

    sql = u'INSERT INTO `%s` (value, type, key) SELECT `value_blob`, `type`, `name` FROM `swvariables`' % dt._DumpTruck__vars_table
    dt.execute(sql, commit = False)
    try:
        value = dt.get_var(name)
    except NameError:
        dt.connection.rollback()
        return default
    dt.execute(u'DROP TABLE `%s`' % dt._DumpTruck__vars_table, commit = False)
    dt.commit()
    return value
<|endoftext|>


<commit_before>import traceback
import uuid
from cStringIO import StringIO

from twisted.python import log

from scrapy.utils.txweb import JsonResource
from .utils import get_spider_list

class WsResource(JsonResource):

    def __init__(self, root):
        JsonResource.__init__(self)
        self.root = root

    def render(self, txrequest):
        try:
            return JsonResource.render(self, txrequest)
        except Exception, e:
            if self.root.debug:
                return traceback.format_exc()
            log.err()
            r = {"status": "error", "message": str(e)}
            return self.render_object(r, txrequest)

class Schedule(WsResource):

    def render_POST(self, txrequest):
        settings = txrequest.args.pop('setting', [])
        settings = dict(x.split('=', 1) for x in settings)
        args = dict((k, v[0]) for k, v in txrequest.args.items())
        project = args.pop('project')
        spider = args.pop('spider')
        args['settings'] = settings
        jobid = uuid.uuid1().hex
        args['_job'] = jobid
        self.root.scheduler.schedule(project, spider, **args)
        return {"status": "ok", "jobid": jobid}

class Cancel(WsResource):

    def render_POST(self, txrequest):
        args = dict((k, v[0]) for k, v in txrequest.args.items())
        project = args['project']
        jobid = args['job']
        signal = args.get('signal', 'TERM')
        prevstate = None
        queue = self.root.poller.queues[project]
        c = queue.remove(lambda x: x["_job"] == jobid)
        if c:
            prevstate = "pending"
        spiders = self.root.launcher.processes.values()
        for s in spiders:
            if s.job == jobid:
                s.transport.signalProcess(signal)
                prevstate = "running"
        return {"status": "ok", "prevstate": prevstate}

class AddVersion(WsResource):

    def render_POST(self, txrequest):
        project = txrequest.args['project'][0]
        version = txrequest.args['version'][0]
        eggf = StringIO(txrequest.args['egg'][0])
        self.root.eggstorage.put(eggf, project, version)
        spiders = get_spider_list(project)
        self.root.update_projects()
        return {"status": "ok", "project": project, "version": version, \
            "spiders": len(spiders)}

class ListProjects(WsResource):

    def render_GET(self, txrequest):
        projects = self.root.scheduler.list_projects()
        return {"status": "ok", "projects": projects}

class ListVersions(WsResource):

    def render_GET(self, txrequest):
        project = txrequest.args['project'][0]
        versions = self.root.eggstorage.list(project)
        return {"status": "ok", "versions": versions}

class ListSpiders(WsResource):

    def render_GET(self, txrequest):
        project = txrequest.args['project'][0]
        spiders = get_spider_list(project, runner=self.root.runner)
        return {"status": "ok", "spiders": spiders}

class ListJobs(WsResource):

    def render_GET(self, txrequest):
        project = txrequest.args['project'][0]
        spiders = self.root.launcher.processes.values()
        running = [{"id": s.job, "spider": s.spider} for s in spiders if s.project == project]
        queue = self.root.poller.queues[project]
        pending = [{"id": x["_job"], "spider": x["name"]} for x in queue.list()]
        finished = [{"id": s.job, "spider": s.spider,
            "start_time": s.start_time.isoformat(' '),
            "end_time": s.end_time.isoformat(' ')} for s in self.root.launcher.finished
            if s.project == project]
        return {"status":"ok", "pending": pending, "running": running, "finished": finished}

class DeleteProject(WsResource):

    def render_POST(self, txrequest):
        project = txrequest.args['project'][0]
        self._delete_version(project)
        return {"status": "ok"}

    def _delete_version(self, project, version=None):
        self.root.eggstorage.delete(project, version)
        self.root.update_projects()

class DeleteVersion(DeleteProject):

    def render_POST(self, txrequest):
        project = txrequest.args['project'][0]
        version = txrequest.args['version'][0]
        self._delete_version(project, version)
        return {"status": "ok"}
<commit_msg>check if a spider exists before schedule it<commit_after>import traceback
import uuid
from cStringIO import StringIO

from twisted.python import log

from scrapy.utils.txweb import JsonResource
from .utils import get_spider_list

class WsResource(JsonResource):

    def __init__(self, root):
        JsonResource.__init__(self)
        self.root = root

    def render(self, txrequest):
        try:
            return JsonResource.render(self, txrequest)
        except Exception, e:
            if self.root.debug:
                return traceback.format_exc()
            log.err()
            r = {"status": "error", "message": str(e)}
            return self.render_object(r, txrequest)

class Schedule(WsResource):

    def render_POST(self, txrequest):
        settings = txrequest.args.pop('setting', [])
        settings = dict(x.split('=', 1) for x in settings)
        args = dict((k, v[0]) for k, v in txrequest.args.items())
        project = args.pop('project')
        spider = args.pop('spider')
        spiders = get_spider_list(project)
        if not spider in spiders:
            return {"status": "error", "message": "spider '%s' not found" % spider}
        args['settings'] = settings
        jobid = uuid.uuid1().hex
        args['_job'] = jobid
        self.root.scheduler.schedule(project, spider, **args)
        return {"status": "ok", "jobid": jobid}

class Cancel(WsResource):

    def render_POST(self, txrequest):
        args = dict((k, v[0]) for k, v in txrequest.args.items())
        project = args['project']
        jobid = args['job']
        signal = args.get('signal', 'TERM')
        prevstate = None
        queue = self.root.poller.queues[project]
        c = queue.remove(lambda x: x["_job"] == jobid)
        if c:
            prevstate = "pending"
        spiders = self.root.launcher.processes.values()
        for s in spiders:
            if s.job == jobid:
                s.transport.signalProcess(signal)
                prevstate = "running"
        return {"status": "ok", "prevstate": prevstate}

class AddVersion(WsResource):

    def render_POST(self, txrequest):
        project = txrequest.args['project'][0]
        version = txrequest.args['version'][0]
        eggf = StringIO(txrequest.args['egg'][0])
        self.root.eggstorage.put(eggf, project, version)
        spiders = get_spider_list(project)
        self.root.update_projects()
        return {"status": "ok", "project": project, "version": version, \
            "spiders": len(spiders)}

class ListProjects(WsResource):

    def render_GET(self, txrequest):
        projects = self.root.scheduler.list_projects()
        return {"status": "ok", "projects": projects}

class ListVersions(WsResource):

    def render_GET(self, txrequest):
        project = txrequest.args['project'][0]
        versions = self.root.eggstorage.list(project)
        return {"status": "ok", "versions": versions}

class ListSpiders(WsResource):

    def render_GET(self, txrequest):
        project = txrequest.args['project'][0]
        spiders = get_spider_list(project, runner=self.root.runner)
        return {"status": "ok", "spiders": spiders}

class ListJobs(WsResource):

    def render_GET(self, txrequest):
        project = txrequest.args['project'][0]
        spiders = self.root.launcher.processes.values()
        running = [{"id": s.job, "spider": s.spider} for s in spiders if s.project == project]
        queue = self.root.poller.queues[project]
        pending = [{"id": x["_job"], "spider": x["name"]} for x in queue.list()]
        finished = [{"id": s.job, "spider": s.spider,
            "start_time": s.start_time.isoformat(' '),
            "end_time": s.end_time.isoformat(' ')} for s in self.root.launcher.finished
            if s.project == project]
        return {"status":"ok", "pending": pending, "running": running, "finished": finished}

class DeleteProject(WsResource):

    def render_POST(self, txrequest):
        project = txrequest.args['project'][0]
        self._delete_version(project)
        return {"status": "ok"}

    def _delete_version(self, project, version=None):
        self.root.eggstorage.delete(project, version)
        self.root.update_projects()

class DeleteVersion(DeleteProject):

    def render_POST(self, txrequest):
        project = txrequest.args['project'][0]
        version = txrequest.args['version'][0]
        self._delete_version(project, version)
        return {"status": "ok"}
<|endoftext|>


<commit_before>from __future__ import division

def reflgen(num_refl):

  from dials.model.data import ReflectionList, Reflection
  from dials.algorithms import shoebox
  import random
  import math

  rlist = ReflectionList(num_refl)

  # FIXME in here generate random cell and random U matrix

  # FIXME decide shape of reflection in reciprocal space for transformation
  # reasons - will need this to assign shoebox size and also generate the
  # basic density

  for j in range(num_refl):
    rlist[j].miller_index = (random.randint(0, 20),
                             random.randint(0, 20),
                             random.randint(0, 20))
    rlist[j].rotation_angle = 2 * math.pi * random.random()

    # FIXME write some prediction code or use some to derive these things
    #
    # P* = RUBh; s1 (beam_vector) = s0 + p*

    rlist[j].beam_vector = (0, 0, 0)
    rlist[j].image_coord_px = (0, 0)
    rlist[j].image_coord_mm = (0, 0)
    rlist[j].frame_number = 0
    rlist[j].panel_number = 0
    rlist[j].bounding_box = (0, 1, 0, 1, 0, 1)
    rlist[j].centroid_position = (0, 0, 0)
    rlist[j].centroid_variance = (0, 0, 0)
    rlist[j].centroid_sq_width = (0, 0, 0)
    rlist[j].intensity = 0
    rlist[j].intensity_variance = 0
    rlist[j].corrected_intensity = 0
    rlist[j].corrected_intensity_variance = 0

  shoebox.allocate(rlist)

  for j in range(num_refl):

    # FIXME in here fake up background and peak and mask but don't assign
    # the "true" values here unless we want to test things? Maybe that
    # would be best ... make sure we can reproduce the observations?

    shoebox = rlist[j].shoebox
    mask = rlist[j].shoebox_mask
    background = rlist[j].shoebox_background

def start(num_refl):
  from dials.model.data import ReflectionList, Reflection
  from dials.algorithms import shoebox
  import random
  import math

  rlist = ReflectionList(num_refl)

  for j in range(num_refl):
    rlist[j].miller_index = (random.randint(0, 20),
                             random.randint(0, 20),
                             random.randint(0, 20))
    rlist[j].rotation_angle = 2 * math.pi * random.random()
    rlist[j].beam_vector = (0, 0, 0)
    rlist[j].image_coord_px = (0, 0)
    rlist[j].image_coord_mm = (0, 0)
    rlist[j].frame_number = 0
    rlist[j].panel_number = 0
    rlist[j].bounding_box = (0, 10, 0, 10, 0, 10)
    rlist[j].centroid_position = (0, 0, 0)
    rlist[j].centroid_variance = (0, 0, 0)
    rlist[j].centroid_sq_width = (0, 0, 0)
    rlist[j].intensity = 0
    rlist[j].intensity_variance = 0
    rlist[j].corrected_intensity = 0
    rlist[j].corrected_intensity_variance = 0

  shoebox.allocate(rlist)

  for refl in rlist:

    counts = random.randint(0, 10000)
    background = random.randint(0, 10000)

    # FIXME in here fake up background and peak and mask but don't assign
    # the "true" values here unless we want to test things? Maybe that
    # would be best ... make sure we can reproduce the observations?

    sbox = refl.shoebox

    x0 = 5
    y0 = 5
    z0 = 5

    # reflection itself

    counts_true = 0
    for j in range(counts):
      x = int(round(random.gauss(x0, 1)))
      y = int(round(random.gauss(x0, 1)))
      z = int(round(random.gauss(x0, 1)))
      if x < 0 or x >= 10:
        continue
      if y < 0 or y >= 10:
        continue
      if z < 0 or z >= 10:
        continue
      sbox[z, y, x] += 1
      counts_true += 1

    refl.intensity = counts_true

    # background:flat

    for j in range(background):
      x = random.randint(0, 9)
      y = random.randint(0, 9)
      z = random.randint(0, 9)
      sbox[z, y, x] += 1

  return rlist

def print_shoebox(sbox):
  ndim = sbox.nd()
  dims = sbox.focus()

  print '-' * 80

  for k in range(dims[0]):
    for j in range(dims[1]):
      for i in range(dims[0]):
        print '%4d' % int(sbox[k, j, i]),
      print
    print '-' * 80

  return

def print_summary(refl):
  print 'Claimed / total intensity: %9.2f %9.2f' % (refl.intensity,
                                                    sum(refl.shoebox))

def main():
  profiles = start(100)
  for p in profiles:
    print_summary(p)

if __name__ == '__main__':
  main()
<commit_msg>Roll in simple summation integration<commit_after>from __future__ import division

def reflgen(num_refl):

  from dials.model.data import ReflectionList, Reflection
  from dials.algorithms import shoebox
  import random
  import math

  rlist = ReflectionList(num_refl)

  # FIXME in here generate random cell and random U matrix

  # FIXME decide shape of reflection in reciprocal space for transformation
  # reasons - will need this to assign shoebox size and also generate the
  # basic density

  for j in range(num_refl):
    rlist[j].miller_index = (random.randint(0, 20),
                             random.randint(0, 20),
                             random.randint(0, 20))
    rlist[j].rotation_angle = 2 * math.pi * random.random()

    # FIXME write some prediction code or use some to derive these things
    #
    # P* = RUBh; s1 (beam_vector) = s0 + p*

    rlist[j].beam_vector = (0, 0, 0)
    rlist[j].image_coord_px = (0, 0)
    rlist[j].image_coord_mm = (0, 0)
    rlist[j].frame_number = 0
    rlist[j].panel_number = 0
    rlist[j].bounding_box = (0, 1, 0, 1, 0, 1)
    rlist[j].centroid_position = (0, 0, 0)
    rlist[j].centroid_variance = (0, 0, 0)
    rlist[j].centroid_sq_width = (0, 0, 0)
    rlist[j].intensity = 0
    rlist[j].intensity_variance = 0
    rlist[j].corrected_intensity = 0
    rlist[j].corrected_intensity_variance = 0

  shoebox.allocate(rlist)

  for j in range(num_refl):

    # FIXME in here fake up background and peak and mask but don't assign
    # the "true" values here unless we want to test things? Maybe that
    # would be best ... make sure we can reproduce the observations?

    shoebox = rlist[j].shoebox
    mask = rlist[j].shoebox_mask
    background = rlist[j].shoebox_background

def start(num_refl):
  from dials.model.data import ReflectionList, Reflection
  from dials.algorithms import shoebox
  import random
  import math

  rlist = ReflectionList(num_refl)

  for j in range(num_refl):
    rlist[j].miller_index = (random.randint(0, 20),
                             random.randint(0, 20),
                             random.randint(0, 20))
    rlist[j].rotation_angle = 2 * math.pi * random.random()
    rlist[j].beam_vector = (0, 0, 0)
    rlist[j].image_coord_px = (0, 0)
    rlist[j].image_coord_mm = (0, 0)
    rlist[j].frame_number = 0
    rlist[j].panel_number = 0
    rlist[j].bounding_box = (0, 10, 0, 10, 0, 10)
    rlist[j].centroid_position = (0, 0, 0)
    rlist[j].centroid_variance = (0, 0, 0)
    rlist[j].centroid_sq_width = (0, 0, 0)
    rlist[j].intensity = 0
    rlist[j].intensity_variance = 0
    rlist[j].corrected_intensity = 0
    rlist[j].corrected_intensity_variance = 0

  shoebox.allocate(rlist)

  for refl in rlist:

    counts = random.randint(0, 10000)
    background = random.randint(0, 10000)

    # FIXME in here fake up background and peak and mask but don't assign
    # the "true" values here unless we want to test things? Maybe that
    # would be best ... make sure we can reproduce the observations?

    sbox = refl.shoebox

    x0 = 5
    y0 = 5
    z0 = 5

    # reflection itself

    counts_true = 0
    for j in range(counts):
      x = int(round(random.gauss(x0, 1)))
      y = int(round(random.gauss(x0, 1)))
      z = int(round(random.gauss(x0, 1)))
      if x < 0 or x >= 10:
        continue
      if y < 0 or y >= 10:
        continue
      if z < 0 or z >= 10:
        continue
      sbox[z, y, x] += 1
      counts_true += 1

    refl.intensity = counts_true

    # background:flat

    for j in range(background):
      x = random.randint(0, 9)
      y = random.randint(0, 9)
      z = random.randint(0, 9)
      sbox[z, y, x] += 1

  # set the mask
  from dials.algorithms.shoebox import MaskCode
  mask_value = MaskCode.Valid|MaskCode.Foreground|MaskCode.Background
  for refl in rlist:
    mask = refl.shoebox_mask
    for j in range(len(mask)):
      mask[j] = mask_value

  return rlist

def print_shoebox(sbox):
  ndim = sbox.nd()
  dims = sbox.focus()

  print '-' * 80

  for k in range(dims[0]):
    for j in range(dims[1]):
      for i in range(dims[0]):
        print '%4d' % int(sbox[k, j, i]),
      print
    print '-' * 80

  return

def print_summary(refl):
  print 'Claimed / total intensity: %9.2f %9.2f' % (refl.intensity,
                                                    sum(refl.shoebox))

def integrate(rlist):

  from dials.algorithms.background import XdsSubtractor
  from dials.algorithms.integration import Summation3d

  background = XdsSubtractor()
  background(None, None, rlist)
  integration = Summation3d()
  integration(None, None, rlist)

def main():
  rlist = start(100)
  correct_intensities = [r.intensity for r in rlist]
  for r in rlist:
    r.intensity = 0
  integrate(rlist)
  integrated_intensities = [r.intensity for r in rlist]

  for c, i in zip(correct_intensities, integrated_intensities):
    print c, i

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import csv
import json

def disp_as(asn):
    if as_info.has_key(asn):
        trans_ref = as_trans.get(asn)
        if trans_ref == None:
            trans_info = [ '-', '-' ]
        else:
            trans_info = [ as_trans[asn]['t'], as_trans[asn]['g'] ]
        info = "{} [{}](t: {} g: {})".format(
                    as_info[asn]['short_descr'], asn, trans_info[0],
                    trans_info[1])
    else:
        info = asn

    return info

# Load the short names for the ASes
with open('data/as-info.json', 'rb') as as_info_file:
    as_info = json.load(as_info_file)

# Load transit degrees
# with open('as-rank/as-transit-degree.json', 'r') as as_trans_file:
#     as_trans = json.load(as_trans_file)
as_trans = {}
with open('as-rank/20140601.degrees.txt', 'r') as as_trans_file:
    as_trans_data = csv.reader(filter(lambda row: row[0]!='#',as_trans_file), delimiter=" ")
    for asn, transit_deg, global_deg in as_trans_data:
        as_trans[asn] = dict(t=transit_deg, g=global_deg)

with open('local-as-rel-info.csv', 'rb') as rel_file:
    rel_info = csv.reader(rel_file, delimiter="|")

    for src,dst,rel in rel_info:
        if rel != '?':
            continue
        print "{0} -> {1}".format(disp_as(src), disp_as(dst))
<commit_msg>Local AS relationship file was pointing to its old location<commit_after>#!/usr/bin/env python

import csv
import json

def disp_as(asn):
    if as_info.has_key(asn):
        trans_ref = as_trans.get(asn)
        if trans_ref == None:
            trans_info = [ '-', '-' ]
        else:
            trans_info = [ as_trans[asn]['t'], as_trans[asn]['g'] ]
        info = "{} [{}](t: {} g: {})".format(
                    as_info[asn]['short_descr'], asn, trans_info[0],
                    trans_info[1])
    else:
        info = asn

    return info

# Load the short names for the ASes
with open('data/as-info.json', 'rb') as as_info_file:
    as_info = json.load(as_info_file)

# Load transit degrees
# with open('as-rank/as-transit-degree.json', 'r') as as_trans_file:
#     as_trans = json.load(as_trans_file)
as_trans = {}
with open('as-rank/20140601.degrees.txt', 'r') as as_trans_file:
    as_trans_data = csv.reader(filter(lambda row: row[0]!='#',as_trans_file), delimiter=" ")
    for asn, transit_deg, global_deg in as_trans_data:
        as_trans[asn] = dict(t=transit_deg, g=global_deg)

with open('data/local-as-rel-info.csv', 'rb') as rel_file:
    rel_info = csv.reader(rel_file, delimiter="|")

    for src,dst,rel in rel_info:
        if rel != '?':
            continue
        print "{0} -> {1}".format(disp_as(src), disp_as(dst))
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
__author__ = 'essepuntato'

# Official configuration
base_home = "/srv/oc/"
base_dir = base_home + "corpus/"
base_iri = "https://w3id.org/oc/corpus/"
triplestore_url = "http://localhost:3000/blazegraph"
context_path = "https://w3id.org/oc/corpus/context.json"
context_file_path = "/home/essepuntato/OC/corpus/context.json"
info_dir = base_home + "id-counter/"
temp_dir_for_rdf_loading = base_home
orcid_conf_path = "/home/essepuntato/OC/script/conf.json"
reference_dir = base_home + "ref/todo/"
reference_dir_error = base_home + "ref/err/"
reference_dir_done = base_home + "ref/done/"
dataset_home = "http://opencitations.net/"

<commit_msg>SPACIN Conf updated<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
__author__ = 'essepuntato'

# Official configuration
base_home = "/srv/oc/"
base_dir = base_home + "corpus/"
base_iri = "https://w3id.org/oc/corpus/"
triplestore_url = "http://localhost:3000/blazegraph/sparql"
context_path = "https://w3id.org/oc/corpus/context.json"
context_file_path = "/home/essepuntato/OC/corpus/context.json"
info_dir = base_home + "id-counter/"
temp_dir_for_rdf_loading = base_home
orcid_conf_path = "/home/essepuntato/OC/script/conf.json"
reference_dir = base_home + "ref/todo/"
reference_dir_error = base_home + "ref/err/"
reference_dir_done = base_home + "ref/done/"
dataset_home = "http://opencitations.net/"

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('huntserver', '0021_auto_20180402_2224'),
    ]

    operations = [
        migrations.RunSQL(
            sql=[
                'ALTER TABLE `auth_user` MODIFY `password` varchar(128) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `auth_user` MODIFY `username` varchar(30) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `auth_user` MODIFY `first_name` varchar(30) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `auth_user` MODIFY `last_name` varchar(30) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `auth_user` MODIFY `email` varchar(254) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `django_admin_log` MODIFY `object_id` longtext CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `django_admin_log` MODIFY `object_repr` varchar(200) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `django_admin_log` MODIFY `change_message` longtext CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_hunt` MODIFY `hunt_name` varchar(200) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_hunt` MODIFY `location` varchar(100) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_hunt` MODIFY `template` longtext CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_huntassetfile` MODIFY `file` varchar(100) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_message` MODIFY `text` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_person` MODIFY `phone` varchar(20) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_person` MODIFY `comments` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_person` MODIFY `allergies` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `puzzle_name` varchar(200) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `puzzle_id` varchar(8) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `answer` varchar(100) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `link` varchar(200) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_response` MODIFY `regex` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_response` MODIFY `text` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_submission` MODIFY `submission_text` varchar(100) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_submission` MODIFY `response_text` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_team` MODIFY `team_name` varchar(200) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_team` MODIFY `location` varchar(80) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_team` MODIFY `join_code` varchar(5) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_unlockable` MODIFY `content_type` varchar(3) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_unlockable` MODIFY `content` varchar(500) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
            ],
            reverse_sql=[
                'ALTER TABLE `auth_user` MODIFY `password` varchar(128) ',
                'ALTER TABLE `auth_user` MODIFY `username` varchar(30) ',
                'ALTER TABLE `auth_user` MODIFY `first_name` varchar(30) ',
                'ALTER TABLE `auth_user` MODIFY `last_name` varchar(30) ',
                'ALTER TABLE `auth_user` MODIFY `email` varchar(254) ',
                'ALTER TABLE `django_admin_log` MODIFY `object_id` longtext ',
                'ALTER TABLE `django_admin_log` MODIFY `object_repr` varchar(200) ',
                'ALTER TABLE `django_admin_log` MODIFY `change_message` longtext ',
                'ALTER TABLE `huntserver_hunt` MODIFY `hunt_name` varchar(200) ',
                'ALTER TABLE `huntserver_hunt` MODIFY `location` varchar(100) ',
                'ALTER TABLE `huntserver_hunt` MODIFY `template` longtext ',
                'ALTER TABLE `huntserver_huntassetfile` MODIFY `file` varchar(100) ',
                'ALTER TABLE `huntserver_message` MODIFY `text` varchar(400) ',
                'ALTER TABLE `huntserver_person` MODIFY `phone` varchar(20) ',
                'ALTER TABLE `huntserver_person` MODIFY `comments` varchar(400) ',
                'ALTER TABLE `huntserver_person` MODIFY `allergies` varchar(400) ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `puzzle_name` varchar(200) ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `puzzle_id` varchar(8) ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `answer` varchar(100) ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `link` varchar(200) ',
                'ALTER TABLE `huntserver_response` MODIFY `regex` varchar(400) ',
                'ALTER TABLE `huntserver_response` MODIFY `text` varchar(400) ',
                'ALTER TABLE `huntserver_submission` MODIFY `submission_text` varchar(100) ',
                'ALTER TABLE `huntserver_submission` MODIFY `response_text` varchar(400) ',
                'ALTER TABLE `huntserver_team` MODIFY `team_name` varchar(200) ',
                'ALTER TABLE `huntserver_team` MODIFY `location` varchar(80) ',
                'ALTER TABLE `huntserver_team` MODIFY `join_code` varchar(5) ',
                'ALTER TABLE `huntserver_unlockable` MODIFY `content_type` varchar(3) ',
                'ALTER TABLE `huntserver_unlockable` MODIFY `content` varchar(500) ',
            ]
        ),
    ]
<commit_msg>added helpful text to migration relevant to existing servers<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

def printHelp(apps, schema_editor):
    print("=== NOTICE: ===")
    print("Do not forget to add the line")
    print("'OPTIONS': {'charset': 'utf8mb4'},")
    print("to your database settings if applying this")
    print("migration to an existing database/server.")
    print("===============")

class Migration(migrations.Migration):

    dependencies = [
        ('huntserver', '0021_auto_20180402_2224'),
    ]

    operations = [
        migrations.RunSQL(
            sql=[
                'ALTER TABLE `auth_user` MODIFY `password` varchar(128) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `auth_user` MODIFY `username` varchar(30) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `auth_user` MODIFY `first_name` varchar(30) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `auth_user` MODIFY `last_name` varchar(30) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `auth_user` MODIFY `email` varchar(254) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `django_admin_log` MODIFY `object_id` longtext CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `django_admin_log` MODIFY `object_repr` varchar(200) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `django_admin_log` MODIFY `change_message` longtext CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_hunt` MODIFY `hunt_name` varchar(200) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_hunt` MODIFY `location` varchar(100) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_hunt` MODIFY `template` longtext CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_huntassetfile` MODIFY `file` varchar(100) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_message` MODIFY `text` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_person` MODIFY `phone` varchar(20) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_person` MODIFY `comments` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_person` MODIFY `allergies` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `puzzle_name` varchar(200) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `puzzle_id` varchar(8) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `answer` varchar(100) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `link` varchar(200) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_response` MODIFY `regex` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_response` MODIFY `text` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_submission` MODIFY `submission_text` varchar(100) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_submission` MODIFY `response_text` varchar(400) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_team` MODIFY `team_name` varchar(200) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_team` MODIFY `location` varchar(80) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_team` MODIFY `join_code` varchar(5) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_unlockable` MODIFY `content_type` varchar(3) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
                'ALTER TABLE `huntserver_unlockable` MODIFY `content` varchar(500) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci ',
            ],
            reverse_sql=[
                'ALTER TABLE `auth_user` MODIFY `password` varchar(128) ',
                'ALTER TABLE `auth_user` MODIFY `username` varchar(30) ',
                'ALTER TABLE `auth_user` MODIFY `first_name` varchar(30) ',
                'ALTER TABLE `auth_user` MODIFY `last_name` varchar(30) ',
                'ALTER TABLE `auth_user` MODIFY `email` varchar(254) ',
                'ALTER TABLE `django_admin_log` MODIFY `object_id` longtext ',
                'ALTER TABLE `django_admin_log` MODIFY `object_repr` varchar(200) ',
                'ALTER TABLE `django_admin_log` MODIFY `change_message` longtext ',
                'ALTER TABLE `huntserver_hunt` MODIFY `hunt_name` varchar(200) ',
                'ALTER TABLE `huntserver_hunt` MODIFY `location` varchar(100) ',
                'ALTER TABLE `huntserver_hunt` MODIFY `template` longtext ',
                'ALTER TABLE `huntserver_huntassetfile` MODIFY `file` varchar(100) ',
                'ALTER TABLE `huntserver_message` MODIFY `text` varchar(400) ',
                'ALTER TABLE `huntserver_person` MODIFY `phone` varchar(20) ',
                'ALTER TABLE `huntserver_person` MODIFY `comments` varchar(400) ',
                'ALTER TABLE `huntserver_person` MODIFY `allergies` varchar(400) ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `puzzle_name` varchar(200) ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `puzzle_id` varchar(8) ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `answer` varchar(100) ',
                'ALTER TABLE `huntserver_puzzle` MODIFY `link` varchar(200) ',
                'ALTER TABLE `huntserver_response` MODIFY `regex` varchar(400) ',
                'ALTER TABLE `huntserver_response` MODIFY `text` varchar(400) ',
                'ALTER TABLE `huntserver_submission` MODIFY `submission_text` varchar(100) ',
                'ALTER TABLE `huntserver_submission` MODIFY `response_text` varchar(400) ',
                'ALTER TABLE `huntserver_team` MODIFY `team_name` varchar(200) ',
                'ALTER TABLE `huntserver_team` MODIFY `location` varchar(80) ',
                'ALTER TABLE `huntserver_team` MODIFY `join_code` varchar(5) ',
                'ALTER TABLE `huntserver_unlockable` MODIFY `content_type` varchar(3) ',
                'ALTER TABLE `huntserver_unlockable` MODIFY `content` varchar(500) ',
            ]
        ),
        migrations.RunPython(printHelp),
    ]
<|endoftext|>


<commit_before>import os
import h5py
import pickle
import numpy as np
from sklearn.decomposition import PCA
from sklearn.neighbors import NearestNeighbors

from flask import current_app

keys = None
features = None
nneighs = None

# def load_features(featuresfile):
#     keys, X = [], []

#     with h5py.File(featuresfile, 'r') as f:
#         for key in f.keys():
#             xx = f[key][...]
#             if np.any(np.isnan(xx)):
#                 print('{} has NaN values in {}'.format(key, featuresfile))
#                 continue
#             keys.append(int(key))
#             X.append(f[key][...])

#     return keys, np.array(X)

def load_features(featuresfile, perplexity=40):
    keys, X = [], []

    with h5py.File(featuresfile, 'r') as f:

        if 'tsne' in featuresfile:
            g = f['perplexity-{}'.format(perplexity)]
        else:
            g = f

        for key in g.keys():
            xx = g[key][...]
            keys.append(int(key))
            X.append(g[key][...])

    return keys, np.array(X)

def reload_features(featuresfile, keys, perplexity=40):
    X = []

    with h5py.File(featuresfile, 'r') as f:
        if 'tsne' in featuresfile:
            g = f['perplexity-{}'.format(perplexity)]
        else:
            g = f

        for key in keys:
            xx = g[key][...]
            X.append(g[key][...])

    return np.array(X)

def build_search_tree(datadir, featurename='vgg16_block5_conv3-vlad-64.h5'):

    ndim = 64
    features_file = os.path.join(datadir, featurename)
    print(features_file)

    global keys, features
    keys, features = load_features(features_file)

    print('reducing features')
    pca = PCA(n_components=ndim)
    features = pca.fit_transform(features)
    print('ready')

    print('building search tree')
    nn = NearestNeighbors()

    global nneighs
    nneighs = nn.fit(features)
    print('ready')

def query(entry_id, n_results=16):
    scikit_id = keys.index(entry_id)
    query_vector = features[scikit_id]

    # nearest neighbor will be a self-match
    scores, results = nneighs.kneighbors(query_vector, n_results+1)
    scores, results = scores.flatten()[1:], results.flatten()[1:]

    result_entries = map(keys.__getitem__, results)
    scores = ['{:0.4f}'.format(score) for score in scores]

    return scores, result_entries

<commit_msg>remove dead code from features.py<commit_after>import os
import h5py
import pickle
import numpy as np
from sklearn.decomposition import PCA
from sklearn.neighbors import NearestNeighbors

from flask import current_app

keys = None
features = None
nneighs = None

def load_features(featuresfile, perplexity=40):
    keys, X = [], []

    with h5py.File(featuresfile, 'r') as f:

        if 'tsne' in featuresfile:
            g = f['perplexity-{}'.format(perplexity)]
        else:
            g = f

        for key in g.keys():
            xx = g[key][...]
            keys.append(int(key))
            X.append(g[key][...])

    return keys, np.array(X)

def reload_features(featuresfile, keys, perplexity=40):
    X = []

    with h5py.File(featuresfile, 'r') as f:
        if 'tsne' in featuresfile:
            g = f['perplexity-{}'.format(perplexity)]
        else:
            g = f

        for key in keys:
            xx = g[key][...]
            X.append(g[key][...])

    return np.array(X)

def build_search_tree(datadir, featurename='vgg16_block5_conv3-vlad-64.h5'):

    ndim = 64
    features_file = os.path.join(datadir, featurename)
    print(features_file)

    global keys, features
    keys, features = load_features(features_file)

    print('reducing features')
    pca = PCA(n_components=ndim)
    features = pca.fit_transform(features)
    print('ready')

    print('building search tree')
    nn = NearestNeighbors()

    global nneighs
    nneighs = nn.fit(features)
    print('ready')

def query(entry_id, n_results=16):
    scikit_id = keys.index(entry_id)
    query_vector = features[scikit_id]

    # nearest neighbor will be a self-match
    scores, results = nneighs.kneighbors(query_vector, n_results+1)
    scores, results = scores.flatten()[1:], results.flatten()[1:]

    result_entries = map(keys.__getitem__, results)
    scores = ['{:0.4f}'.format(score) for score in scores]

    return scores, result_entries

<|endoftext|>


<commit_before>import math
import os
import signal
import sys
import threading
import time

import psutil

from common.constants import bcolors
from common.log import logUtils as log
from constants import serverPackets
from helpers import consoleHelper
from objects import glob

def dispose():
	"""
	Perform some clean up. Called on shutdown.

	:return:
	"""
	print("> Disposing server... ")
	glob.fileBuffers.flushAll()
	consoleHelper.printColored("Goodbye!", bcolors.GREEN)

def runningUnderUnix():
	"""
	Get if the server is running under UNIX or NT

	:return: True if running under UNIX, otherwise False
	"""
	return True if os.name == "posix" else False

def scheduleShutdown(sendRestartTime, restart, message = "", delay=20):
	"""
	Schedule a server shutdown/restart

	:param sendRestartTime: time (seconds) to wait before sending server restart packets to every client
	:param restart: if True, server will restart. if False, server will shudown
	:param message: if set, send that message to every client to warn about the shutdown/restart
	:param delay: additional restart delay in seconds. Default: 20
	:return:
	"""
	# Console output
	log.info("Pep.py will {} in {} seconds!".format("restart" if restart else "shutdown", sendRestartTime+delay))
	log.info("Sending server restart packets in {} seconds...".format(sendRestartTime))

	# Send notification if set
	if message != "":
		glob.streams.broadcast("main", serverPackets.notification(message))

	# Schedule server restart packet
	threading.Timer(sendRestartTime, glob.streams.broadcast, ["main", serverPackets.banchoRestart(delay*2*1000)]).start()
	glob.restarting = True

	# Restart/shutdown
	if restart:
		action = restartServer
	else:
		action = shutdownServer

	# Schedule actual server shutdown/restart some seconds after server restart packet, so everyone gets it
	threading.Timer(sendRestartTime+delay, action).start()

def restartServer():
	"""
	Restart pep.py

	:return:
	"""
	log.info("Restarting pep.py...")
	dispose()
	os.execv(sys.executable, [sys.executable] + sys.argv)

def shutdownServer():
	"""
	Shutdown pep.py

	:return:
	"""
	log.info("Shutting down pep.py...")
	dispose()
	sig = signal.SIGKILL if runningUnderUnix() else signal.CTRL_C_EVENT
	os.kill(os.getpid(), sig)

def getSystemInfo():
	"""
	Get a dictionary with some system/server info

	:return: ["unix", "connectedUsers", "webServer", "cpuUsage", "totalMemory", "usedMemory", "loadAverage"]
	"""
	data = {"unix": runningUnderUnix(), "connectedUsers": len(glob.tokens.tokens), "matches": len(glob.matches.matches)}

	# General stats
	delta = time.time()-glob.startTime
	days = math.floor(delta/86400)
	delta -= days*86400

	hours = math.floor(delta/3600)
	delta -= hours*3600

	minutes = math.floor(delta/60)
	delta -= minutes*60

	seconds = math.floor(delta)

	data["uptime"] = "{}d {}h {}m {}s".format(days, hours, minutes, seconds)
	data["cpuUsage"] = psutil.cpu_percent()
	memory = psutil.virtual_memory()
	data["totalMemory"] = "{0:.2f}".format(memory.total/1074000000)
	data["usedMemory"] = "{0:.2f}".format(memory.active/1074000000)

	# Unix only stats
	if data["unix"]:
		data["loadAverage"] = os.getloadavg()
	else:
		data["loadAverage"] = (0,0,0)

	return data
<commit_msg>.BANCHO. Removed some schiavo logs<commit_after>import math
import os
import signal
import sys
import threading
import time

import psutil

from common.constants import bcolors
from common.log import logUtils as log
from constants import serverPackets
from helpers import consoleHelper
from objects import glob

def dispose():
	"""
	Perform some clean up. Called on shutdown.

	:return:
	"""
	print("> Disposing server... ")
	glob.fileBuffers.flushAll()
	consoleHelper.printColored("Goodbye!", bcolors.GREEN)

def runningUnderUnix():
	"""
	Get if the server is running under UNIX or NT

	:return: True if running under UNIX, otherwise False
	"""
	return True if os.name == "posix" else False

def scheduleShutdown(sendRestartTime, restart, message = "", delay=20):
	"""
	Schedule a server shutdown/restart

	:param sendRestartTime: time (seconds) to wait before sending server restart packets to every client
	:param restart: if True, server will restart. if False, server will shudown
	:param message: if set, send that message to every client to warn about the shutdown/restart
	:param delay: additional restart delay in seconds. Default: 20
	:return:
	"""
	# Console output
	log.info("Pep.py will {} in {} seconds!".format("restart" if restart else "shutdown", sendRestartTime+delay), "bunker")
	log.info("Sending server restart packets in {} seconds...".format(sendRestartTime))

	# Send notification if set
	if message != "":
		glob.streams.broadcast("main", serverPackets.notification(message))

	# Schedule server restart packet
	threading.Timer(sendRestartTime, glob.streams.broadcast, ["main", serverPackets.banchoRestart(delay*2*1000)]).start()
	glob.restarting = True

	# Restart/shutdown
	if restart:
		action = restartServer
	else:
		action = shutdownServer

	# Schedule actual server shutdown/restart some seconds after server restart packet, so everyone gets it
	threading.Timer(sendRestartTime+delay, action).start()

def restartServer():
	"""
	Restart pep.py

	:return:
	"""
	log.info("Restarting pep.py...")
	dispose()
	os.execv(sys.executable, [sys.executable] + sys.argv)

def shutdownServer():
	"""
	Shutdown pep.py

	:return:
	"""
	log.info("Shutting down pep.py...")
	dispose()
	sig = signal.SIGKILL if runningUnderUnix() else signal.CTRL_C_EVENT
	os.kill(os.getpid(), sig)

def getSystemInfo():
	"""
	Get a dictionary with some system/server info

	:return: ["unix", "connectedUsers", "webServer", "cpuUsage", "totalMemory", "usedMemory", "loadAverage"]
	"""
	data = {"unix": runningUnderUnix(), "connectedUsers": len(glob.tokens.tokens), "matches": len(glob.matches.matches)}

	# General stats
	delta = time.time()-glob.startTime
	days = math.floor(delta/86400)
	delta -= days*86400

	hours = math.floor(delta/3600)
	delta -= hours*3600

	minutes = math.floor(delta/60)
	delta -= minutes*60

	seconds = math.floor(delta)

	data["uptime"] = "{}d {}h {}m {}s".format(days, hours, minutes, seconds)
	data["cpuUsage"] = psutil.cpu_percent()
	memory = psutil.virtual_memory()
	data["totalMemory"] = "{0:.2f}".format(memory.total/1074000000)
	data["usedMemory"] = "{0:.2f}".format(memory.active/1074000000)

	# Unix only stats
	if data["unix"]:
		data["loadAverage"] = os.getloadavg()
	else:
		data["loadAverage"] = (0,0,0)

	return data
<|endoftext|>


<commit_before>"""Custom request handlers for pushing data to connected clients."""

import logging
from tornado import gen
from tornado.web import RequestHandler
from tornado.websocket import WebSocketHandler, WebSocketClosedError
from tornado.queues import Queue
from tornado.iostream import StreamClosedError
from tornado.log import access_log
from . import stores

logger = logging.getLogger('tornadose.handlers')

class BaseHandler(RequestHandler):
    """Base handler for subscribers. To be compatible with data stores
    defined in :mod:`tornadose.stores`, custom handlers should inherit
    this class and implement the :meth:`submit` and :meth:`publish`
    methods.

    """
    def initialize(self, store):
        """Common initialization of handlers happens here. If additional
        initialization is required, this method *must* be called with
        ``super`` in order to properly initialize the handler.

        """
        assert isinstance(store, stores.BaseStore)
        self.store = store
        self.store.register(self.store)

    def submit(self, message):
        """Submit a new message to be published. This method must be
        implemented by child classes.

        """
        raise NotImplementedError('submit must be implemented!')

    def publish(self):
        """Push a message to the subscriber. This method must be
        implemented by child classes.

        """
        raise NotImplementedError('publish must be implemented!')

class EventSource(BaseHandler):
    """Handler for server-sent events a.k.a. EventSource.

    The EventSource__ interface has a few advantages over websockets:

    * It is a normal HTTP connection and so can be more easily monitored
      than websockets using tools like curl__ or HTTPie__.
    * Browsers generally automatically try to reestablish a lost
      connection.
    * The publish/subscribe pattern is better suited to some applications
      than the full duplex model of websockets.

    __ https://developer.mozilla.org/en-US/docs/Web/API/EventSource
    __ http://curl.haxx.se/
    __ https://github.com/jkbrzt/httpie

    """
    def initialize(self, store, period=None):
        """If ``period`` is given, publishers will sleep for
        approximately the given time in order to throttle data
        speeds.

        """
        super(EventSource, self).initialize(store)
        assert isinstance(period, (int, float)) or period is None
        self.period = period
        self.finished = False
        self.set_header('content-type', 'text/event-stream')
        self.set_header('cache-control', 'no-cache')

    def prepare(self):
        """Log access."""
        request_time = 1000.0 * self.request.request_time()
        access_log.info(
            "%d %s %.2fms", self.get_status(),
            self._request_summary(), request_time)

    @gen.coroutine
    def submit(self, message):
        """Receive incoming data."""
        logger.debug('Incoming message: ' + message)
        yield self.publish(message)

    @gen.coroutine
    def publish(self, message):
        """Pushes data to a listener."""
        try:
            self.write('data: {}\n\n'.format(message))
            yield self.flush()
        except StreamClosedError:
            self.finished = True

    @gen.coroutine
    def get(self, *args, **kwargs):
        try:
            while not self.finished:
                if self.period is not None:
                    yield gen.sleep(self.period)
                else:
                    yield gen.moment
        except Exception:
            pass
        finally:
            self.store.deregister(self)
            self.finish()

class WebSocketSubscriber(BaseHandler, WebSocketHandler):
    """A Websocket-based subscription handler to be used with
    :class:`tornadose.stores.QueueStore`.

    """
    def initialize(self, store):
        super(WebSocketSubscriber, self).initialize(store)
        self.messages = Queue()
        self.finished = False

    @gen.coroutine
    def open(self):
        """Register with the publisher."""
        self.store.register(self)
        while not self.finished:
            message = yield self.messages.get()
            yield self.publish(message)

    def on_close(self):
        self._close()

    def _close(self):
        self.store.deregister(self)
        self.finished = True

    @gen.coroutine
    def submit(self, message):
        yield self.messages.put(message)

    @gen.coroutine
    def publish(self, message):
        """Push a new message to the client. The data will be
        available as a JSON object with the key ``data``.

        """
        try:
            self.write_message(dict(data=message))
        except WebSocketClosedError:
            self._close()
<commit_msg>Make autoregistration actually work<commit_after>"""Custom request handlers for pushing data to connected clients."""

import logging
from tornado import gen
from tornado.web import RequestHandler
from tornado.websocket import WebSocketHandler, WebSocketClosedError
from tornado.queues import Queue
from tornado.iostream import StreamClosedError
from tornado.log import access_log
from . import stores

logger = logging.getLogger('tornadose.handlers')

class BaseHandler(RequestHandler):
    """Base handler for subscribers. To be compatible with data stores
    defined in :mod:`tornadose.stores`, custom handlers should inherit
    this class and implement the :meth:`submit` and :meth:`publish`
    methods.

    """
    def initialize(self, store):
        """Common initialization of handlers happens here. If additional
        initialization is required, this method *must* be called with
        ``super`` in order to properly initialize the handler.

        """
        assert isinstance(store, stores.BaseStore)
        self.store = store
        self.store.register(self)

    def submit(self, message):
        """Submit a new message to be published. This method must be
        implemented by child classes.

        """
        raise NotImplementedError('submit must be implemented!')

    def publish(self):
        """Push a message to the subscriber. This method must be
        implemented by child classes.

        """
        raise NotImplementedError('publish must be implemented!')

class EventSource(BaseHandler):
    """Handler for server-sent events a.k.a. EventSource.

    The EventSource__ interface has a few advantages over websockets:

    * It is a normal HTTP connection and so can be more easily monitored
      than websockets using tools like curl__ or HTTPie__.
    * Browsers generally automatically try to reestablish a lost
      connection.
    * The publish/subscribe pattern is better suited to some applications
      than the full duplex model of websockets.

    __ https://developer.mozilla.org/en-US/docs/Web/API/EventSource
    __ http://curl.haxx.se/
    __ https://github.com/jkbrzt/httpie

    """
    def initialize(self, store, period=None):
        """If ``period`` is given, publishers will sleep for
        approximately the given time in order to throttle data
        speeds.

        """
        super(EventSource, self).initialize(store)
        assert isinstance(period, (int, float)) or period is None
        self.period = period
        self.finished = False
        self.set_header('content-type', 'text/event-stream')
        self.set_header('cache-control', 'no-cache')

    def prepare(self):
        """Log access."""
        request_time = 1000.0 * self.request.request_time()
        access_log.info(
            "%d %s %.2fms", self.get_status(),
            self._request_summary(), request_time)

    @gen.coroutine
    def submit(self, message):
        """Receive incoming data."""
        logger.debug('Incoming message: ' + message)
        yield self.publish(message)

    @gen.coroutine
    def publish(self, message):
        """Pushes data to a listener."""
        try:
            self.write('data: {}\n\n'.format(message))
            yield self.flush()
        except StreamClosedError:
            self.finished = True

    @gen.coroutine
    def get(self, *args, **kwargs):
        try:
            while not self.finished:
                if self.period is not None:
                    yield gen.sleep(self.period)
                else:
                    yield gen.moment
        except Exception:
            pass
        finally:
            self.store.deregister(self)
            self.finish()

class WebSocketSubscriber(BaseHandler, WebSocketHandler):
    """A Websocket-based subscription handler to be used with
    :class:`tornadose.stores.QueueStore`.

    """
    def initialize(self, store):
        super(WebSocketSubscriber, self).initialize(store)
        self.messages = Queue()
        self.finished = False

    @gen.coroutine
    def open(self):
        """Register with the publisher."""
        self.store.register(self)
        while not self.finished:
            message = yield self.messages.get()
            yield self.publish(message)

    def on_close(self):
        self._close()

    def _close(self):
        self.store.deregister(self)
        self.finished = True

    @gen.coroutine
    def submit(self, message):
        yield self.messages.put(message)

    @gen.coroutine
    def publish(self, message):
        """Push a new message to the client. The data will be
        available as a JSON object with the key ``data``.

        """
        try:
            self.write_message(dict(data=message))
        except WebSocketClosedError:
            self._close()
<|endoftext|>