<commit_before>import blpapi
import logging
import datetime
import pandas as pd
from collections import defaultdict
from pandas import DataFrame

class BCon(object):
    def __init__(self, host='localhost', port=8194, debug=False):
        """
        Starting bloomberg API session
        close with session.close()

        Parameters
        ----------
        host: str
            Host name
        port: int
            Port to connect to
        debug: Boolean {True, False}
            Boolean corresponding to whether to log requests messages to stdout
        """
        # Fill SessionOptions
        sessionOptions = blpapi.SessionOptions()
        sessionOptions.setServerHost(host)
        sessionOptions.setServerPort(port)
        # Create a Session
        self.session = blpapi.Session(sessionOptions)
        # initialize logger
        self.debug = False

    @property
    def debug(self):
        return self._debug

    @debug.setter
    def debug(self, value):
        """
        Set whether logging is True or False
        """
        self._debug = value
        root = logging.getLogger()
        if self._debug:
            # log requests and responses
            root.setLevel(logging.DEBUG)
        else:
            # log only failed connections
            root.setLevel(logging.INFO)

    def start(self):
        """
        start connection and init service for refData
        """
        # Start a Session
        if not self.session.start():
            logging.info("Failed to start session.")
            return
        self.session.nextEvent()
        # Open service to get historical data from
        if not self.session.openService("//blp/refdata"):
            logging.info("Failed to open //blp/refdata")
            return
        self.session.nextEvent()
        # Obtain previously opened service
        self.refDataService = self.session.getService("//blp/refdata")
        self.session.nextEvent()

    def bdh(self, tickers, flds, start_date,
            end_date=datetime.date.today().strftime('%Y%m%d'),
            periodselection='DAILY'):
        """
        Get tickers and fields, return pandas dataframe with column MultiIndex
        of tickers and fields

        Parameters
        ----------
        tickers: {list, string}
            String or list of strings corresponding to tickers
        flds: {list, string}
            String or list of strings corresponding to FLDS
        start_date: string
            String in format YYYYmmdd
        end_date: string
            String in format YYYYmmdd
        """
        if type(tickers) is not list:
            tickers = [tickers]
        if type(flds) is not list:
            flds = [flds]
        # Create and fill the request for the historical data
        request = self.refDataService.createRequest("HistoricalDataRequest")
        for t in tickers:
            request.getElement("securities").appendValue(t)
        for f in flds:
            request.getElement("fields").appendValue(f)
        request.set("periodicityAdjustment", "ACTUAL")
        request.set("periodicitySelection", periodselection)
        request.set("startDate", start_date)
        request.set("endDate", end_date)

        logging.debug("Sending Request:\n %s" % request)
        # Send the request
        self.session.sendRequest(request)
        # defaultdict - later convert to pandas
        data = defaultdict(dict)
        # Process received events
        while(True):
            # We provide timeout to give the chance for Ctrl+C handling:
            ev = self.session.nextEvent(500)
            for msg in ev:
                logging.debug("Message Received:\n %s" % msg)
                ticker = msg.getElement('securityData')\
                    .getElement('security').getValue()
                fldData = msg.getElement('securityData')\
                    .getElement('fieldData')
                for i in range(fldData.numValues()):
                    dt = fldData.getValue(i).getElement(0).getValue()
                    for j in range(1, fldData.getValue(i).numElements()):
                        val = fldData.getValue(i).getElement(j).getValue()
                        data[(ticker, flds[j-1])][dt] = val

            if ev.eventType() == blpapi.Event.RESPONSE:
                # Response completely received, so we could exit
                break
        data = DataFrame(data)
        data.columns.names = ['ticker', 'field']
        data.index = pd.to_datetime(data.index)
        return data

    def bdib(self, ticker, startDateTime, endDateTime, eventType='TRADE',
             interval=1):
        """
        Get Open, High, Low, Close, Volume, for a ticker.
        Return pandas dataframe

        Parameters
        ----------
        ticker: string
            String corresponding to ticker
        startDateTime: string
            UTC datetime in format YYYY-mm-ddTHH:MM:SS
        endDateTime: string
            UTC datetime in format YYYY-mm-ddTHH:MM:SS
        eventType: string {TRADE, BID, ASK, BID_BEST, ASK_BEST, BEST_BID,
                           BEST_ASK}
            Requested data event type
        interval: int {1... 1440}
            Length of time bars
        """
        # Create and fill the request for the historical data
        request = self.refDataService.createRequest("IntradayBarRequest")
        request.set("security", ticker)
        request.set("eventType", eventType)
        request.set("interval", interval)  # bar interval in minutes
        request.set("startDateTime", startDateTime)
        request.set("endDateTime", endDateTime)

        logging.debug("Sending Request:\n %s" % request)
        # Send the request
        self.session.sendRequest(request)
        # defaultdict - later convert to pandas
        data = defaultdict(dict)
        # Process received events
        flds = ['open', 'high', 'low', 'close', 'volume']
        while(True):
            # We provide timeout to give the chance for Ctrl+C handling:
            ev = self.session.nextEvent(500)
            for msg in ev:
                logging.debug("Message Received:\n %s" % msg)
                barTick = msg.getElement('barData').getElement('barTickData')
                for i in range(barTick.numValues()):
                    for fld in flds:
                        dt = barTick.getValue(i).getElement(0).getValue()
                        val = barTick.getValue(i).getElement(fld).getValue()
                        data[(fld)][dt] = val

            if ev.eventType() == blpapi.Event.RESPONSE:
                # Response completly received, so we could exit
                break
        data = DataFrame(data)
        data.index = pd.to_datetime(data.index)
        data = data[flds]
        return data

    def stop(self):
        self.session.stop()
<commit_msg>fixed debug flag init, added reference data request function<commit_after>import blpapi
import logging
import datetime
import pandas as pd
from collections import defaultdict
from pandas import DataFrame

class BCon(object):
    def __init__(self, host='localhost', port=8194, debug=False):
        """
        Starting bloomberg API session
        close with session.close()

        Parameters
        ----------
        host: str
            Host name
        port: int
            Port to connect to
        debug: Boolean {True, False}
            Boolean corresponding to whether to log requests messages to stdout
        """
        # Fill SessionOptions
        sessionOptions = blpapi.SessionOptions()
        sessionOptions.setServerHost(host)
        sessionOptions.setServerPort(port)
        # Create a Session
        self.session = blpapi.Session(sessionOptions)
        # initialize logger
        self.debug = debug

    @property
    def debug(self):
        return self._debug

    @debug.setter
    def debug(self, value):
        """
        Set whether logging is True or False
        """
        self._debug = value
        root = logging.getLogger()
        if self._debug:
            # log requests and responses
            root.setLevel(logging.DEBUG)
        else:
            # log only failed connections
            root.setLevel(logging.INFO)

    def start(self):
        """
        start connection and init service for refData
        """
        # Start a Session
        if not self.session.start():
            logging.info("Failed to start session.")
            return
        self.session.nextEvent()
        # Open service to get historical data from
        if not self.session.openService("//blp/refdata"):
            logging.info("Failed to open //blp/refdata")
            return
        self.session.nextEvent()
        # Obtain previously opened service
        self.refDataService = self.session.getService("//blp/refdata")
        self.session.nextEvent()

    def bdh(self, tickers, flds, start_date,
            end_date=datetime.date.today().strftime('%Y%m%d'),
            periodselection='DAILY'):
        """
        Get tickers and fields, return pandas dataframe with column MultiIndex
        of tickers and fields

        Parameters
        ----------
        tickers: {list, string}
            String or list of strings corresponding to tickers
        flds: {list, string}
            String or list of strings corresponding to FLDS
        start_date: string
            String in format YYYYmmdd
        end_date: string
            String in format YYYYmmdd
        """
        if type(tickers) is not list:
            tickers = [tickers]
        if type(flds) is not list:
            flds = [flds]
        # Create and fill the request for the historical data
        request = self.refDataService.createRequest("HistoricalDataRequest")
        for t in tickers:
            request.getElement("securities").appendValue(t)
        for f in flds:
            request.getElement("fields").appendValue(f)
        request.set("periodicityAdjustment", "ACTUAL")
        request.set("periodicitySelection", periodselection)
        request.set("startDate", start_date)
        request.set("endDate", end_date)

        logging.debug("Sending Request:\n %s" % request)
        # Send the request
        self.session.sendRequest(request)
        # defaultdict - later convert to pandas
        data = defaultdict(dict)
        # Process received events
        while(True):
            # We provide timeout to give the chance for Ctrl+C handling:
            ev = self.session.nextEvent(500)
            for msg in ev:
                logging.debug("Message Received:\n %s" % msg)
                ticker = msg.getElement('securityData')\
                    .getElement('security').getValue()
                fldData = msg.getElement('securityData')\
                    .getElement('fieldData')
                for i in range(fldData.numValues()):
                    dt = fldData.getValue(i).getElement(0).getValue()
                    for j in range(1, fldData.getValue(i).numElements()):
                        val = fldData.getValue(i).getElement(j).getValue()
                        data[(ticker, flds[j-1])][dt] = val

            if ev.eventType() == blpapi.Event.RESPONSE:
                # Response completely received, so we could exit
                break
        data = DataFrame(data)
        data.columns.names = ['ticker', 'field']
        data.index = pd.to_datetime(data.index)
        return data

    def ref(self, tickers, flds):
        """
        Make a reference data request, get tickers and fields, return pandas
        dataframe with column of tickers and index of flds

        Parameters
        ----------
        tickers: {list, string}
            String or list of strings corresponding to tickers
        flds: {list, string}
            String or list of strings corresponding to FLDS
        """
        if type(tickers) is not list:
            tickers = [tickers]
        if type(flds) is not list:
            flds = [flds]
        # Create and fill the request for the historical data
        request = self.refDataService.createRequest("ReferenceDataRequest")
        for t in tickers:
            request.getElement("securities").appendValue(t)
        for f in flds:
            request.getElement("fields").appendValue(f)

        logging.debug("Sending Request:\n %s" % request)
        # Send the request
        self.session.sendRequest(request)
        data = []
        # Process received events
        while(True):
            # We provide timeout to give the chance for Ctrl+C handling:
            ev = self.session.nextEvent(500)
            for msg in ev:
                logging.debug("Message Received:\n %s" % msg)
                fldData = msg.getElement('securityData')
                for i in range(fldData.numValues()):
                    ticker = fldData.getValue(i).getElement("security")\
                        .getValue()
                    reqFldsData = fldData.getValue(i).getElement('fieldData')
                    for j in range(reqFldsData.numElements()):
                        fld = flds[j]
                        val = reqFldsData.getElement(fld).getValue()
                        data.append((fld, ticker, val))

            if ev.eventType() == blpapi.Event.RESPONSE:
                # Response completely received, so we could exit
                break
        data = DataFrame(data)
        data = data.pivot(0, 1, 2)
        data.index.name = None
        data.columns.name = None
        return data

    def bdib(self, ticker, startDateTime, endDateTime, eventType='TRADE',
             interval=1):
        """
        Get Open, High, Low, Close, Volume, for a ticker.
        Return pandas dataframe

        Parameters
        ----------
        ticker: string
            String corresponding to ticker
        startDateTime: string
            UTC datetime in format YYYY-mm-ddTHH:MM:SS
        endDateTime: string
            UTC datetime in format YYYY-mm-ddTHH:MM:SS
        eventType: string {TRADE, BID, ASK, BID_BEST, ASK_BEST, BEST_BID,
                           BEST_ASK}
            Requested data event type
        interval: int {1... 1440}
            Length of time bars
        """
        # Create and fill the request for the historical data
        request = self.refDataService.createRequest("IntradayBarRequest")
        request.set("security", ticker)
        request.set("eventType", eventType)
        request.set("interval", interval)  # bar interval in minutes
        request.set("startDateTime", startDateTime)
        request.set("endDateTime", endDateTime)

        logging.debug("Sending Request:\n %s" % request)
        # Send the request
        self.session.sendRequest(request)
        # defaultdict - later convert to pandas
        data = defaultdict(dict)
        # Process received events
        flds = ['open', 'high', 'low', 'close', 'volume']
        while(True):
            # We provide timeout to give the chance for Ctrl+C handling:
            ev = self.session.nextEvent(500)
            for msg in ev:
                logging.debug("Message Received:\n %s" % msg)
                barTick = msg.getElement('barData').getElement('barTickData')
                for i in range(barTick.numValues()):
                    for fld in flds:
                        dt = barTick.getValue(i).getElement(0).getValue()
                        val = barTick.getValue(i).getElement(fld).getValue()
                        data[(fld)][dt] = val

            if ev.eventType() == blpapi.Event.RESPONSE:
                # Response completly received, so we could exit
                break
        data = DataFrame(data)
        data.index = pd.to_datetime(data.index)
        data = data[flds]
        return data

    def stop(self):
        self.session.stop()
<|endoftext|>


<commit_before>"""Module containing code related to *sandman2* ORM models."""

# Standard library imports
import datetime
from decimal import Decimal

# Third-party imports
from sqlalchemy.inspection import inspect
from flask_sqlalchemy import SQLAlchemy  # pylint: disable=import-error,no-name-in-module
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.ext.declarative import declarative_base

db = SQLAlchemy()

class Model(object):

    """The sandman2 Model class is the base class for all RESTful resources.
    There is a one-to-one mapping between a table in the database and a
    :class:`sandman2.model.Model`.
    """

    #: The relative URL this resource should live at.
    __url__ = None

    #: The API version of this resource (not yet used).
    __version__ = '1'

    #: The HTTP methods this resource supports (default=all).
    __methods__ = {
        'GET',
        'POST',
        'PUT',
        'PATCH',
        'DELETE',
        'HEAD',
        'OPTIONS'
        }

    @classmethod
    def required(cls):
        """Return a list of all columns required by the database to create the
        resource.

        :param cls: The Model class to gather attributes from
        :rtype: list
        """
        columns = []
        for column in cls.__table__.columns:  # pylint: disable=no-member
            is_autoincrement = 'int' in str(column.type).lower() and column.autoincrement
            if (not column.nullable and not column.primary_key) or (column.primary_key and not is_autoincrement):
                columns.append(column.name)
        return columns

    @classmethod
    def optional(cls):
        """Return a list of all nullable columns for the resource's table.

        :rtype: list
        """
        columns = []
        for column in cls.__table__.columns:  # pylint: disable=no-member
            if column.nullable:
                columns.append(column.name)
        return columns

    @classmethod
    def primary_key(cls):
        """Return the key of the model's primary key field.

        :rtype: string
        """
        return list(
            cls.__table__.primary_key.columns)[  # pylint: disable=no-member
                0].key

    def to_dict(self):
        """Return the resource as a dictionary.

        :rtype: dict
        """
        result_dict = {}
        for column in self.__table__.columns.keys():  # pylint: disable=no-member
            value = result_dict[column] = getattr(self, column, None)
            if isinstance(value, Decimal):
                result_dict[column] = float(result_dict[column])
            elif isinstance(value, datetime.datetime):
                result_dict[column] = value.isoformat()
        return result_dict

    def links(self):
        """Return a dictionary of links to related resources that should be
        included in the *Link* header of an HTTP response.

        :rtype: dict

        """
        link_dict = {'self': self.resource_uri()}
        for relationship in inspect(  # pylint: disable=maybe-no-member
                self.__class__).relationships:
            if 'collection' not in relationship.key:
                instance = getattr(self, relationship.key)
                if instance:
                    link_dict[str(relationship.key)] = instance.resource_uri()
        return link_dict

    def resource_uri(self):
        """Return the URI to this specific resource.

        :rtype: str

        """
        return self.__url__ + '/' + str(getattr(self, self.primary_key()))

    def update(self, attributes):
        """Update the current instance based on attribute->value items in
        *attributes*.

        :param dict attributes: Dictionary of attributes to be updated
        :rtype: :class:`sandman2.model.Model`
        """
        for attribute in attributes:
            setattr(self, attribute, attributes[attribute])
        return self

    @classmethod
    def description(cls):
        """Return a field->data type dictionary describing this model
        as reported by the database.

        :rtype: dict
        """

        description = {}
        for column in cls.__table__.columns:  # pylint: disable=no-member
            column_description = str(column.type)
            if not column.nullable:
                column_description += ' (required)'
            description[column.name] = column_description
        return description

DeclarativeModel = declarative_base(cls=(db.Model, Model))
AutomapModel = automap_base(DeclarativeModel)
<commit_msg>Update model.py<commit_after>"""Module containing code related to *sandman2* ORM models."""

# Standard library imports
import datetime
from decimal import Decimal

# Third-party imports
from sqlalchemy.inspection import inspect
from flask_sqlalchemy import SQLAlchemy  # pylint: disable=import-error,no-name-in-module
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.ext.declarative import declarative_base

db = SQLAlchemy()

class Model(object):

    """The sandman2 Model class is the base class for all RESTful resources.
    There is a one-to-one mapping between a table in the database and a
    :class:`sandman2.model.Model`.
    """

    #: The relative URL this resource should live at.
    __url__ = None

    #: The API version of this resource (not yet used).
    __version__ = '1'

    #: The HTTP methods this resource supports (default=all).
    __methods__ = {
        'GET',
        'POST',
        'PUT',
        'PATCH',
        'DELETE',
        'HEAD',
        'OPTIONS'
        }

    @classmethod
    def required(cls):
        """Return a list of all columns required by the database to create the
        resource.

        :param cls: The Model class to gather attributes from
        :rtype: list
        """
        columns = []
        for column in cls.__table__.columns:  # pylint: disable=no-member
            is_autoincrement = 'int' in str(column.type).lower() and column.autoincrement
            if (not column.nullable and not column.primary_key) or (column.primary_key and not is_autoincrement):
                columns.append(column.name)
        return columns

    @classmethod
    def optional(cls):
        """Return a list of all nullable columns for the resource's table.

        :rtype: list
        """
        columns = []
        for column in cls.__table__.columns:  # pylint: disable=no-member
            if column.nullable:
                columns.append(column.name)
        return columns

    @classmethod
    def primary_key(cls):
        """Return the key of the model's primary key field.

        :rtype: string
        """
        return list(
            cls.__table__.primary_key.columns)[  # pylint: disable=no-member
                0].key

    def to_dict(self):
        """Return the resource as a dictionary.

        :rtype: dict
        """
        result_dict = {}
        for column in self.__table__.columns.keys():  # pylint: disable=no-member
            value = result_dict[column] = getattr(self, column, None)
            if isinstance(value, Decimal):
                result_dict[column] = float(result_dict[column])
            elif isinstance(value, datetime.datetime):
                result_dict[column] = value.isoformat()
            elif isinstance(value, datetime.time):
                result_dict[column] = value.strftime("%H:%M:%S")
        return result_dict

    def links(self):
        """Return a dictionary of links to related resources that should be
        included in the *Link* header of an HTTP response.

        :rtype: dict

        """
        link_dict = {'self': self.resource_uri()}
        for relationship in inspect(  # pylint: disable=maybe-no-member
                self.__class__).relationships:
            if 'collection' not in relationship.key:
                instance = getattr(self, relationship.key)
                if instance:
                    link_dict[str(relationship.key)] = instance.resource_uri()
        return link_dict

    def resource_uri(self):
        """Return the URI to this specific resource.

        :rtype: str

        """
        return self.__url__ + '/' + str(getattr(self, self.primary_key()))

    def update(self, attributes):
        """Update the current instance based on attribute->value items in
        *attributes*.

        :param dict attributes: Dictionary of attributes to be updated
        :rtype: :class:`sandman2.model.Model`
        """
        for attribute in attributes:
            setattr(self, attribute, attributes[attribute])
        return self

    @classmethod
    def description(cls):
        """Return a field->data type dictionary describing this model
        as reported by the database.

        :rtype: dict
        """

        description = {}
        for column in cls.__table__.columns:  # pylint: disable=no-member
            column_description = str(column.type)
            if not column.nullable:
                column_description += ' (required)'
            description[column.name] = column_description
        return description

DeclarativeModel = declarative_base(cls=(db.Model, Model))
AutomapModel = automap_base(DeclarativeModel)
<|endoftext|>


<commit_before>
import re
import logging
import os.path
import httplib
from time import sleep

from utilities import UnicodeDictReader, UnicodeDictWriter

class Tasks(object):
    """
    A helper wrapper class for task list.
    """
    def __init__(self, tasks = [], mentors_data=None, **kwargs):
        self._tasks = tasks
        self._fieldnames = []
        self._mentors_data = mentors_data
        self._issues_dir = kwargs.get("issues_dir")
        self._current_filename = None

    def __iter__(self):
        self._counter = 0
        return self

    def next(self):
        if self._counter < len(self._tasks):
            t = self._tasks[self._counter]
            self._counter += 1
            return t
        else:
            raise StopIteration

    def load(self, filename):
        self._current_filename = filename
        self._tasks = []
        with open(filename, 'rb') as f:
            reader = UnicodeDictReader(f)
            self._fieldnames = reader.fieldnames
            for row in reader:
                self._tasks.append(Task(row, mentors_data=self._mentors_data, issues_dir=self._issues_dir))

        logging.debug("Task list is loaded from the file `%s`" % filename)
        logging.debug("Columns: `%s`" % self._fieldnames)

    def extend_fieldnames(self, fieldnames):
        for name in fieldnames:
            if name not in self._fieldnames:
                self._fieldnames.append(name)

    def save(self, filename=None):
        if filename==None:
            filename = self._current_filename
        with open(filename, 'wb') as f:
            writer = UnicodeDictWriter(f, fieldnames=self._fieldnames)

            titles = {}
            for t in self._fieldnames:
                titles[t] = t
            writer.writerow(titles)

            for task in self:
                writer.writerow(task)

        logging.debug("Task list was saved to:`%s`" % filename)

    def update(self, new_tasks, unique_id = "Key"):
        """
        Join new task list to self, by unique id.
        """
        for task in new_tasks:
            _key = task[unique_id]
            if not self.exists(_key):
                logging.info("New task  #%s is appended.", _key)

    def exists(self, key):
        """
        Check if task is exists
        """
        for task in self._tasks:
            if task.Key == key:
                return True
        return False

    @property
    def fieldnames(self):
        return self._fieldnames

    @property
    def categories(self):
        """
        Retrieve a set of possible categories from a Categories column.
        """
        s = set()
        for task in self._tasks:
            value = task.Categories
            s.update([value])
        return s

    @property
    def difficulties(self):
        """
        Retrieve a set of possible difficulties from a Difficulty column.
        """
        s = set()
        for task in self._tasks:
            value = task.Difficulty
            s.update([value])
        return s

    def by_difficulty(self, difficulty):
        """
        Return tasks list which has pointed difficulty.
        """
        res = [task for task in self._tasks if task.Difficulty == difficulty]
        return Tasks(res)

    def by_category(self, category):
        """
        Return tasks list with pointed category.
        """
        res = [task for task in self._tasks if task.Difficulty == difficulty]
        return Tasks(res)

    def by_issue_id(self, issue_id):
        """
        Return tasks list with pointed category.
        """
        res = [task for task in self._tasks if (task.Id == issue_id) and (task['is inserted']=='False')]
        return Tasks(res)

class Task(dict):
    def __init__(self, row_dict, **kwargs):
        dict.__init__(self, row_dict)
        self._mentors_data  = kwargs.get("mentors_data")
        self._issues_dir    = kwargs.get("issues_dir")
        self._description = None
        self.__initialised = True

    def __getattr__(self, title):
        if title[0] <> "_":
            title = title.replace("_", " ")
        try:
            return super(Task,self).__getitem__(title)
        except KeyError:
            raise AttributeError(title)

    @property
    def filtered_lables(self):
        lables = self.AllLabels
        a = re.split("[,\s]+", lables)
        a = [ s for s in a if not re.match("CodeIn", s)]
        return ", ".join(a)

    @property
    def description(self):
        if self._description <> None:
            return self._description
        else:
            if self._issues_dir:
                if os.path.exists(self._issues_dir):
                    i = self.Id
                    fn = os.path.join(self._issues_dir, "%s.txt" % i)
                    if os.path.exists(fn):
                        with open(fn, "r") as f:
                            _description = f.read()
                            self._description = unicode(_description, "utf-8")
                            return _description
                    else:
                        logging.warning("There is not %s.txt in the issue_dir directory" % self.Id)
                else:
                    logging.warning("issues_dir is not exists: '%s'" % self._issues_dir)
            else:
                logging.warning("issues_dir is not defined")
        self._description = u''
        return self._description

    @property
    def mentors(self):
        """
        Obtain mentors list by a string with Mentor_IDs
        """
        return self._mentors_data.by_cs_linkid_string(self.Mentor_IDs)

    def retrieve_task_page(self):
        """
        Retrieve task page from google
        """
        logging.debug("Load the task page with key: %s" % self.Key)

        server = "www.google-melange.com"
        path_format = "/gci/task/view/google/gci2011/%s"

        path = path_format % self.Key

        headers = {"Accept":"Accept: text/html, application/xml, application/xhtml+xml"}
        conn = httplib.HTTPConnection(server)
        conn.request("GET", path)
        response = conn.getresponse()

        logging.debug("Response: %s, %s" % (response.status, response.reason))
        if response.status<>200:
            logging.warning("Response is not ok: %s, %s" % (response.status, response.reason))

        results = response.read()
        conn.close()
        sleep(1)
        return results

    def serach_issue_id(self):
        import re
        re_issue =re.compile("""Please see(\s| )+<a href="http://code.google.com/p/sympy/issues/detail[^=]+=(?P<id1>[\d]+)">[^=]+=(?P<id2>[\d]+)""", re.M)
        re_issue2 =re.compile("""Please see(\s| )+http://code.google.com/p/sympy/issues/detail[^=]+=(?P<id1>[\d]+)""", re.M)

        content = self.retrieve_task_page()
        m = re_issue.search(content)
        if m:
            assert m.group('id2') == m.group('id1')
        else:
            m = re_issue2.search(content)
        if m:
            _id = unicode(m.group('id1'))
            logging.info("For task '%s' issue id is found: %s" % (self.Key, _id))
            return _id
        else:
            logging.warning("Issue id is not found for the task #%s" % self.Key)
            return None

    def check_if_comment_is_inserted(self):
        """
        If task has issue_id (Id), check issue tracker for comments about the task id.
        """

        logging.info("Check whether comment for the task #%s (issue %s) is inserted." % (self.Key, self.Id))

        if not self.Key:
            return None
        if not self.Id:
            return None

        server = "code.google.com"
        path_format = "/p/sympy/issues/detail?id=%s"

        path = path_format % self.Id

        headers = {"Accept":"Accept: text/html, application/xml, application/xhtml+xml"}
        conn = httplib.HTTPConnection(server)
        conn.request("GET", path)
        response = conn.getresponse()

        logging.debug("Response: %s, %s" % (response.status, response.reason))
        if response.status<>200:
            logging.warning("Response is not ok: %s, %s" % (response.status, response.reason))

        results = response.read()
        conn.close()
        #s = """http://www.google-melange.com/gci/task/view/google/gci2011/(?P<id1>[\d]+)"""
        s = """http://www.google-melange.com/gci/task/view/google/gci2011/%s""" % self.Key
        re_task = re.compile(s, re.M)
        m = re_task.search(results)
        res = False
        if m:
            res = True

        logging.info("Checking result: %s" % res)
        sleep(1)
        return res

<commit_msg>Implement method Tasks.update(new_tasks) completly<commit_after>
import re
import logging
import os.path
import httplib
from time import sleep

from utilities import UnicodeDictReader, UnicodeDictWriter

class Tasks(object):
    """
    A helper wrapper class for task list.
    """
    def __init__(self, tasks = [], mentors_data=None, **kwargs):
        self._tasks = tasks
        self._fieldnames = []
        self._mentors_data = mentors_data
        self._issues_dir = kwargs.get("issues_dir")
        self._current_filename = None

    def __iter__(self):
        self._counter = 0
        return self

    def next(self):
        if self._counter < len(self._tasks):
            t = self._tasks[self._counter]
            self._counter += 1
            return t
        else:
            raise StopIteration

    def load(self, filename):
        self._current_filename = filename
        self._tasks = []
        with open(filename, 'rb') as f:
            reader = UnicodeDictReader(f)
            self._fieldnames = reader.fieldnames
            for row in reader:
                self._tasks.append(Task(row, mentors_data=self._mentors_data, issues_dir=self._issues_dir))

        logging.debug("Task list is loaded from the file `%s`" % filename)
        logging.debug("Columns: `%s`" % self._fieldnames)

    def extend_fieldnames(self, fieldnames):
        for name in fieldnames:
            if name not in self._fieldnames:
                self._fieldnames.append(name)

    def save(self, filename=None):
        if filename==None:
            filename = self._current_filename
        with open(filename, 'wb') as f:
            writer = UnicodeDictWriter(f, fieldnames=self._fieldnames)

            titles = {}
            for t in self._fieldnames:
                titles[t] = t
            writer.writerow(titles)

            for task in self:
                writer.writerow(task)

        logging.debug("Task list was saved to:`%s`" % filename)

    def update(self, new_tasks, unique_id = "Key"):
        """
        Join new task list to self, by unique id.
        """
        for task in new_tasks:
            _key = task[unique_id]
            if not self.exists(_key):
                self._tasks.append(task)
                logging.info("New task  #%s is appended.", _key)

    def exists(self, key):
        """
        Check if task is exists
        """
        for task in self._tasks:
            if task.Key == key:
                return True
        return False

    @property
    def fieldnames(self):
        return self._fieldnames

    @property
    def categories(self):
        """
        Retrieve a set of possible categories from a Categories column.
        """
        s = set()
        for task in self._tasks:
            value = task.Categories
            s.update([value])
        return s

    @property
    def difficulties(self):
        """
        Retrieve a set of possible difficulties from a Difficulty column.
        """
        s = set()
        for task in self._tasks:
            value = task.Difficulty
            s.update([value])
        return s

    def by_difficulty(self, difficulty):
        """
        Return tasks list which has pointed difficulty.
        """
        res = [task for task in self._tasks if task.Difficulty == difficulty]
        return Tasks(res)

    def by_category(self, category):
        """
        Return tasks list with pointed category.
        """
        res = [task for task in self._tasks if task.Difficulty == difficulty]
        return Tasks(res)

    def by_issue_id(self, issue_id):
        """
        Return tasks list with pointed category.
        """
        res = [task for task in self._tasks if (task.Id == issue_id) and (task['is inserted']=='False')]
        return Tasks(res)

class Task(dict):
    def __init__(self, row_dict, **kwargs):
        dict.__init__(self, row_dict)
        self._mentors_data  = kwargs.get("mentors_data")
        self._issues_dir    = kwargs.get("issues_dir")
        self._description = None
        self.__initialised = True

    def __getattr__(self, title):
        if title[0] <> "_":
            title = title.replace("_", " ")
        try:
            return super(Task,self).__getitem__(title)
        except KeyError:
            raise AttributeError(title)

    @property
    def filtered_lables(self):
        lables = self.AllLabels
        a = re.split("[,\s]+", lables)
        a = [ s for s in a if not re.match("CodeIn", s)]
        return ", ".join(a)

    @property
    def description(self):
        if self._description <> None:
            return self._description
        else:
            if self._issues_dir:
                if os.path.exists(self._issues_dir):
                    i = self.Id
                    fn = os.path.join(self._issues_dir, "%s.txt" % i)
                    if os.path.exists(fn):
                        with open(fn, "r") as f:
                            _description = f.read()
                            self._description = unicode(_description, "utf-8")
                            return _description
                    else:
                        logging.warning("There is not %s.txt in the issue_dir directory" % self.Id)
                else:
                    logging.warning("issues_dir is not exists: '%s'" % self._issues_dir)
            else:
                logging.warning("issues_dir is not defined")
        self._description = u''
        return self._description

    @property
    def mentors(self):
        """
        Obtain mentors list by a string with Mentor_IDs
        """
        return self._mentors_data.by_cs_linkid_string(self.Mentor_IDs)

    def retrieve_task_page(self):
        """
        Retrieve task page from google
        """
        logging.debug("Load the task page with key: %s" % self.Key)

        server = "www.google-melange.com"
        path_format = "/gci/task/view/google/gci2011/%s"

        path = path_format % self.Key

        headers = {"Accept":"Accept: text/html, application/xml, application/xhtml+xml"}
        conn = httplib.HTTPConnection(server)
        conn.request("GET", path)
        response = conn.getresponse()

        logging.debug("Response: %s, %s" % (response.status, response.reason))
        if response.status<>200:
            logging.warning("Response is not ok: %s, %s" % (response.status, response.reason))

        results = response.read()
        conn.close()
        sleep(1)
        return results

    def serach_issue_id(self):
        import re
        re_issue =re.compile("""Please see(\s| )+<a href="http://code.google.com/p/sympy/issues/detail[^=]+=(?P<id1>[\d]+)">[^=]+=(?P<id2>[\d]+)""", re.M)
        re_issue2 =re.compile("""Please see(\s| )+http://code.google.com/p/sympy/issues/detail[^=]+=(?P<id1>[\d]+)""", re.M)

        content = self.retrieve_task_page()
        m = re_issue.search(content)
        if m:
            assert m.group('id2') == m.group('id1')
        else:
            m = re_issue2.search(content)
        if m:
            _id = unicode(m.group('id1'))
            logging.info("For task '%s' issue id is found: %s" % (self.Key, _id))
            return _id
        else:
            logging.warning("Issue id is not found for the task #%s" % self.Key)
            return None

    def check_if_comment_is_inserted(self):
        """
        If task has issue_id (Id), check issue tracker for comments about the task id.
        """

        logging.info("Check whether comment for the task #%s (issue %s) is inserted." % (self.Key, self.Id))

        if not self.Key:
            return None
        if not self.Id:
            return None

        server = "code.google.com"
        path_format = "/p/sympy/issues/detail?id=%s"

        path = path_format % self.Id

        headers = {"Accept":"Accept: text/html, application/xml, application/xhtml+xml"}
        conn = httplib.HTTPConnection(server)
        conn.request("GET", path)
        response = conn.getresponse()

        logging.debug("Response: %s, %s" % (response.status, response.reason))
        if response.status<>200:
            logging.warning("Response is not ok: %s, %s" % (response.status, response.reason))

        results = response.read()
        conn.close()
        #s = """http://www.google-melange.com/gci/task/view/google/gci2011/(?P<id1>[\d]+)"""
        s = """http://www.google-melange.com/gci/task/view/google/gci2011/%s""" % self.Key
        re_task = re.compile(s, re.M)
        m = re_task.search(results)
        res = False
        if m:
            res = True

        logging.info("Checking result: %s" % res)
        sleep(1)
        return res

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

# Base configuration
AUTHOR = 'Jonathan Sharpe'
AVATAR = 'avatar.jpg'
SITENAME = 'textbook'
SITESUBTITLE = "These are my opinions - if you don't like them, I have others"
#SITEURL = 'http://textbook.github.io'
SITEURL = ''
PATH = 'content'
TIMEZONE = 'Europe/London'
DEFAULT_LANG = 'en'

# Appearance
THEME = 'meadowlark'
TYPOGRIFY = True
#CSS_FILE = 'wide.css'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (
	('Atom IDE', 'http://blog.atom.io/'),
	('Campo Santo', 'http://blog.camposanto.com/'),
	('Code as Craft', 'https://codeascraft.com/'),
)

# Social widget
SOCIAL = (
	('BitBucket', 'https://bitbucket.org/jonrsharpe'),
	('GitHub', 'https://github.com/textbook'),
    ('Stack Overflow', 'http://stackoverflow.com/users/3001761/jonrsharpe'),
    ('Twitter', 'https://twitter.com/jonrsharpe'),
)

# Other settings
DISQUS_SITENAME = 'textbook-dev'
GITHUB_URL = 'https://github.com/textbook/textbook.github.io-source'
GOOGLE_ANALYTICS = 'UA-64837080-1'
TWITTER_USERNAME = 'jonrsharpe'

# Static files
STATIC_PATHS = [
	'images',
	'extra/robots.txt',
	'extra/favicon.ico'
]
EXTRA_PATH_METADATA = {
    'extra/robots.txt': {'path': 'robots.txt'},
    'extra/favicon.ico': {'path': 'favicon.ico'}
}

DEFAULT_PAGINATION = 4

# URL settings
# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True
ARTICLE_SAVE_AS = '{date:%Y}/{date:%b}/{date:%d}/{slug}.html'
ARTICLE_URL = '{date:%Y}/{date:%b}/{date:%d}/{slug}.html'
# Don't need the author pages
AUTHOR_SAVE_AS = ''
AUTHOR_URL = ''
YEAR_ARCHIVE_SAVE_AS = '{date:%Y}/index.html'
<commit_msg>Enabling Disqus comments<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

# Base configuration
AUTHOR = 'Jonathan Sharpe'
AVATAR = 'avatar.jpg'
SITENAME = 'textbook'
SITESUBTITLE = "These are my opinions - if you don't like them, I have others"
SITEURL = 'http://blog.jonathansharpe.me.uk'
PATH = 'content'
TIMEZONE = 'Europe/London'
DEFAULT_LANG = 'en'

# Appearance
THEME = 'meadowlark'
TYPOGRIFY = True
#CSS_FILE = 'wide.css'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (
	('Atom IDE', 'http://blog.atom.io/'),
	('Campo Santo', 'http://blog.camposanto.com/'),
	('Code as Craft', 'https://codeascraft.com/'),
)

# Social widget
SOCIAL = (
	('BitBucket', 'https://bitbucket.org/jonrsharpe'),
	('GitHub', 'https://github.com/textbook'),
    ('Stack Overflow', 'http://stackoverflow.com/users/3001761/jonrsharpe'),
    ('Twitter', 'https://twitter.com/jonrsharpe'),
)

# Other settings
DISQUS_SITENAME = 'textbook-dev'
GITHUB_URL = 'https://github.com/textbook/textbook.github.io-source'
GOOGLE_ANALYTICS = 'UA-64837080-1'
TWITTER_USERNAME = 'jonrsharpe'

# Static files
STATIC_PATHS = [
	'images',
	'extra/robots.txt',
	'extra/favicon.ico'
]
EXTRA_PATH_METADATA = {
    'extra/robots.txt': {'path': 'robots.txt'},
    'extra/favicon.ico': {'path': 'favicon.ico'}
}

DEFAULT_PAGINATION = 4

# URL settings
# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True
ARTICLE_SAVE_AS = '{date:%Y}/{date:%b}/{date:%d}/{slug}.html'
ARTICLE_URL = '{date:%Y}/{date:%b}/{date:%d}/{slug}.html'
# Don't need the author pages
AUTHOR_SAVE_AS = ''
AUTHOR_URL = ''
YEAR_ARCHIVE_SAVE_AS = '{date:%Y}/index.html'
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

import datetime
import os
import sys
sys.path.append(os.path.abspath("."))

from stormwolves.scanner import Scanner
from stormwolves.plugins.pages import pages
from stormwolves.plugins.articles import articles
from stormwolves.plugins.tipuesearch import tipue_search

PLUGINS = [pages, articles, tipue_search]
SCANNER = Scanner("content").scan()
COPYRIGHT_LABEL = "© {0} Stormwolves".format(datetime.datetime.now().year)

AUTHOR = u'Test'
SITENAME = u'Stormwolves Team'
SITEURL = ''
THEME = 'stormwolves-theme'
PATH = 'content'
MENUITEMS = (('Archives', '/archives.html'),)

TIMEZONE = 'Europe/Berlin'

DEFAULT_LANG = u'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None
DISPLAY_PAGES_ON_MENU = True

ARTICLE_PATHS = ['news']
ARTICLE_SAVE_AS = 'news/{slug}.html'
# Blogroll
LINKS = (('Pelican', 'http://getpelican.com/'),
         ('Python.org', 'http://python.org/'),
         ('Jinja2', 'http://jinja.pocoo.org/'),
         ('You can modify those links in your config file', '#'),)

# Social widget
SOCIAL = (('You can add links in your config file', '#'),
          ('Another social link', '#'),)

DEFAULT_PAGINATION = 3
PAGINATED_DIRECT_TEMPLATES = ['index', 'archives']
DIRECT_TEMPLATES = ['index', 'tags', 'categories', 'authors', 'archives', 'search']

# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True
<commit_msg>Use page hieararchy plugin<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

import datetime
import os
import sys
sys.path.append(os.path.abspath("."))

from stormwolves.scanner import Scanner
from stormwolves.plugins.pages import pages
from stormwolves.plugins.articles import articles
from stormwolves.plugins.tipuesearch import tipue_search
from stormwolves.plugins.pagehierarchy import page_hierarchy

PLUGINS = [pages, articles, tipue_search, page_hierarchy]
SCANNER = Scanner("content").scan()
COPYRIGHT_LABEL = "© {0} Stormwolves".format(datetime.datetime.now().year)

AUTHOR = u'Test'
SITENAME = u'Stormwolves Team'
SITEURL = ''
THEME = 'stormwolves-theme'
PATH = 'content'
MENUITEMS = (('Archives', '/archives.html'),)

TIMEZONE = 'Europe/Berlin'

DEFAULT_LANG = u'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None
DISPLAY_PAGES_ON_MENU = True

ARTICLE_PATHS = ['news']
ARTICLE_SAVE_AS = 'news/{slug}.html'
# Blogroll
LINKS = (('Pelican', 'http://getpelican.com/'),
         ('Python.org', 'http://python.org/'),
         ('Jinja2', 'http://jinja.pocoo.org/'),
         ('You can modify those links in your config file', '#'),)

# Social widget
SOCIAL = (('You can add links in your config file', '#'),
          ('Another social link', '#'),)

DEFAULT_PAGINATION = 3
PAGINATED_DIRECT_TEMPLATES = ['index', 'archives']
DIRECT_TEMPLATES = ['index', 'tags', 'categories', 'authors', 'archives', 'search']

# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

PLUGIN_PATHS = ['/home/real/projects/websites/pelican_generic/plugins/pelican-plugins']
PLUGINS = ['render_math','tipue_search','simple_footnotes']

THEME = 'freedomlayer_theme'

DIRECT_TEMPLATES = \
    (('index','tags','categories','authors','archives','search','404'))

STATIC_PATHS = ['articles','pages']
ARTICLE_PATHS = ['articles']
PAGE_PATHS = ['pages']

# Files that we will not copy to output dir:
# Ignore vim swap files:
IGNORE_FILES = ['*.swp','*.swo']

# Directories and categories settings.
# Every article will be in a single directory, which will
# also contain the images and other related files.

# We don't use the directory name as the category, as we have many nested
# directories. Instead, we use the default category 'Blog':
USE_FOLDER_AS_CATEGORY = False
DEFAULT_CATEGORY = 'Articles'

# We display pages and categories on the website menu:
DISPLAY_PAGES_ON_MENU = False
DISPLAY_CATEGORIES_ON_MENU = False
MENUITEMS = (\
        ('Research','/pages/research.html'),\
        ('Passerby','/pages/passerby.html'),\
        ('About','/pages/about.html'),\
        )

# Delete the output directory before generating new output files:
DELETE_OUTPUT_DIRECTORY = True

AUTHOR = u'real'
SITENAME = u'Freedomlayer'
SITEURL = 'https://www.freedomlayer.org'

PATH = 'content'

TIMEZONE = 'UTC'

DEFAULT_LANG = u'en'

DATE_FORMATS = {
        u'en': '%a, %d %b %Y',
}

# Domain for feed generation:
FEED_DOMAIN = SITEURL
FEED_RSS = 'rss.xml'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
# LINKS = (('Pelican', 'http://getpelican.com/'),
#          ('Python.org', 'http://python.org/'),
#          ('Jinja2', 'http://jinja.pocoo.org/'),
#          ('You can modify those links in your config file', '#'),)

# Social widget
# SOCIAL = (('You can add links in your config file', '#'),
#           ('Another social link', '#'),)

# License text at the end of every page:
LICENSE_TEXT = """
This work
is licensed under a Creative Commons
Attribution-NonCommercial 4.0 International License."""

TYPOGRIFY = True

DEFAULT_PAGINATION = False

NUM_RECENT_ARTICLES = 10

# Uncomment following line if you want document-relative URLs when developing
RELATIVE_URLS = True
<commit_msg>Fixed problem with locale for article date.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

PLUGIN_PATHS = ['/home/real/projects/websites/pelican_generic/plugins/pelican-plugins']
PLUGINS = ['render_math','tipue_search','simple_footnotes']

THEME = 'freedomlayer_theme'

DIRECT_TEMPLATES = \
    (('index','tags','categories','authors','archives','search','404'))

STATIC_PATHS = ['articles','pages']
ARTICLE_PATHS = ['articles']
PAGE_PATHS = ['pages']

# Files that we will not copy to output dir:
# Ignore vim swap files:
IGNORE_FILES = ['*.swp','*.swo']

# Directories and categories settings.
# Every article will be in a single directory, which will
# also contain the images and other related files.

# We don't use the directory name as the category, as we have many nested
# directories. Instead, we use the default category 'Blog':
USE_FOLDER_AS_CATEGORY = False
DEFAULT_CATEGORY = 'Articles'

# We display pages and categories on the website menu:
DISPLAY_PAGES_ON_MENU = False
DISPLAY_CATEGORIES_ON_MENU = False
MENUITEMS = (\
        ('Research','/pages/research.html'),\
        ('Passerby','/pages/passerby.html'),\
        ('About','/pages/about.html'),\
        )

# Delete the output directory before generating new output files:
DELETE_OUTPUT_DIRECTORY = True

AUTHOR = u'real'
SITENAME = u'Freedomlayer'
SITEURL = 'https://www.freedomlayer.org'

PATH = 'content'

TIMEZONE = 'UTC'

DEFAULT_LANG = 'en'

LOCALE = ('en_US.UTF-8')

DATE_FORMATS = {
        'en': '%a, %d %b %Y',
}

# Domain for feed generation:
FEED_DOMAIN = SITEURL
FEED_RSS = 'rss.xml'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
# LINKS = (('Pelican', 'http://getpelican.com/'),
#          ('Python.org', 'http://python.org/'),
#          ('Jinja2', 'http://jinja.pocoo.org/'),
#          ('You can modify those links in your config file', '#'),)

# Social widget
# SOCIAL = (('You can add links in your config file', '#'),
#           ('Another social link', '#'),)

# License text at the end of every page:
LICENSE_TEXT = """
This work
is licensed under a Creative Commons
Attribution-NonCommercial 4.0 International License."""

TYPOGRIFY = True

DEFAULT_PAGINATION = False

NUM_RECENT_ARTICLES = 10

# Uncomment following line if you want document-relative URLs when developing
RELATIVE_URLS = True
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = u'Amjith'
SITENAME = u'pgcli'
SITEURL = 'http://pgcli.com'

PATH = 'content'

TIMEZONE = 'America/Los_Angeles'

DEFAULT_LANG = u'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = 'feeds/all.atom.xml'
CATEGORY_FEED_ATOM = 'feeds/%s.atom.xml'
TAG_FEED_ATOM = 'feeds/tag.%s.atom.xml'
FEED_ALL_RSS = 'feeds/all.rss.xml'
CATEGORY_FEED_RSS = 'feeds/%s.rss.xml'
TAG_FEED_RSS = 'feeds/tag.%s.rss.xml'
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (('mycli', 'http://mycli.net/'),
        )

# Social widget
SOCIAL = (('twitter', 'http://twitter.com/amjithr'),
          ('github', 'http://github.com/dbcli/pgcli'),
         )

DEFAULT_PAGINATION = 4

# Uncomment following line if you want document-relative URLs when developing
RELATIVE_URLS = True

PAGE_URL = '{slug}'
PAGE_SAVE_AS = '{slug}.html'

DIRECT_TEMPLATES = {}

STATIC_PATHS = {'images', 'extra/CNAME'}
EXTRA_PATH_METADATA = {'extra/CNAME': {'path': 'CNAME'},}

THEME = 'theme/notmyidea'

<commit_msg>Fix the https in site url.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = u'Amjith'
SITENAME = u'pgcli'
SITEURL = 'https://www.pgcli.com'

PATH = 'content'

TIMEZONE = 'America/Los_Angeles'

DEFAULT_LANG = u'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = 'feeds/all.atom.xml'
CATEGORY_FEED_ATOM = 'feeds/%s.atom.xml'
TAG_FEED_ATOM = 'feeds/tag.%s.atom.xml'
FEED_ALL_RSS = 'feeds/all.rss.xml'
CATEGORY_FEED_RSS = 'feeds/%s.rss.xml'
TAG_FEED_RSS = 'feeds/tag.%s.rss.xml'
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (('mycli', 'http://mycli.net/'),
        )

# Social widget
SOCIAL = (('twitter', 'http://twitter.com/amjithr'),
          ('github', 'http://github.com/dbcli/pgcli'),
         )

DEFAULT_PAGINATION = 4

# Uncomment following line if you want document-relative URLs when developing
RELATIVE_URLS = True

PAGE_URL = '{slug}'
PAGE_SAVE_AS = '{slug}.html'

DIRECT_TEMPLATES = {}

STATIC_PATHS = {'images', 'extra/CNAME'}
EXTRA_PATH_METADATA = {'extra/CNAME': {'path': 'CNAME'},}

THEME = 'theme/notmyidea'

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = u'BabuSubashChandar'
SITENAME = u'BabuSubashChandar'
SITETITLE = u'Babuenir'
SITESUBTITLE = u'Random musings on Technology, Art & Science'
SITEURL = u'http://192.168.1.7:8000'
GRAVATAR_IMAGE = 'https://secure.gravatar.com/avatar/aaf9dadf96ba3361de25e04a6c9fdfc3'
SITELOGO = SITEURL + u'/static/images/babuenir.png'

DISQUS_SITENAME = u'babuenir'

PATH = 'content'

TIMEZONE = 'Asia/Kolkata'

DEFAULT_LANG = u'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (('Pelican', 'http://getpelican.com/'),
         ('Python.org', 'http://python.org/'),
         ('Jinja2', 'http://jinja.pocoo.org/'),
         ('You can modify those links in your config file', '#'),)

# Social widget
SOCIAL = (('Github', 'https://github.com/babuenir'),
          ('Twitter', 'https://twitter.com/babuenir'),
          ('Linkedin', 'https://www.linkedin.com/in/babuenir'),)

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
# RELATIVE_URLS = True

ABOUT = """BabuSubashChandar is an Open Source Software enthusiast,
Embedded Systems Engineer and trainer by profession, an ardent lover of
arts and science. This is his personal blog, where you can find his
experiences and works related to GNU/Linux, Embedded Systems,
Literature, Paintings, and so on.
"""

STATIC_PATHS = ['static', ]

THEME = "theme"

ARTICLE_SAVE_AS = '{slug}.html'
ARTICLE_URL = '{slug}.html'
<commit_msg>Fixing siteurl.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = u'BabuSubashChandar'
SITENAME = u'BabuSubashChandar'
SITETITLE = u'Babuenir'
SITESUBTITLE = u'Random musings on Technology, Art & Science'
SITEURL = u'http://babuenir.github.io/blog'
GRAVATAR_IMAGE = 'https://secure.gravatar.com/avatar/aaf9dadf96ba3361de25e04a6c9fdfc3'
SITELOGO = SITEURL + u'/static/images/babuenir.png'

DISQUS_SITENAME = u'babuenir'

PATH = 'content'

TIMEZONE = 'Asia/Kolkata'

DEFAULT_LANG = u'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (('Pelican', 'http://getpelican.com/'),
         ('Python.org', 'http://python.org/'),
         ('Jinja2', 'http://jinja.pocoo.org/'),
         ('You can modify those links in your config file', '#'),)

# Social widget
SOCIAL = (('Github', 'https://github.com/babuenir'),
          ('Twitter', 'https://twitter.com/babuenir'),
          ('Linkedin', 'https://www.linkedin.com/in/babuenir'),)

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
# RELATIVE_URLS = True

ABOUT = """BabuSubashChandar is an Open Source Software enthusiast,
Embedded Systems Engineer and trainer by profession, an ardent lover of
arts and science. This is his personal blog, where you can find his
experiences and works related to GNU/Linux, Embedded Systems,
Literature, Paintings, and so on.
"""

STATIC_PATHS = ['static', ]

THEME = "theme"

ARTICLE_SAVE_AS = '{slug}.html'
ARTICLE_URL = '{slug}.html'
<|endoftext|>


<commit_before># Copyright 2013 Answers for AWS LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__author__ = 'Peter Sankauskas'
__version__ = '0.7.0'
<commit_msg>Version bump to 0.8.0<commit_after># Copyright 2013 Answers for AWS LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__author__ = 'Peter Sankauskas'
__version__ = '0.8.0'
<|endoftext|>


<commit_before>'''
	Authored: 8th June (first version)
'''
from pprint import pprint
import numpy as np

import node

class Graph:
	'''
		This class creates a graph based on given data, in an adjacency matrix and runs belief propagation on it,
		as described in the algorithm in the Tensorlog paper (https://arxiv.org/pdf/1605.06523.pdf)

		Input:
			_variables: (type: list of Variable objects) consisting of all the unique variables occuring in the query.
			_factors: (type: list of Factor objeccts) consisting of all the triples existing in the given rule.
			_fictional_factor: (type: Factor) one factor representing the predicate occuring in the head of the query.
			_rule: (type: string) the string from which everything was parsed (bookkeeping purposes).

	'''

	def __init__(self, _variables, _factors, _fictional_factor, _rule = None):

		'''
			Create a null variable.
			Initialize matrix
			Put values in the matrix.
		'''
		print "Creating graph for: ", _rule
		null_node = node.Node("phi")

		#Keep the list of variables in order to map the indices of the matrix to actual graph nodes
		self.variables = _variables + [null_node]

		#Keep the predicate that appeared in the head of the query in order to know where do we start the
		self.head_predicate = _fictional_factor

		#Initialize a numpy matrix
		self.graph = np.zeros(( len(self.variables), len(self.variables) )).astype('object')
		for factor in _factors:
			self.graph[self.variables.index(factor.i)][self.variables.index(factor.o)] = factor

		#Done making the graph and populating it.

		# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
		# 				DEBUG

		# for v in self.variables:
		# 	print v,
		# 	print self.variables.index(v)
		# print ""
		# print self.graph
		self.print_graph()

		# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

	def _get_neighbours(self, _node, _exclude = None):
		'''
			Internal Function.
			Returns the neighbouring factors of the given node.
			Can exclude a specific factor from it, if required.
		'''
		candidates = self.graph[self.variables.index(_node)].tolist()
		candidates += self.graph[:,self.variables.index(_node)].tolist()
		neighbors = list(set([ x for x in candidates if not x == 0.0]))

		#Exclude the excluding thing.
		neighbors = [ x for x in neighbors if not x == _exclude]

		return neighbors

	def print_graph(self):
		'''
			Function used to print the matrix with all its fancy tools and contraptions.
		'''
		graph = self.graph.tolist()
		matrix = [[""] + [ v.label for v in self.variables ]] + [ [self.variables[i].label] + [ "--" if v == 0.0 else v.label for v in graph[i]] for i in range(len(graph)) ]

		# data = np.asarray(data)
		# print data

		#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
		''' Code to print a matrix beautifully.
		Copied from: https://stackoverflow.com/questions/13214809/pretty-print-2d-python-list'''
		s = [[str(e) for e in row] for row in matrix]
		lens = [max(map(len, col)) for col in zip(*s)]
		fmt = '\t'.join('{{:{}}}'.format(x) for x in lens)
		table = [fmt.format(*row) for row in s]
		print '\n'.join(table)
		#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

	def propagate_thy_beliefs(self):
		'''
			Call this function to receive a string containing the path of the belief propagation algorithm.
			We implement the algorithm listed in the paper mentioned in the comments above

			Pseudocode:
				-> Call compile_message_node_to_factor from the o node of the head predicate.
				-> Let the functions recursively call each other
				-> Collect their things somehow. @TODO: how. what format. Shall we use theano variables altogether or what
				-> Return said stuff.
		'''
		print "graph:bp: Starting belief propagation."
		equation = self._comiple_message_node_(self.head_predicate.o, "Fictional Label")
		print equation
		print "graph:bp: Belief propagation complete."

		return equation

	def _comiple_message_node_(self, _node, _factor):
		'''
			Pseudocode:
				(treat _node as X and _factor as L )

				if X is the input variable (global) then
					return u_c , the input
				else
					generate a new variable name v_x
					collect neighbouring L_i of X excluding L
					for [L_1, L_2 .. L_i ], do
						v_i = compile_message(L_i -> X)
					emit(v_x = v1 dot v2 ... dot vi)
					return v_x
		'''

		if _node == self.head_predicate.i:
			#This is the input variable.
			return "u_"+_node.label

		#This is NOT the input variable.
		neighbors = self._get_neighbours(_node, _exclude=_factor) #Will be a list of factors.

		#Send the neighbour + current node to compilemessage_factor and collect what they have to say.
		v_x = ' \circle '.join([ self._compile_message_factor_(_factor = factor, _node = _node) for factor in neighbors ])

		return v_x

	def _compile_message_factor_(self, _factor, _node):
		'''
			Pseudocode:
				(treat _node as X and _factor as L)

			if L is a unary factor:
				emit v_L,_X = v_L

			elif X is the output node of L
				v_i = compilemessage_node(X_o, L)

			elif X is the output node of L
				v_i =  compilemessage_node(X_i, L)

			return this
		'''

		if _factor.o.null:
			#If the factor is unary.
			return "v_"+factor.label

		elif _factor.o == _node:
			#If the node is the output node for this factor
			v_i = self._comiple_message_node_(_factor.i, _factor)
			return v_i+ " \dot M_"+_factor.label

		elif _factor.i == _node:
			 #If the node is the input node for this factor
			v_i = self._comiple_message_node_(_factor.o, _factor)
			return v_i+ " \dot M_"+_factor.label<commit_msg>*UNTESTED* Transformed the Belief Prop to work with theano symbolic variables instead of strings (talk about promotions, eh). It's late enough in the night to not test them.<commit_after>'''
	Authored: 8th June (first version)
'''
from pprint import pprint
import theano.tensor as T
import numpy as np

import node

class Graph:
	'''
		This class creates a graph based on given data, in an adjacency matrix and runs belief propagation on it,
		as described in the algorithm in the Tensorlog paper (https://arxiv.org/pdf/1605.06523.pdf)

		Input:
			_variables: (type: list of Variable objects) consisting of all the unique variables occuring in the query.
			_factors: (type: list of Factor objeccts) consisting of all the triples existing in the given rule.
			_fictional_factor: (type: Factor) one factor representing the predicate occuring in the head of the query.
			_rule: (type: string) the string from which everything was parsed (bookkeeping purposes).

	'''

	def __init__(self, _variables, _factors, _fictional_factor, _rule = None):

		'''
			Create a null variable.
			Initialize matrix
			Put values in the matrix.
		'''
		print "Creating graph for: ", _rule
		null_node = node.Node("phi")

		#Keep the list of variables in order to map the indices of the matrix to actual graph nodes
		self.variables = _variables + [null_node]
		#Keep the constant appearing in the i position of the head variable to be used while doing the BP
		self.u_c = None

		#Keep the predicate that appeared in the head of the query in order to know where do we start the
		self.head_predicate = _fictional_factor

		#Initialize a numpy matrix
		self.graph = np.zeros(( len(self.variables), len(self.variables) )).astype('object')
		for factor in _factors:
			self.graph[self.variables.index(factor.i)][self.variables.index(factor.o)] = factor

		#Done making the graph and populating it.

		# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
		# 				DEBUG

		# for v in self.variables:
		# 	print v,
		# 	print self.variables.index(v)
		# print ""
		# print self.graph
		self.print_graph()

		# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

	def _get_neighbours(self, _node, _exclude = None):
		'''
			Internal Function.
			Returns the neighbouring factors of the given node.
			Can exclude a specific factor from it, if required.
		'''
		candidates = self.graph[self.variables.index(_node)].tolist()
		candidates += self.graph[:,self.variables.index(_node)].tolist()
		neighbors = list(set([ x for x in candidates if not x == 0.0]))

		#Exclude the excluding thing.
		neighbors = [ x for x in neighbors if not x == _exclude]

		return neighbors

	def print_graph(self):
		'''
			Function used to print the matrix with all its fancy tools and contraptions.
		'''
		graph = self.graph.tolist()
		matrix = [[""] + [ v.label for v in self.variables ]] + [ [self.variables[i].label] + [ "--" if v == 0.0 else v.label for v in graph[i]] for i in range(len(graph)) ]

		# data = np.asarray(data)
		# print data

		#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
		''' Code to print a matrix beautifully.
		Copied from: https://stackoverflow.com/questions/13214809/pretty-print-2d-python-list'''
		s = [[str(e) for e in row] for row in matrix]
		lens = [max(map(len, col)) for col in zip(*s)]
		fmt = '\t'.join('{{:{}}}'.format(x) for x in lens)
		table = [fmt.format(*row) for row in s]
		print '\n'.join(table)
		#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

	def propagate_thy_beliefs(self, _u_c):
		'''
			Call this function to receive a string containing the path of the belief propagation algorithm.
			We implement the algorithm listed in the paper mentioned in the comments above

			Input: the constant appearing in the first position of the head predicate.

			Pseudocode:
				-> Create an empty theano vector whose definitions will be iteratively changed.
				-> Call compile_message_node_to_factor from the o node of the head predicate.
				-> Let the functions recursively call each other
				-> Collect their things somehow. @TODO: how. what format. Shall we use theano variables altogether or what
				-> Return said stuff.
		'''

		self.u_c = _u_c

		print "graph:bp: Starting belief propagation."
		equation = self._comiple_message_node_(self.head_predicate.o, "Fictional Label")
		print equation
		print "graph:bp: Belief propagation complete."

		return equation

	def _comiple_message_node_(self, _node, _factor):
		'''
			Pseudocode:
				(treat _node as X and _factor as L )

				if X is the input variable (global) then
					return u_c , the input
				else
					generate a new variable name v_x
					collect neighbouring L_i of X excluding L
					for [L_1, L_2 .. L_i ], do
						v_i = compile_message(L_i -> X)
					emit(v_x = v1 dot v2 ... dot vi)
					return v_x
		'''

		if _node == self.head_predicate.i:
			#This is the input variable.
			return self.u_c

		#This is NOT the input variable.
		neighbors = self._get_neighbours(_node, _exclude=_factor) #Will be a list of factors.

		#Send the neighbour + current node to compilemessage_factor and collect what they have to say.
		# v_x = ' \circle '.join([ self._compile_message_factor_(_factor = factor, _node = _node) for factor in neighbors ])
		neighboring_values = [ self._compile_message_factor_(_factor = factor, _node = _node) for factor in neighbors ]
		if len(neighboring_values) > 0:
			v_x = neighboring_values[0]
			for remaining_values in neighboring_values[1:]:
				v_x = v_x * remaining_values
		else:
			#In this case, since there are no neighbors, there's literally nothing to return.
			#@TODO: What do we do here
			pass

		return v_x

	def _compile_message_factor_(self, _factor, _node):
		'''
			Pseudocode:
				(treat _node as X and _factor as L)

			if L is a unary factor:
				emit v_L,_X = v_L

			elif X is the output node of L
				v_i = compilemessage_node(X_o, L)

			elif X is the output node of L
				v_i =  compilemessage_node(X_i, L)

			return this
		'''

		if _factor.o.null:
			#If the factor is unary.
			return _factor.M 	#@TODO: See if we need another variable like self.v to represent the value of unary predicates.

		elif _factor.o == _node:
			#If the node is the output node for this factor
			v_i = self._comiple_message_node_(_factor.i, _factor)
			# return v_i+ " \dot M_"+_factor.label
			return v_i.dot(_factor.label)

		elif _factor.i == _node:
			 #If the node is the input node for this factor
			v_i = self._comiple_message_node_(_factor.o, _factor)
			# return v_i+ " \dot M_"+_factor.label
			return v_i.dot(_factor.label)<|endoftext|>


<commit_before>from . import exceptions
import idaapi
from . import sark
import idautils
import idc
from collections import namedtuple, defaultdict
import operator

FF_TYPES = [idc.FF_BYTE, idc.FF_WORD, idc.FF_DWRD, idc.FF_QWRD, idc.FF_OWRD, ]
FF_SIZES = [1, 2, 4, 8, 16, ]

SIZE_TO_TYPE = dict(zip(FF_SIZES, FF_TYPES))

STRUCT_ERROR_MAP = {
    idc.STRUC_ERROR_MEMBER_NAME:
        (exceptions.SarkErrorStructMemberName, "already has member with this name (bad name)"),
    idc.STRUC_ERROR_MEMBER_OFFSET:
        (exceptions.SarkErrorStructMemberOffset, "already has member at this offset"),
    idc.STRUC_ERROR_MEMBER_SIZE:
        (exceptions.SarkErrorStructMemberSize, "bad number of bytes or bad sizeof(type)"),
    idc.STRUC_ERROR_MEMBER_TINFO:
        (exceptions.SarkErrorStructMemberTinfo, "bad typeid parameter"),
    idc.STRUC_ERROR_MEMBER_STRUCT:
        (exceptions.SarkErrorStructMemberStruct, "bad struct id (the 1st argument)"),
    idc.STRUC_ERROR_MEMBER_UNIVAR:
        (exceptions.SarkErrorStructMemberUnivar, "unions can't have variable sized members"),
    idc.STRUC_ERROR_MEMBER_VARLAST:
        (exceptions.SarkErrorStructMemberVarlast, "variable sized member should be the last member in the structure"),
}

def struct_member_error(err, sid, name, offset, size):
    exception, msg = STRUCT_ERROR_MAP[err]
    struct_name = idc.GetStrucName(sid)
    return exception(('AddStructMember(struct="{}", member="{}", offset={}, size={}) '
                      'failed: {}').format(
        struct_name,
        name,
        offset,
        size,
        msg
    ))

def create_struct(name):
    sid = idc.GetStrucIdByName(name)
    if sid != 0xFFFFFFFF:
        # The struct already exists.
        raise exceptions.SarkStructAlreadyExists()

    sid = idc.AddStrucEx(-1, name, 0)
    if sid == 0xFFFFFFFF:
        raise exceptions.SarkStructCreationFailed()

    return sid

def get_struct(name):
    sid = idc.GetStrucIdByName(name)
    if sid == 0xFFFFFFFF:
        raise exceptions.SarkStructNotFound()

    return sid

def size_to_flags(size):
    return SIZE_TO_TYPE[size] | idc.FF_DATA

def add_struct_member(sid, name, offset, size):
    idaapi.msg("{}, {}\n".format(offset, size))
    failure = idc.AddStrucMember(sid, name, offset, size_to_flags(size), -1, size)

    if failure:
        raise struct_member_error(failure, sid, name, offset, size)

StructOffset = namedtuple("StructOffset", "offset size")
OperandRef = namedtuple("OperandRef", "ea n")

DTYP_TO_SIZE = {
        idaapi.dt_byte : 1,
        idaapi.dt_word : 2,
        idaapi.dt_dword: 4,
        idaapi.dt_float: 4,
        idaapi.dt_double: 8,
        idaapi.dt_qword: 8,
        idaapi.dt_byte16: 16,
        idaapi.dt_fword: 6,
        idaapi.dt_3byte: 3,
        idaapi.dt_byte32: 32,
        idaapi.dt_byte64: 64,
    }

def dtyp_to_size(dtyp):
    return DTYP_TO_SIZE[dtyp]

def infer_struct_offsets(start, end, reg_name):
    offsets = set()
    operands = []
    for line in sark.iter_lines(start, end):
        inst = line.inst
        if not sark.is_reg_in_inst(inst, reg_name):
            continue

        for operand in inst.Operands:
            if not sark.is_reg_in_operand(operand, reg_name):
                continue

            if not sark.operand_has_displacement(operand):
                continue

            offset = sark.operand_get_displacement(operand)
            size = dtyp_to_size(operand.dtyp)
            offsets.add(StructOffset(offset, size))
            operands.append(OperandRef(line.ea, operand.n))

    return offsets, operands

def get_common_register(start, end):
    registers = defaultdict(int)
    for line in sark.iter_lines(start, end):
        inst = line.inst

        for operand in inst.Operands:

            if not sark.operand_has_displacement(operand):
                continue

            register_name = sark.get_register_name(operand.reg)
            registers[register_name] += 1

    return max(registers.iteritems(), key=operator.itemgetter(1))[0]

def offset_name(offset):
    return "offset_{:X}".format(offset.offset)

def set_struct_offsets(offsets, sid):
    for offset in offsets:
        try:
            add_struct_member(sid,
                              offset_name(offset),
                              offset.offset,
                              offset.size)
        except exceptions.SarkErrorStructMemberName:
            # Get the offset of the member with the same name
            existing_offset = idc.GetMemberOffset(sid, offset_name(offset))
            if offset.offset == existing_offset:
                pass
            else:
                raise

def create_struct_from_offsets(name, offsets):
    sid = create_struct(name)

    set_struct_offsets(offsets, sid)

def apply_struct(start, end, reg_name, struct_name):
    offsets, operands = infer_struct_offsets(start, end, reg_name)

    sid = get_struct(struct_name)

    for ea, n in operands:
        idc.OpStroff(ea, n, sid)<commit_msg>Added todo item.<commit_after>from . import exceptions
import idaapi
from . import sark
import idautils
import idc
from collections import namedtuple, defaultdict
import operator

FF_TYPES = [idc.FF_BYTE, idc.FF_WORD, idc.FF_DWRD, idc.FF_QWRD, idc.FF_OWRD, ]
FF_SIZES = [1, 2, 4, 8, 16, ]

SIZE_TO_TYPE = dict(zip(FF_SIZES, FF_TYPES))

STRUCT_ERROR_MAP = {
    idc.STRUC_ERROR_MEMBER_NAME:
        (exceptions.SarkErrorStructMemberName, "already has member with this name (bad name)"),
    idc.STRUC_ERROR_MEMBER_OFFSET:
        (exceptions.SarkErrorStructMemberOffset, "already has member at this offset"),
    idc.STRUC_ERROR_MEMBER_SIZE:
        (exceptions.SarkErrorStructMemberSize, "bad number of bytes or bad sizeof(type)"),
    idc.STRUC_ERROR_MEMBER_TINFO:
        (exceptions.SarkErrorStructMemberTinfo, "bad typeid parameter"),
    idc.STRUC_ERROR_MEMBER_STRUCT:
        (exceptions.SarkErrorStructMemberStruct, "bad struct id (the 1st argument)"),
    idc.STRUC_ERROR_MEMBER_UNIVAR:
        (exceptions.SarkErrorStructMemberUnivar, "unions can't have variable sized members"),
    idc.STRUC_ERROR_MEMBER_VARLAST:
        (exceptions.SarkErrorStructMemberVarlast, "variable sized member should be the last member in the structure"),
}

def struct_member_error(err, sid, name, offset, size):
    exception, msg = STRUCT_ERROR_MAP[err]
    struct_name = idc.GetStrucName(sid)
    return exception(('AddStructMember(struct="{}", member="{}", offset={}, size={}) '
                      'failed: {}').format(
        struct_name,
        name,
        offset,
        size,
        msg
    ))

def create_struct(name):
    sid = idc.GetStrucIdByName(name)
    if sid != 0xFFFFFFFF:
        # The struct already exists.
        raise exceptions.SarkStructAlreadyExists()

    sid = idc.AddStrucEx(-1, name, 0)
    if sid == 0xFFFFFFFF:
        raise exceptions.SarkStructCreationFailed()

    return sid

def get_struct(name):
    sid = idc.GetStrucIdByName(name)
    if sid == 0xFFFFFFFF:
        raise exceptions.SarkStructNotFound()

    return sid

def size_to_flags(size):
    return SIZE_TO_TYPE[size] | idc.FF_DATA

def add_struct_member(sid, name, offset, size):
    idaapi.msg("{}, {}\n".format(offset, size))
    failure = idc.AddStrucMember(sid, name, offset, size_to_flags(size), -1, size)

    if failure:
        raise struct_member_error(failure, sid, name, offset, size)

StructOffset = namedtuple("StructOffset", "offset size")
OperandRef = namedtuple("OperandRef", "ea n")

DTYP_TO_SIZE = {
        idaapi.dt_byte : 1,
        idaapi.dt_word : 2,
        idaapi.dt_dword: 4,
        idaapi.dt_float: 4,
        idaapi.dt_double: 8,
        idaapi.dt_qword: 8,
        idaapi.dt_byte16: 16,
        idaapi.dt_fword: 6,
        idaapi.dt_3byte: 3,
        idaapi.dt_byte32: 32,
        idaapi.dt_byte64: 64,
    }

def dtyp_to_size(dtyp):
    return DTYP_TO_SIZE[dtyp]

def infer_struct_offsets(start, end, reg_name):
    offsets = set()
    operands = []
    for line in sark.iter_lines(start, end):
        inst = line.inst
        if not sark.is_reg_in_inst(inst, reg_name):
            continue

        for operand in inst.Operands:
            if not sark.is_reg_in_operand(operand, reg_name):
                continue

            if not sark.operand_has_displacement(operand):
                continue

            offset = sark.operand_get_displacement(operand)
            size = dtyp_to_size(operand.dtyp)
            offsets.add(StructOffset(offset, size))
            operands.append(OperandRef(line.ea, operand.n))

    return offsets, operands

def get_common_register(start, end):
    registers = defaultdict(int)
    for line in sark.iter_lines(start, end):
        inst = line.inst

        for operand in inst.Operands:

            if not sark.operand_has_displacement(operand):
                continue

            # TODO: use the operand dtyp size here to get the proper register name.
            register_name = sark.get_register_name(operand.reg)
            registers[register_name] += 1

    return max(registers.iteritems(), key=operator.itemgetter(1))[0]

def offset_name(offset):
    return "offset_{:X}".format(offset.offset)

def set_struct_offsets(offsets, sid):
    for offset in offsets:
        try:
            add_struct_member(sid,
                              offset_name(offset),
                              offset.offset,
                              offset.size)
        except exceptions.SarkErrorStructMemberName:
            # Get the offset of the member with the same name
            existing_offset = idc.GetMemberOffset(sid, offset_name(offset))
            if offset.offset == existing_offset:
                pass
            else:
                raise

def create_struct_from_offsets(name, offsets):
    sid = create_struct(name)

    set_struct_offsets(offsets, sid)

def apply_struct(start, end, reg_name, struct_name):
    offsets, operands = infer_struct_offsets(start, end, reg_name)

    sid = get_struct(struct_name)

    for ea, n in operands:
        idc.OpStroff(ea, n, sid)<|endoftext|>


<commit_before>#!/usr/bin/env python

__author__ = 'third'

import requests
from io import BytesIO
from matplotlib import image as img
from  greengraph.map import Map
import geopy
import numpy as np
import numpy.testing as npt
import os
import requests
from mock import Mock, patch

lat = 51.5073509
long = -0.1277583

test_map = Map(lat, long)

def test_Map():
    mock_image = open(os.path.join(os.path.dirname(__file__),
                                   'london_png.png'), 'rb')
    with patch('requests.get', return_value=Mock(content=mock_image.read())) as mock_get:
        npt.assert_allclose(test_map.pixels, np.load(os.path.join(os.path.dirname(__file__),
                                                                  'london_numpy.npy')))
        print("map_test")

        # class Map(object):
        # def __init__(self, lat, long, satellite=True,
        # zoom=10, size=(400, 400), sensor=False):
        # base = "http://maps.googleapis.com/maps/api/staticmap?"
        #
        # params = dict(
        #             sensor=str(sensor).lower(),
        #             zoom=zoom,
        #             size="x".join(map(str, size)),
        #             center=",".join(map(str, (lat, long))),
        #             style="feature:all|element:labels|visibility:off"
        #         )
        #
        #         if satellite:
        #             params["maptype"] = "satellite"
        #
        #         self.image = requests.get(base, params=params).content
        #         # Fetch our PNG image data
        #         self.pixels = img.imread(BytesIO(self.image))
        #         # Parse our PNG image as a numpy array
        #
        #     def green(self, threshold):
        #         # Use NumPy to build an element-by-element logical array
        #         greener_than_red = self.pixels[:, :, 1] > threshold * self.pixels[:, :, 0]
        #         greener_than_blue = self.pixels[:, :, 1] > threshold * self.pixels[:, :, 2]
        #         green = np.logical_and(greener_than_red, greener_than_blue)
        #         return green
        #
        #     def count_green(self, threshold=1.1):
        #         return np.sum(self.green(threshold))
        #
        #     def show_green(data, threshold=1.1):
        #         green = self.green(threshold)
        #         out = green[:, :, np.newaxis] * array([0, 1, 0])[np.newaxis, np.newaxis, :]
        #         buffer = BytesIO()
        #         result = img.imsave(buffer, out, format='png')
        #         return buffer.getvalue()<commit_msg>Edit unit test so no internet required<commit_after>#!/usr/bin/env python

__author__ = 'third'

import requests
from io import BytesIO
from matplotlib import image as img
from greengraph.map import Map
import geopy
import numpy as np
import numpy.testing as npt
import os
import requests
from mock import Mock, patch

lat = 51.5073509
long = -0.1277583

def test_Map():
    mock_image = open(os.path.join(os.path.dirname(__file__),
                                   'london_png.png'), 'rb')
    with patch('requests.get', return_value=Mock(content=mock_image.read())) as mock_get:
        test_map = Map(lat, long)
        npt.assert_allclose(test_map.pixels, np.load(os.path.join(os.path.dirname(__file__),
                                                                  'london_numpy.npy')))
        print("map_test")

        # class Map(object):
        #
        # def __init__(self, lat, long, satellite=True,
        #     zoom=10, size=(400, 400), sensor=False):
        #     base = "http://maps.googleapis.com/maps/api/staticmap?"
        #
        #     params = dict(
        #     sensor=str(sensor).lower(),
        #     zoom=zoom,
        #                 size="x".join(map(str, size)),
        #                 center=",".join(map(str, (lat, long))),
        #                 style="feature:all|element:labels|visibility:off"
        #             )
        #
        #             if satellite:
        #                 params["maptype"] = "satellite"
        #
        #             self.image = requests.get(base, params=params).content
        #             # Fetch our PNG image data
        #             self.pixels = img.imread(BytesIO(self.image))
        #             # Parse our PNG image as a numpy array
        #
        #     def green(self, threshold):
        #         # Use NumPy to build an element-by-element logical array
        #         greener_than_red = self.pixels[:, :, 1] > threshold * self.pixels[:, :, 0]
        #         greener_than_blue = self.pixels[:, :, 1] > threshold * self.pixels[:, :, 2]
        #         green = np.logical_and(greener_than_red, greener_than_blue)
        #         return green
        #
        #     def count_green(self, threshold=1.1):
        #         return np.sum(self.green(threshold))
        #
        #     def show_green(data, threshold=1.1):
        #         green = self.green(threshold)
        #         out = green[:, :, np.newaxis] * array([0, 1, 0])[np.newaxis, np.newaxis, :]
        #         buffer = BytesIO()
        #         result = img.imsave(buffer, out, format='png')
        #         return buffer.getvalue()<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
'''
Created on Jul 24, 2017

@author: dan.ellard@raytheon.com

Copyright 2017 Raytheon BBN Technologies

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

HOW TO USE:

The module or file that is serving as the "main" should begin
with something like the following, BEFORE the importing of
any of the QGL files:

from . import config_location
config_location.config(PATH_TO_CONFIG_FILE)
from . inport config

If you don't wish to set the path to the configuration file
explicitly, then the first two lines should be omitted.
The last line may be unnecessary because many of the files in
this directory begin with something like "from QGL import *",
which imports config.

NOTE: This file must be imported, and any defaults overridden,
before config.py is imported.  When config.py is imported, it
will use the get_config_path method from this module to find the
location of the JSON config file to use.  After this happens,
the config variables are set and will not be altered by changing
the CONFIG_PATH or the _CONFIG_FILE_NAME (and they probably
SHOULD NOT be modified, because some of the values taken from
the config file directly change the behavior of the other modules,
so changing things on fly may result in an inconsistent state).
'''

import os.path

CONFIG_PATH = None

_CONFIG_FILE_NAME = 'config.json'

def _set_default_config_path():
    """
    Set the path for the "default" configuration file:

    1. if there's one in the current working directory, then use it

    2. otherwise, if there's one in dirname(__file__), then use it

    3. otherwise, leave the path at None and assume that the user will
        set it explicitly instead of relying on a default
    """

    cwd_path = os.path.join(os.getcwd(), _CONFIG_FILE_NAME)
    def_path = os.path.join(os.path.dirname(__file__), _CONFIG_FILE_NAME)

    # We could also check whether the file is readable, etc.
    # I think the error messages make more sense if we let this
    # fail later, but I don't know what the users will think is
    # the most intelligible.

    if os.path.isfile(cwd_path):
        return cwd_path
    elif os.path.isfile(def_path):
        return def_path
    else:
        return None

def config(path):
    """
    Set CONFIG_FILE, the path to the configuration JSON file,
    to the given path, and return the new path.

    No error or sanity checking is performed.
    """

    global CONFIG_FILE

    CONFIG_FILE = path
    return CONFIG_FILE

def set_config_filename(filename):
    """
    Change the "default" filename for the config file from the
    current value of CONFIG_FILE_NAME to the given filename string.

    The filename is assumed to be a simple filename (no directory
    components).  Partial directory paths might work, or might not.
    No sanity checking is done on the filename.

    This function has no useful effect if CONFIG_FILE is non-None;
    it is used only to compute the CONFIG_FILE, and if the CONFIG_FILE
    has already been set, then CONFIG_FILE will not change as a
    result of changing the CONFIG_FILE_NAME.
    """

    global _CONFIG_FILE_NAME

    _CONFIG_FILE_NAME = filename
    return filename

def get_config_path():
    """
    Retrieve the current value of the path to the configuration file
    """

    global CONFIG_PATH

    if not CONFIG_PATH:
        CONFIG_PATH = _set_default_config_path()

    return CONFIG_PATH
<commit_msg>methods to control the path to config.json<commit_after># -*- coding: UTF-8 -*-
'''
Created on Jul 24, 2017

@author: dan.ellard@raytheon.com

Copyright 2017 Raytheon BBN Technologies

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

HOW TO USE:

The module or file that is serving as the "main" should begin
with something like the following, BEFORE the importing of
any of the QGL files:

from . import config_location
config_location.config(PATH_TO_CONFIG_FILE)
from . inport config

If you don't wish to set the path to the configuration file
explicitly, then the first two lines should be omitted.
The last line may be unnecessary because many of the files in
this directory begin with something like "from QGL import *",
which imports config.

NOTE: This file must be imported, and any defaults overridden,
before config.py is imported.  When config.py is imported, it
will use the get_config_path method from this module to find the
location of the JSON config file to use.  After this happens,
the config variables are set and will not be altered by changing
the CONFIG_PATH or the _CONFIG_FILE_NAME (and they probably
SHOULD NOT be modified, because some of the values taken from
the config file directly change the behavior of the other modules,
so changing things on fly may result in an inconsistent state).

'''

import os.path

CONFIG_PATH = None

_CONFIG_FILE_NAME = 'config.json'

def _set_default_config_path():
    """
    Set the path for the "default" configuration file:

    1. if there's one in the current working directory, then use it

    2. otherwise, if there's one in dirname(__file__), then use it

    3. otherwise, leave the path at None and assume that the user will
        set it explicitly instead of relying on a default
    """

    cwd_path = os.path.join(os.getcwd(), _CONFIG_FILE_NAME)
    def_path = os.path.join(os.path.dirname(__file__), _CONFIG_FILE_NAME)

    # We could also check whether the file is readable, etc.
    # I think the error messages make more sense if we let this
    # fail later, but I don't know what the users will think is
    # the most intelligible.

    if os.path.isfile(cwd_path):
        return cwd_path
    elif os.path.isfile(def_path):
        return def_path
    else:
        return None

def config(path):
    """
    Set CONFIG_FILE, the path to the configuration JSON file,
    to the given path, and return the new path.

    No error or sanity checking is performed.
    """

    global CONFIG_FILE

    CONFIG_FILE = path
    return CONFIG_FILE

def set_config_filename(filename):
    """
    Change the "default" filename for the config file from the
    current value of CONFIG_FILE_NAME to the given filename string.

    The filename is assumed to be a simple filename (no directory
    components).  Partial directory paths might work, or might not.
    No sanity checking is done on the filename.

    This function has no useful effect if CONFIG_FILE is non-None;
    it is used only to compute the CONFIG_FILE, and if the CONFIG_FILE
    has already been set, then CONFIG_FILE will not change as a
    result of changing the CONFIG_FILE_NAME.
    """

    global _CONFIG_FILE_NAME

    _CONFIG_FILE_NAME = filename
    return filename

def get_config_path():
    """
    Retrieve the current value of the path to the configuration file
    """

    global CONFIG_PATH

    if not CONFIG_PATH:
        CONFIG_PATH = _set_default_config_path()

    return CONFIG_PATH
<|endoftext|>