<commit_before>#!/usr/bin/python

# Copyright (C) 2010-2011 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Extended function library for request handlers for persistent-cal"""

__author__ = 'dhermes@google.com (Daniel Hermes)'

# General libraries
import datetime
import logging
import re
import simplejson
from time import sleep
from urllib2 import urlopen

# Third-party libraries
import atom
import gdata.gauth
import gdata.calendar.client
import gdata.calendar.data
from gdata.client import RedirectError
from icalendar import Calendar

# App engine specific libraries
from google.appengine.ext import db

# App specific libraries
from models import Event
from secret_key import CONSUMER_KEY
from secret_key import CONSUMER_SECRET
from secret_key import TOKEN
from secret_key import TOKEN_SECRET

URI = ('https://www.google.com/calendar/feeds/'
       'vhoam1gb7uqqoqevu91liidi80%40group.calendar.google.com/private/full')
RESPONSES = {1: ['once a week', 'week'],
             4: ['every two days', 'two-day'],
             7: ['once a day', 'day'],
             14: ['twice a day', 'half-day'],
             28: ['every six hours', 'six-hrs'],
             56: ['every three hours', 'three-hrs']}

def JsonAscii(obj):
  """Returns an object in JSON with ensure_ascii explicitly set True"""
  return simplejson.dumps(obj, ensure_ascii=True)

def UpdateString(update_intervals):
  """Returns a JSON object to represent the frequency of updates"""
  length = len(update_intervals)
  if length not in RESPONSES:
    raise Exception('Bad interval length')
  else:
    return simplejson.dumps(RESPONSES[length])

def InitGCAL():
  """Initializes a calendar client based on a stored auth token"""
  gcal = gdata.calendar.client.CalendarClient(source='persistent-cal')

  auth_token = gdata.gauth.OAuthHmacToken(consumer_key=CONSUMER_KEY,
                                          consumer_secret=CONSUMER_SECRET,
                                          token=TOKEN,
                                          token_secret=TOKEN_SECRET,
                                          auth_state=3)

  gcal.auth_token = auth_token
  return gcal

def ConvertToInterval(timestamp):
  """Converts a datetime timestamp to a time interval for a cron job"""
  # Monday 0, sunday 6
  # 8 intervals in a day, round up hours
  interval = 8*timestamp.weekday() + timestamp.hour/3 + 1

  # If we are exactly on an hour that is a multiple of three
  # we do not wish to round up since floor(x) == ceil(x), contrary
  # to all other cases where ceil(x) == floor(x) + 1
  relative_seconds = sum([3600*(timestamp.hour % 3 == 0),
                          60*timestamp.minute,
                          timestamp.second,
                          timestamp.microsecond/1000.0])
  if relative_seconds < 300:  # under 5 minutes past schedule
    interval -= 1

  return interval % 56

def FormatTime(time_value):
  """Takes a datetime object and returns a formatted time stamp"""
  # Fails if not datetime.datetime or datetime.datetime

  # strftime('%Y-%m-%dT%H:%M:%S.000Z') for datetime
  # strftime('%Y-%m-%d') for date

  # Default TZ is UTC/GMT (as is TZ in GCal)
  time_parse = '%Y-%m-%d'
  if isinstance(time_value, datetime.datetime):
    time_parse += 'T%H:%M:%S.000Z'
    return time_value.strftime(time_parse)
  elif isinstance(time_value, datetime.date):
    return time_value.strftime(time_parse)

def TimeToDTStamp(time_as_str):
  """Takes time as string and returns a datetime object"""
  # strftime('%Y-%m-%dT%H:%M:%S.000Z') for datetime
  # strftime('%Y-%m-%d') for date

  # Default TZ is UTC/GMT
  time_parse = '%Y-%m-%d'
  try:
    converted_val = datetime.datetime.strptime(time_as_str, time_parse)
    return converted_val
  except ValueError:
    pass

  time_parse += 'T%H:%M:%S.000Z'
  try:
    converted_val = datetime.datetime.strptime(time_as_str, time_parse)
    return converted_val
  except ValueError:
    pass

def RemoveTimezone(time_value):
  """Takes a datetime object and removes the timezone"""
  if isinstance(time_value, datetime.datetime):
    if time_value.tzinfo is not None:
      time_parse = '%Y-%m-%dT%H:%M:%S.%fZ'
      time_value = time_value.strftime(time_parse) # TZ is lost
      time_value = datetime.datetime.strptime(time_value, time_parse)
  return time_value

def WhiteList(link):
  """Determines if a link is on the whitelist and transforms it if needed"""
  # If we WhiteList is updated, ParseEvent must be as well
  valid = False
  transformed = link

  pattern_webcal = ('^webcal://www.tripit.com/feed/ical/private/'
                    '[A-Za-z0-9-]+/tripit.ics$')
  pattern_http = ('^http://www.tripit.com/feed/ical/private/'
                  '[A-Za-z0-9-]+/tripit.ics$')
  if re.match(pattern_webcal, link):
    valid = True

    len_webcal = len('webcal')
    transformed = 'http%s' % link[len_webcal:]
  elif re.match(pattern_http, link):
    valid = True
    transformed = link

  return valid, transformed

def AddOrUpdateEvent(event_data, gcal, event=None, push_update=True):
  """Create event in main application calendar and add user as attendee"""
  update = (event is not None)
  if not update:
    event = gdata.calendar.data.CalendarEventEntry()

  event.title = atom.data.Title(text=event_data['summary'])
  event.content = atom.data.Content(text=event_data['description'])

  # Where
  event.where.append(gdata.calendar.data.CalendarWhere(
      value=event_data['location']))

  # When
  start_time = event_data['when:from']
  end_time = event_data['when:to']
  event.when.append(gdata.calendar.data.When(start=start_time, end=end_time))

  if update:
    attempts = 3
    if push_update:
      while attempts:
        try:
          gcal.Update(event)
          break
        except RedirectError:
          attempts -= 1
          sleep(3)
          pass

    # Returns none if event did not get updated (if it needed to)
    return event if attempts else None
  else:
    # Who
    who_add = gdata.calendar.data.EventWho(email=event_data['email'])
    event.who.append(who_add)

    attempts = 3
    new_event = None
    while attempts:
      try:
        new_event = gcal.InsertEvent(event, insert_uri=URI)
        break
      except RedirectError:
        attempts -= 1
        sleep(3)
        pass

    return new_event

def ParseEvent(event):
  """Parses an iCalendar.cal.Event instance to a predefined format"""
  # Assumes event is type icalendar.cal.Event

  # all events have a UID, and almost all begin with 'item-';
  # those that don't are an away event for the entire trip
  uid = unicode(event.get('uid'))
  description = unicode(event.get('description'))
  location = unicode(event.get('location'))
  # No destination specified does not match up
  if location == 'No destination specified':
    location = 'an unspecified location'

  # # WITNESS:
  # 4754cd75888cac4a53c7cf003980e195b46dc9fd@tripit.com
  # via 3F43994D-4591D1AA4C63B1472D8D5D0E9568E5A8/tripit.ics
  # description: Daniel Hermes is in San Diego, CA from Sep 1...
  # and via 4A025929-DCB74CB87F330487615696811896215A/tripit.ics
  # description: Sharona Franko is in San Diego, CA from Sep 1...
  if not uid.startswith('item-'):
    target = ' is in %s ' % location
    if description.count(target) != 1:
      # Since whitelisted, we expect the same format
      raise Exception('Unrecognized event format')

    description = 'In %s %s' % (location, description.split(target)[1])

  event_data = {'when:from': FormatTime(event.get('dtstart').dt),
                'when:to': FormatTime(event.get('dtend').dt),
                'summary': unicode(event.get('summary')),
                'location': location,
                'description': description}
  return uid, event_data

def UpdateSubscription(link, current_user, gcal):
  """
  Updates the GCal instance with the events in link for the current_user

  Returns a list of all uid's for events which have yet to occur (can be used
  to delete removed events)
  """
  result = []
  current_user_id = current_user.user_id()

  valid, link = WhiteList(link)
  if not valid:
    # Do nothing if not on the whitelist
    return result

  import_feed = urlopen(link)
  ical = Calendar.from_string(import_feed.read())
  import_feed.close()

  now = datetime.datetime.utcnow()
  for component in ical.walk():
    if component.name != 'VEVENT':
      logging.info('iCal at % has unexpected event type %s' % (
        link, component.name))
    else:
      uid, event_data = ParseEvent(component)
      event = Event.get_by_key_name(uid)
      if event is None:
        # Create new event
        # (leaving out the event argument creates a new event)
        event_data['email'] = current_user.email()
        cal_event = AddOrUpdateEvent(event_data, gcal)
        # TODO(dhermes) add to failed queue to be updated by a cron
        if cal_event is None:
          continue

        gcal_edit = cal_event.get_edit_link().href
        event = Event(key_name=uid,
                      who=[current_user_id],  # id is string
                      event_data=db.Text(JsonAscii(event_data)),
                      gcal_edit=gcal_edit)
        event.put()

        # execution has successfully completed
        # TODO(dhermes) catch error on get call below
        if RemoveTimezone(component.get('dtend').dt) > now:
          result.append(uid)
      else:
        # We need to make changes for new event data or a new owner
        if (current_user_id not in event.who or
            db.Text(JsonAscii(event_data)) != event.event_data):
          # Grab GCal event to edit
          attempts = 3
          cal_event = None
          while attempts:
            try:
              cal_event = gcal.GetEventEntry(uri=event.gcal_edit)
              break
            except RedirectError:
              attempts -= 1
              sleep(3)
              pass

          # TODO(dhermes) add to failed queue to be updated by a cron
          if cal_event is None:
            continue

          # Update who
          if current_user_id not in event.who:
            event.who.append(current_user_id)  # id is string

            # add existing event to current_user's calendar
            who_add = gdata.calendar.data.EventWho(email=current_user.email())
            cal_event.who.append(who_add)

          # Update existing event
          if db.Text(JsonAscii(event_data)) != event.event_data:
            event.event_data = db.Text(JsonAscii(event_data))

            # Don't push update to avoid pushing twice (if both changed)
            AddOrUpdateEvent(event_data,
                             gcal,
                             event=cal_event,
                             push_update=False)
            # push_update=False, impossible to have RedirectError

          # Push all updates to calendar event
          attempts = 3
          new_event = None
          while attempts:
            try:
              gcal.Update(cal_event)

              # After all possible changes to the Event instance have occurred
              event.put()
              break
            except RedirectError:
              attempts -= 1
              sleep(3)
              pass

          # If attempts is 0, we have failed and don't want to add the
          # uid to results
          if attempts == 0:
            continue

        # execution has successfully completed
        # TODO(dhermes) catch error on get call below
        if RemoveTimezone(component.get('dtend').dt) > now:
          result.append(uid)

  return result

def UpdateUpcoming(user_cal, upcoming, gcal):
  """
  Updates the GCal instance by deleting pending events removed from ext calendar

  If the new upcoming events list is different from that on the user_cal, it
  will iterate through the difference and delete those events that have not yet
  passed which are still on the calendar.
  """
  if user_cal.upcoming != upcoming:
    now = datetime.datetime.utcnow()
    for uid in set(user_cal.upcoming).difference(upcoming):
      event = Event.get_by_key_name(uid)
      event_data = simplejson.loads(event.event_data)
      if TimeToDTStamp(event_data['when:to']) > now:
        gcal.delete(event.gcal_edit, force=True)
        event.delete()
    user_cal.upcoming = upcoming
    user_cal.put()

import simplejson
with open('db_events.json', 'rb') as fh:
  data = simplejson.load(fh)
for key, value in data.items():
  event_data = simplejson.loads(value['event_data'])
  print event_data['when:to']
  print TimeToDTStamp(event_data['when:to']), '\n'
  print event_data['when:from']
  print TimeToDTStamp(event_data['when:from']), '\n'
<commit_msg>Fixed typo<commit_after>#!/usr/bin/python

# Copyright (C) 2010-2011 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Extended function library for request handlers for persistent-cal"""

__author__ = 'dhermes@google.com (Daniel Hermes)'

# General libraries
import datetime
import logging
import re
import simplejson
from time import sleep
from urllib2 import urlopen

# Third-party libraries
import atom
import gdata.gauth
import gdata.calendar.client
import gdata.calendar.data
from gdata.client import RedirectError
from icalendar import Calendar

# App engine specific libraries
from google.appengine.ext import db

# App specific libraries
from models import Event
from secret_key import CONSUMER_KEY
from secret_key import CONSUMER_SECRET
from secret_key import TOKEN
from secret_key import TOKEN_SECRET

URI = ('https://www.google.com/calendar/feeds/'
       'vhoam1gb7uqqoqevu91liidi80%40group.calendar.google.com/private/full')
RESPONSES = {1: ['once a week', 'week'],
             4: ['every two days', 'two-day'],
             7: ['once a day', 'day'],
             14: ['twice a day', 'half-day'],
             28: ['every six hours', 'six-hrs'],
             56: ['every three hours', 'three-hrs']}

def JsonAscii(obj):
  """Returns an object in JSON with ensure_ascii explicitly set True"""
  return simplejson.dumps(obj, ensure_ascii=True)

def UpdateString(update_intervals):
  """Returns a JSON object to represent the frequency of updates"""
  length = len(update_intervals)
  if length not in RESPONSES:
    raise Exception('Bad interval length')
  else:
    return simplejson.dumps(RESPONSES[length])

def InitGCAL():
  """Initializes a calendar client based on a stored auth token"""
  gcal = gdata.calendar.client.CalendarClient(source='persistent-cal')

  auth_token = gdata.gauth.OAuthHmacToken(consumer_key=CONSUMER_KEY,
                                          consumer_secret=CONSUMER_SECRET,
                                          token=TOKEN,
                                          token_secret=TOKEN_SECRET,
                                          auth_state=3)

  gcal.auth_token = auth_token
  return gcal

def ConvertToInterval(timestamp):
  """Converts a datetime timestamp to a time interval for a cron job"""
  # Monday 0, sunday 6
  # 8 intervals in a day, round up hours
  interval = 8*timestamp.weekday() + timestamp.hour/3 + 1

  # If we are exactly on an hour that is a multiple of three
  # we do not wish to round up since floor(x) == ceil(x), contrary
  # to all other cases where ceil(x) == floor(x) + 1
  relative_seconds = sum([3600*(timestamp.hour % 3 == 0),
                          60*timestamp.minute,
                          timestamp.second,
                          timestamp.microsecond/1000.0])
  if relative_seconds < 300:  # under 5 minutes past schedule
    interval -= 1

  return interval % 56

def FormatTime(time_value):
  """Takes a datetime object and returns a formatted time stamp"""
  # Fails if not datetime.datetime or datetime.datetime

  # strftime('%Y-%m-%dT%H:%M:%S.000Z') for datetime
  # strftime('%Y-%m-%d') for date

  # Default TZ is UTC/GMT (as is TZ in GCal)
  time_parse = '%Y-%m-%d'
  if isinstance(time_value, datetime.datetime):
    time_parse += 'T%H:%M:%S.000Z'
    return time_value.strftime(time_parse)
  elif isinstance(time_value, datetime.date):
    return time_value.strftime(time_parse)

def TimeToDTStamp(time_as_str):
  """Takes time as string and returns a datetime object"""
  # strftime('%Y-%m-%dT%H:%M:%S.000Z') for datetime
  # strftime('%Y-%m-%d') for date

  # Default TZ is UTC/GMT
  time_parse = '%Y-%m-%d'
  try:
    converted_val = datetime.datetime.strptime(time_as_str, time_parse)
    return converted_val
  except ValueError:
    pass

  time_parse += 'T%H:%M:%S.000Z'
  try:
    converted_val = datetime.datetime.strptime(time_as_str, time_parse)
    return converted_val
  except ValueError:
    pass

def RemoveTimezone(time_value):
  """Takes a datetime object and removes the timezone"""
  if isinstance(time_value, datetime.datetime):
    if time_value.tzinfo is not None:
      time_parse = '%Y-%m-%dT%H:%M:%S.%fZ'
      time_value = time_value.strftime(time_parse) # TZ is lost
      time_value = datetime.datetime.strptime(time_value, time_parse)
  return time_value

def WhiteList(link):
  """Determines if a link is on the whitelist and transforms it if needed"""
  # If we WhiteList is updated, ParseEvent must be as well
  valid = False
  transformed = link

  pattern_webcal = ('^webcal://www.tripit.com/feed/ical/private/'
                    '[A-Za-z0-9-]+/tripit.ics$')
  pattern_http = ('^http://www.tripit.com/feed/ical/private/'
                  '[A-Za-z0-9-]+/tripit.ics$')
  if re.match(pattern_webcal, link):
    valid = True

    len_webcal = len('webcal')
    transformed = 'http%s' % link[len_webcal:]
  elif re.match(pattern_http, link):
    valid = True
    transformed = link

  return valid, transformed

def AddOrUpdateEvent(event_data, gcal, event=None, push_update=True):
  """Create event in main application calendar and add user as attendee"""
  update = (event is not None)
  if not update:
    event = gdata.calendar.data.CalendarEventEntry()

  event.title = atom.data.Title(text=event_data['summary'])
  event.content = atom.data.Content(text=event_data['description'])

  # Where
  event.where.append(gdata.calendar.data.CalendarWhere(
      value=event_data['location']))

  # When
  start_time = event_data['when:from']
  end_time = event_data['when:to']
  event.when.append(gdata.calendar.data.When(start=start_time, end=end_time))

  if update:
    attempts = 3
    if push_update:
      while attempts:
        try:
          gcal.Update(event)
          break
        except RedirectError:
          attempts -= 1
          sleep(3)
          pass

    # Returns none if event did not get updated (if it needed to)
    return event if attempts else None
  else:
    # Who
    who_add = gdata.calendar.data.EventWho(email=event_data['email'])
    event.who.append(who_add)

    attempts = 3
    new_event = None
    while attempts:
      try:
        new_event = gcal.InsertEvent(event, insert_uri=URI)
        break
      except RedirectError:
        attempts -= 1
        sleep(3)
        pass

    return new_event

def ParseEvent(event):
  """Parses an iCalendar.cal.Event instance to a predefined format"""
  # Assumes event is type icalendar.cal.Event

  # all events have a UID, and almost all begin with 'item-';
  # those that don't are an away event for the entire trip
  uid = unicode(event.get('uid'))
  description = unicode(event.get('description'))
  location = unicode(event.get('location'))
  # No destination specified does not match up
  if location == 'No destination specified':
    location = 'an unspecified location'

  # # WITNESS:
  # 4754cd75888cac4a53c7cf003980e195b46dc9fd@tripit.com
  # via 3F43994D-4591D1AA4C63B1472D8D5D0E9568E5A8/tripit.ics
  # description: Daniel Hermes is in San Diego, CA from Sep 1...
  # and via 4A025929-DCB74CB87F330487615696811896215A/tripit.ics
  # description: Sharona Franko is in San Diego, CA from Sep 1...
  if not uid.startswith('item-'):
    target = ' is in %s ' % location
    if description.count(target) != 1:
      # Since whitelisted, we expect the same format
      raise Exception('Unrecognized event format')

    description = 'In %s %s' % (location, description.split(target)[1])

  event_data = {'when:from': FormatTime(event.get('dtstart').dt),
                'when:to': FormatTime(event.get('dtend').dt),
                'summary': unicode(event.get('summary')),
                'location': location,
                'description': description}
  return uid, event_data

def UpdateSubscription(link, current_user, gcal):
  """
  Updates the GCal instance with the events in link for the current_user

  Returns a list of all uid's for events which have yet to occur (can be used
  to delete removed events)
  """
  result = []
  current_user_id = current_user.user_id()

  valid, link = WhiteList(link)
  if not valid:
    # Do nothing if not on the whitelist
    return result

  import_feed = urlopen(link)
  ical = Calendar.from_string(import_feed.read())
  import_feed.close()

  now = datetime.datetime.utcnow()
  for component in ical.walk():
    if component.name != 'VEVENT':
      logging.info('iCal at %s has unexpected event type %s' % (
        link, component.name))
    else:
      uid, event_data = ParseEvent(component)
      event = Event.get_by_key_name(uid)
      if event is None:
        # Create new event
        # (leaving out the event argument creates a new event)
        event_data['email'] = current_user.email()
        cal_event = AddOrUpdateEvent(event_data, gcal)
        # TODO(dhermes) add to failed queue to be updated by a cron
        if cal_event is None:
          continue

        gcal_edit = cal_event.get_edit_link().href
        event = Event(key_name=uid,
                      who=[current_user_id],  # id is string
                      event_data=db.Text(JsonAscii(event_data)),
                      gcal_edit=gcal_edit)
        event.put()

        # execution has successfully completed
        # TODO(dhermes) catch error on get call below
        if RemoveTimezone(component.get('dtend').dt) > now:
          result.append(uid)
      else:
        # We need to make changes for new event data or a new owner
        if (current_user_id not in event.who or
            db.Text(JsonAscii(event_data)) != event.event_data):
          # Grab GCal event to edit
          attempts = 3
          cal_event = None
          while attempts:
            try:
              cal_event = gcal.GetEventEntry(uri=event.gcal_edit)
              break
            except RedirectError:
              attempts -= 1
              sleep(3)
              pass

          # TODO(dhermes) add to failed queue to be updated by a cron
          if cal_event is None:
            continue

          # Update who
          if current_user_id not in event.who:
            event.who.append(current_user_id)  # id is string

            # add existing event to current_user's calendar
            who_add = gdata.calendar.data.EventWho(email=current_user.email())
            cal_event.who.append(who_add)

          # Update existing event
          if db.Text(JsonAscii(event_data)) != event.event_data:
            event.event_data = db.Text(JsonAscii(event_data))

            # Don't push update to avoid pushing twice (if both changed)
            AddOrUpdateEvent(event_data,
                             gcal,
                             event=cal_event,
                             push_update=False)
            # push_update=False, impossible to have RedirectError

          # Push all updates to calendar event
          attempts = 3
          new_event = None
          while attempts:
            try:
              gcal.Update(cal_event)

              # After all possible changes to the Event instance have occurred
              event.put()
              break
            except RedirectError:
              attempts -= 1
              sleep(3)
              pass

          # If attempts is 0, we have failed and don't want to add the
          # uid to results
          if attempts == 0:
            continue

        # execution has successfully completed
        # TODO(dhermes) catch error on get call below
        if RemoveTimezone(component.get('dtend').dt) > now:
          result.append(uid)

  return result

def UpdateUpcoming(user_cal, upcoming, gcal):
  """
  Updates the GCal instance by deleting pending events removed from ext calendar

  If the new upcoming events list is different from that on the user_cal, it
  will iterate through the difference and delete those events that have not yet
  passed which are still on the calendar.
  """
  if user_cal.upcoming != upcoming:
    now = datetime.datetime.utcnow()
    for uid in set(user_cal.upcoming).difference(upcoming):
      event = Event.get_by_key_name(uid)
      event_data = simplejson.loads(event.event_data)
      if TimeToDTStamp(event_data['when:to']) > now:
        gcal.delete(event.gcal_edit, force=True)
        event.delete()
    user_cal.upcoming = upcoming
    user_cal.put()
<|endoftext|>


<commit_before>import contextlib
import os
import os.path

from httpretty import HTTPretty
import mock
import requests

def get_test_file_path(file_path):
    """translates a file path to be relative to the test files directory"""
    return os.path.join(os.path.dirname(__file__), 'files', file_path)

@contextlib.contextmanager
def mocked_requests(mocked_urls):
    """mocks the requests library to return a given file's content"""
    # if environment variable is set, then don't mock the tests just grab files
    # over the network. Example:
    #    env ULMO_DONT_MOCK_TESTS=1 py.test
    if os.environ.get('ULMO_DONT_MOCK_TESTS', False):
        yield

    else:
        if isinstance(mocked_urls, basestring):
            file_paths = [mocked_urls]
        else:
            file_paths = mocked_urls.values()

        with _open_multiple(file_paths) as open_files:
            if isinstance(mocked_urls, basestring):
                url_files = open_files.values()[0]
            else:
                url_files = dict([
                    (url, open_files.get(file_path))
                    for url, file_path in mocked_urls.iteritems()
                ])
            side_effect = _mock_request_side_effect(url_files)
            with contextlib.nested(
                    mock.patch('requests.get', side_effect=side_effect),
                    mock.patch('requests.head', side_effect=side_effect),
                    ):
                yield

@contextlib.contextmanager
def mocked_suds_client(waterml_version, mocked_service_calls):
    """mocks the suds library to return a given file's content"""
    # if environment variable is set, then don't mock the tests just grab files
    # over the network. Example:
    #    env ULMO_DONT_MOCK_TESTS=1 py.test
    if os.environ.get('ULMO_DONT_MOCK_TESTS', False):
        yield

    else:
        tns_str = 'http://www.cuahsi.org/his/%s/ws/' % waterml_version
        with _open_multiple(mocked_service_calls.values()) as open_files:
            client = mock.MagicMock()
            client.wsdl.tns = ('tns', tns_str)

            for service_call, filename in mocked_service_calls.iteritems():
                open_file = open_files[filename]

                def _func(*args, **kwargs):
                    return open_file.read()
                setattr(client.service, service_call, _func)

            with mock.patch('suds.client.Client', return_value=client):
                yield

@contextlib.contextmanager
def mocked_url(url, response_file_path, request_type='POST'):
    """mocks the underlying python sockets library to return a given file's
    content
    """
    # if environment variable is set, then don't mock the tests just grab files
    # over the network. Example:
    #    env ULMO_DONT_MOCK_TESTS=1 py.test
    if os.environ.get('ULMO_DONT_MOCK_TESTS', False):
        yield

    else:
        with open(os.path.join('files', response_file_path)) as f:
            response = f.read()

        HTTPretty.enable()

        request_class = getattr(HTTPretty, request_type)
        HTTPretty.register_uri(request_class, url, body=response)

        yield
        HTTPretty.disable()

def _mock_request_side_effect(url_files):
    def _side_effect(url, *args, **kwargs):
        mock_response = requests.Response()
        mock_response.request = requests.Request(url, *args, **kwargs)
        mock_response.status_code = 200
        mock_response.url = url
        mock_response.mocked = True
        if isinstance(url_files, dict):
            mock_response.raw = url_files.get(mock_response.request.full_url)
        elif hasattr(url_files, 'read'):
            mock_response.raw = url_files

        # seek to beginning in case this handle has been used before
        mock_response.raw.seek(0)

        return mock_response
    return _side_effect

@contextlib.contextmanager
def _open_multiple(paths, handlers=None, new_paths=None):
    """context manager that yields a dict containing file paths as keys mapped
    to their corresponding open file handles
    """
    # note: recursion is the pattern here
    if not handlers and not new_paths:
        with _open_multiple([], [], paths) as return_dict:
            yield return_dict
    elif len(new_paths) == 0:
        yield dict(zip(paths, handlers))
    else:
        next_path = new_paths.pop(0)
        test_path = get_test_file_path(next_path)
        with open(test_path, 'rb') as f:
            paths.append(next_path)
            handlers.append(f)
            with _open_multiple(paths, handlers, new_paths) as return_dict:
                yield return_dict
<commit_msg>use helper method to find test files path, so that tests can be run from root using setup.py test<commit_after>import contextlib
import os
import os.path

from httpretty import HTTPretty
import mock
import requests

def get_test_file_path(file_path):
    """translates a file path to be relative to the test files directory"""
    return os.path.join(os.path.dirname(__file__), 'files', file_path)

@contextlib.contextmanager
def mocked_requests(mocked_urls):
    """mocks the requests library to return a given file's content"""
    # if environment variable is set, then don't mock the tests just grab files
    # over the network. Example:
    #    env ULMO_DONT_MOCK_TESTS=1 py.test
    if os.environ.get('ULMO_DONT_MOCK_TESTS', False):
        yield

    else:
        if isinstance(mocked_urls, basestring):
            file_paths = [mocked_urls]
        else:
            file_paths = mocked_urls.values()

        with _open_multiple(file_paths) as open_files:
            if isinstance(mocked_urls, basestring):
                url_files = open_files.values()[0]
            else:
                url_files = dict([
                    (url, open_files.get(file_path))
                    for url, file_path in mocked_urls.iteritems()
                ])
            side_effect = _mock_request_side_effect(url_files)
            with contextlib.nested(
                    mock.patch('requests.get', side_effect=side_effect),
                    mock.patch('requests.head', side_effect=side_effect),
                    ):
                yield

@contextlib.contextmanager
def mocked_suds_client(waterml_version, mocked_service_calls):
    """mocks the suds library to return a given file's content"""
    # if environment variable is set, then don't mock the tests just grab files
    # over the network. Example:
    #    env ULMO_DONT_MOCK_TESTS=1 py.test
    if os.environ.get('ULMO_DONT_MOCK_TESTS', False):
        yield

    else:
        tns_str = 'http://www.cuahsi.org/his/%s/ws/' % waterml_version
        with _open_multiple(mocked_service_calls.values()) as open_files:
            client = mock.MagicMock()
            client.wsdl.tns = ('tns', tns_str)

            for service_call, filename in mocked_service_calls.iteritems():
                open_file = open_files[filename]

                def _func(*args, **kwargs):
                    return open_file.read()
                setattr(client.service, service_call, _func)

            with mock.patch('suds.client.Client', return_value=client):
                yield

@contextlib.contextmanager
def mocked_url(url, response_file_path, request_type='POST'):
    """mocks the underlying python sockets library to return a given file's
    content
    """
    # if environment variable is set, then don't mock the tests just grab files
    # over the network. Example:
    #    env ULMO_DONT_MOCK_TESTS=1 py.test
    if os.environ.get('ULMO_DONT_MOCK_TESTS', False):
        yield

    else:
        with open(get_test_file_path(response_file_path)) as f:
            response = f.read()

        HTTPretty.enable()

        request_class = getattr(HTTPretty, request_type)
        HTTPretty.register_uri(request_class, url, body=response)

        yield
        HTTPretty.disable()

def _mock_request_side_effect(url_files):
    def _side_effect(url, *args, **kwargs):
        mock_response = requests.Response()
        mock_response.request = requests.Request(url, *args, **kwargs)
        mock_response.status_code = 200
        mock_response.url = url
        mock_response.mocked = True
        if isinstance(url_files, dict):
            mock_response.raw = url_files.get(mock_response.request.full_url)
        elif hasattr(url_files, 'read'):
            mock_response.raw = url_files

        # seek to beginning in case this handle has been used before
        mock_response.raw.seek(0)

        return mock_response
    return _side_effect

@contextlib.contextmanager
def _open_multiple(paths, handlers=None, new_paths=None):
    """context manager that yields a dict containing file paths as keys mapped
    to their corresponding open file handles
    """
    # note: recursion is the pattern here
    if not handlers and not new_paths:
        with _open_multiple([], [], paths) as return_dict:
            yield return_dict
    elif len(new_paths) == 0:
        yield dict(zip(paths, handlers))
    else:
        next_path = new_paths.pop(0)
        test_path = get_test_file_path(next_path)
        with open(test_path, 'rb') as f:
            paths.append(next_path)
            handlers.append(f)
            with _open_multiple(paths, handlers, new_paths) as return_dict:
                yield return_dict
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import json
import logging
import os
import shutil
import time
import re

from six.moves import configparser, input

from autotorrent.at import AutoTorrent
from autotorrent.clients import TORRENT_CLIENTS
from autotorrent.db import Database
from autotorrent.humanize import humanize_bytes

def query_yes_no(question, default="yes"):
    """Ask a yes/no question via raw_input() and return their answer.

    "question" is a string that is presented to the user.
    "default" is the presumed answer if the user just hits <Enter>.
        It must be "yes" (the default), "no" or None (meaning
        an answer is required of the user).

    The "answer" return value is True for "yes" or False for "no".
    """
    valid = {"yes": True, "y": True, "ye": True,
             "no": False, "n": False}
    if default is None:
        prompt = " [y/n] "
    elif default == "yes":
        prompt = " [Y/n] "
    elif default == "no":
        prompt = " [y/N] "
    else:
        raise ValueError("invalid default answer: '%s'" % default)

    while True:
        print(question + prompt)
        choice = input().lower()
        if default is not None and choice == '':
            return valid[default]
        elif choice in valid:
            return valid[choice]
        else:
            print("Please respond with 'yes' or 'no' (or 'y' or 'n').\n")

def commandline_handler():
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--config", dest="config_file", default="autotorrent.conf", help="Path to config file")
    parser.add_argument("-l", "--client", dest="client", default="default", help="Name of client to use (when multiple configured)")

    parser.add_argument("--create_config", dest="create_config_file", nargs='?', const='autotorrent.conf', default=None, help="Creates a new configuration file")
    parser.add_argument("-t", "--test_connection", action="store_true", dest="test_connection", default=False, help='Tests the connection to the torrent client')
    parser.add_argument("--dry-run", nargs='?', const='txt', default=None, dest="dry_run", choices=['txt', 'json'], help="Don't do any actual adding, just scan for files needed for torrents.")
    parser.add_argument("-r", "--rebuild", dest="rebuild", default=False, help='Rebuild the database', nargs='*')
    parser.add_argument("-a", "--addfile", dest="addfile", default=False, help='Add a new torrent file to client', nargs='+')
    parser.add_argument("-d", "--delete_torrents", action="store_true", dest="delete_torrents", default=False, help='Delete torrents when they are added to the client')
    parser.add_argument("--verbose", help="increase output verbosity", action="store_true", dest="verbose")
    parser.add_argument("-o", "--loopmode", dest="loopmode", default=False, help='Enable loop mode (scan directory '
                                                                                'for torrents every few seconds)', nargs='?')

    args = parser.parse_args()

    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.ERROR)

    if args.create_config_file: # autotorrent.conf
        if os.path.exists(args.create_config_file):
            parser.error("Target %r already exists, not creating" % args.create_config_file)
        else:
            src = os.path.join(os.path.dirname(__file__), 'autotorrent.conf.dist')
            shutil.copy(src, args.create_config_file)
            print('Created configuration file %r' % args.create_config_file)

        client_config = {
            'client': 'rtorrent',
            'url': 'http://user:pass@127.0.0.1/RPC2',
            'label': 'autotorrent',
        }

        if query_yes_no('Do you want to try and auto-configure torrent client?'):
            working_clients = []
            for client_name, cls in TORRENT_CLIENTS.items():
                obj = cls.auto_config()
                try:
                    if obj.test_connection():
                        working_clients.append(obj)
                except:
                    continue

            if working_clients:
                print('Found %i clients - please choose a client to use' % len(working_clients))
                for i, client in enumerate(working_clients, 1):
                    print('[%i] %s' % (i, client.identifier))
                print('[0] None of the above - do not auto-configure any client\n')

                while True:
                    error = False
                    try:
                        choice = int(input('> '))
                    except ValueError:
                        error = True
                    else:
                        if len(working_clients) < choice or choice < 0:
                            error = True

                    if error:
                        print('Invalid choice, please choose again')
                    else:
                        if choice > 0:
                            client = working_clients[choice-1]
                            print('Setting client to %s' % client.identifier)
                            client_config = client.get_config()
                            client_config['client'] = client.identifier

                        break
            else:
                print('Unable to auto-detect any clients, you will have to configure it manually.')

        config = configparser.ConfigParser()
        config.read(args.create_config_file)
        for k, v in client_config.items():
            config.set('client', k, v)

        with open(args.create_config_file, 'w') as configfile:
            config.write(configfile)

        quit()

    if not os.path.isfile(args.config_file):
        parser.error("Config file not found %r" % args.config_file)

    print('Using config file %s' % args.config_file)
    config = configparser.ConfigParser()
    config.read(args.config_file)

    current_path = os.getcwd()
    os.chdir(os.path.dirname(os.path.abspath(args.config_file))) # Changing directory to where the config file is.

    if not config.has_section('general'):
        parser.error('AutoTorrent is not properly configured, please edit %r' % args.config_file)
        quit(1)

    i = 1
    disks = []
    while config.has_option('disks', 'disk%s' % i):
        disks.append(config.get('disks', 'disk%s' % i))
        i += 1

    scan_mode = set(config.get('general', 'scan_mode').split(','))

    exact_mode = 'exact' in scan_mode
    unsplitable_mode = 'unsplitable' in scan_mode
    normal_mode = 'normal' in scan_mode

    hash_name_mode = 'hash_name' in scan_mode
    hash_size_mode = 'hash_size' in scan_mode
    hash_slow_mode = 'hash_slow' in scan_mode

    db = Database(config.get('general', 'db'), disks,
                  config.get('general', 'ignore_files').split(','),
                  normal_mode, unsplitable_mode, exact_mode,
                  hash_name_mode, hash_size_mode, hash_slow_mode)

    client_option = 'client'
    if args.client != 'default':
        client_option += '-%s' % args.client

    try:
        client_name = config.get(client_option, 'client')
    except configparser.NoSectionError:
        print('It seems like %r is not a configured client' % args.client)
        quit(1)

    if client_name not in TORRENT_CLIENTS:
        print('Unknown client %r - Known clients are: %s' % (client_name, ', '.join(TORRENT_CLIENTS.keys())))
        quit(1)

    client_options = dict(config.items(client_option))
    client_options.pop('client')
    client = TORRENT_CLIENTS[client_name](**client_options)

    at = AutoTorrent(
        db,
        client,
        config.get('general', 'store_path'),
        config.getint('general', 'add_limit_size'),
        config.getfloat('general', 'add_limit_percent'),
        args.delete_torrents,
        (config.get('general', 'link_type') if config.has_option('general', 'link_type') else 'soft'),
    )

    if args.test_connection:
        proxy_test_result = client.test_connection()
        if proxy_test_result:
            print('Connected to torrent client successfully!')
            print('  result: %s' % proxy_test_result)

    if isinstance(args.rebuild, list):
        if args.rebuild:
            print('Adding new folders to database')
            db.rebuild(args.rebuild)
            print('Added to database')
        else:
            print('Rebuilding database')
            db.rebuild()
            print('Database rebuilt')

    """if args.addfile:
        addtfile(at, config.get('general', 'store_path'), args.addfile, args.dry_run)"""

    if args.loopmode:
        print('Entering loop mode (%s) (ctrl-c to exit)' % args.loopmode)
        while 1 == 1:
            for fn in os.listdir(args.loopmode):
                if fn.endswith('.torrent'):
                    fn_woext = os.path.splitext(fn)[0]
                    fn_scenename = re.search('-(.*)$', fn_woext).group(1).replace(' ', '.').lower()
                    print('!FOUND %s (%s)' % (fn_woext, fn_scenename))

                    at.populate_torrents_seeded_names()
                    print('!  There is currently %i seeded torrents in client' % len(at.torrents_seeded_names))

                    # Check if torrent exists
                    added = False
                    for thash, tname in at.torrents_seeded_names:
                        if tname == fn_scenename:
                            # If exists, check if seeding
                            seeding = at.get_complete(thash)
                            print('!  This release is already in the client and is %s' % ("seeding" if seeding else
                                                                                "downloading"))
                            # If seeding
                            if seeding:
                                # Add to cross-seed
                                print('!  Adding torrent in cross-seed mode')
                                addtfile(at, args.loopmode, fn, args.dry_run)
                                # delete torrent file
                                print('!  Deleting file')
                                os.remove(os.path.join(args.loopmode, fn))
                                added = True
                            else:
                                print('!  Skipping')
                    if not added:
                        # If not exists, add new
                        print("!  Adding torrent")
                        print('!  Adding new folders to database')
                        db.rebuild(config.get('general', 'store_path'))
                        # delete torrent file
                        print("!  Deleting file")
                        # os.remove(os.path.join(current_path, fn))

            time.sleep(5)

def addtfile(at, current_path, afiles, adry_run):
    print(current_path)
    print(afiles)
    dry_run = bool(adry_run)
    dry_run_data = []
    if not dry_run:
        at.populate_torrents_seeded()

    for torrent in afiles:
        result = at.handle_torrentfile(os.path.join(current_path, torrent), dry_run)
        if dry_run:
            dry_run_data.append({
                'torrent': torrent,
                'found_bytes': result[0],
                'missing_bytes': result[1],
                'would_add': not result[2],
                'local_files': result[3],
            })

    if dry_run:
        if dry_run == 'json':
            print(json.dumps(dry_run_data))
        elif dry_run == 'txt':
            for torrent in dry_run_data:
                print('Torrent: %s' % torrent['torrent'])
                print(' Found data: %s - Missing data: %s - Would add: %s' % (humanize_bytes(torrent['found_bytes']),
                                                                              humanize_bytes(torrent['missing_bytes']),
                                                                              torrent['would_add'] and 'Yes' or 'No'))
                print(' Local files used:')
                for f in torrent['local_files']:
                    print('  %s' % f)
                print('')

if __name__ == '__main__':
    commandline_handler()
<commit_msg>Starting loop mode coding<commit_after>#!/usr/bin/env python

import argparse
import json
import logging
import os
import shutil
import time
import re

from six.moves import configparser, input

from autotorrent.at import AutoTorrent
from autotorrent.clients import TORRENT_CLIENTS
from autotorrent.db import Database
from autotorrent.humanize import humanize_bytes

def query_yes_no(question, default="yes"):
    """Ask a yes/no question via raw_input() and return their answer.

    "question" is a string that is presented to the user.
    "default" is the presumed answer if the user just hits <Enter>.
        It must be "yes" (the default), "no" or None (meaning
        an answer is required of the user).

    The "answer" return value is True for "yes" or False for "no".
    """
    valid = {"yes": True, "y": True, "ye": True,
             "no": False, "n": False}
    if default is None:
        prompt = " [y/n] "
    elif default == "yes":
        prompt = " [Y/n] "
    elif default == "no":
        prompt = " [y/N] "
    else:
        raise ValueError("invalid default answer: '%s'" % default)

    while True:
        print(question + prompt)
        choice = input().lower()
        if default is not None and choice == '':
            return valid[default]
        elif choice in valid:
            return valid[choice]
        else:
            print("Please respond with 'yes' or 'no' (or 'y' or 'n').\n")

def commandline_handler():
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--config", dest="config_file", default="autotorrent.conf", help="Path to config file")
    parser.add_argument("-l", "--client", dest="client", default="default", help="Name of client to use (when multiple configured)")

    parser.add_argument("--create_config", dest="create_config_file", nargs='?', const='autotorrent.conf', default=None, help="Creates a new configuration file")
    parser.add_argument("-t", "--test_connection", action="store_true", dest="test_connection", default=False, help='Tests the connection to the torrent client')
    parser.add_argument("--dry-run", nargs='?', const='txt', default=None, dest="dry_run", choices=['txt', 'json'], help="Don't do any actual adding, just scan for files needed for torrents.")
    parser.add_argument("-r", "--rebuild", dest="rebuild", default=False, help='Rebuild the database', nargs='*')
    parser.add_argument("-a", "--addfile", dest="addfile", default=False, help='Add a new torrent file to client', nargs='+')
    parser.add_argument("-d", "--delete_torrents", action="store_true", dest="delete_torrents", default=False, help='Delete torrents when they are added to the client')
    parser.add_argument("--verbose", help="increase output verbosity", action="store_true", dest="verbose")
    parser.add_argument("-o", "--loopmode", dest="loopmode", default=False, help='Enable loop mode (scan directory '
                                                                                'for torrents every few seconds)', nargs='?')

    args = parser.parse_args()

    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.ERROR)

    if args.create_config_file: # autotorrent.conf
        if os.path.exists(args.create_config_file):
            parser.error("Target %r already exists, not creating" % args.create_config_file)
        else:
            src = os.path.join(os.path.dirname(__file__), 'autotorrent.conf.dist')
            shutil.copy(src, args.create_config_file)
            print('Created configuration file %r' % args.create_config_file)

        client_config = {
            'client': 'rtorrent',
            'url': 'http://user:pass@127.0.0.1/RPC2',
            'label': 'autotorrent',
        }

        if query_yes_no('Do you want to try and auto-configure torrent client?'):
            working_clients = []
            for client_name, cls in TORRENT_CLIENTS.items():
                obj = cls.auto_config()
                try:
                    if obj.test_connection():
                        working_clients.append(obj)
                except:
                    continue

            if working_clients:
                print('Found %i clients - please choose a client to use' % len(working_clients))
                for i, client in enumerate(working_clients, 1):
                    print('[%i] %s' % (i, client.identifier))
                print('[0] None of the above - do not auto-configure any client\n')

                while True:
                    error = False
                    try:
                        choice = int(input('> '))
                    except ValueError:
                        error = True
                    else:
                        if len(working_clients) < choice or choice < 0:
                            error = True

                    if error:
                        print('Invalid choice, please choose again')
                    else:
                        if choice > 0:
                            client = working_clients[choice-1]
                            print('Setting client to %s' % client.identifier)
                            client_config = client.get_config()
                            client_config['client'] = client.identifier

                        break
            else:
                print('Unable to auto-detect any clients, you will have to configure it manually.')

        config = configparser.ConfigParser()
        config.read(args.create_config_file)
        for k, v in client_config.items():
            config.set('client', k, v)

        with open(args.create_config_file, 'w') as configfile:
            config.write(configfile)

        quit()

    if not os.path.isfile(args.config_file):
        parser.error("Config file not found %r" % args.config_file)

    print('Using config file %s' % args.config_file)
    config = configparser.ConfigParser()
    config.read(args.config_file)

    current_path = os.getcwd()
    os.chdir(os.path.dirname(os.path.abspath(args.config_file))) # Changing directory to where the config file is.

    if not config.has_section('general'):
        parser.error('AutoTorrent is not properly configured, please edit %r' % args.config_file)
        quit(1)

    i = 1
    disks = []
    while config.has_option('disks', 'disk%s' % i):
        disks.append(config.get('disks', 'disk%s' % i))
        i += 1

    scan_mode = set(config.get('general', 'scan_mode').split(','))

    exact_mode = 'exact' in scan_mode
    unsplitable_mode = 'unsplitable' in scan_mode
    normal_mode = 'normal' in scan_mode

    hash_name_mode = 'hash_name' in scan_mode
    hash_size_mode = 'hash_size' in scan_mode
    hash_slow_mode = 'hash_slow' in scan_mode

    db = Database(config.get('general', 'db'), disks,
                  config.get('general', 'ignore_files').split(','),
                  normal_mode, unsplitable_mode, exact_mode,
                  hash_name_mode, hash_size_mode, hash_slow_mode)

    client_option = 'client'
    if args.client != 'default':
        client_option += '-%s' % args.client

    try:
        client_name = config.get(client_option, 'client')
    except configparser.NoSectionError:
        print('It seems like %r is not a configured client' % args.client)
        quit(1)

    if client_name not in TORRENT_CLIENTS:
        print('Unknown client %r - Known clients are: %s' % (client_name, ', '.join(TORRENT_CLIENTS.keys())))
        quit(1)

    client_options = dict(config.items(client_option))
    client_options.pop('client')
    client = TORRENT_CLIENTS[client_name](**client_options)

    at = AutoTorrent(
        db,
        client,
        config.get('general', 'store_path'),
        config.getint('general', 'add_limit_size'),
        config.getfloat('general', 'add_limit_percent'),
        args.delete_torrents,
        (config.get('general', 'link_type') if config.has_option('general', 'link_type') else 'soft'),
    )

    if args.test_connection:
        proxy_test_result = client.test_connection()
        if proxy_test_result:
            print('Connected to torrent client successfully!')
            print('  result: %s' % proxy_test_result)

    if isinstance(args.rebuild, list):
        if args.rebuild:
            print('Adding new folders to database')
            db.rebuild(args.rebuild)
            print('Added to database')
        else:
            print('Rebuilding database')
            db.rebuild()
            print('Database rebuilt')

    """if args.addfile:
        addtfile(at, config.get('general', 'store_path'), args.addfile, args.dry_run)"""

    if args.loopmode:
        print('Entering loop mode (%s) (ctrl-c to exit)' % args.loopmode)
        while 1 == 1:
            for fn in os.listdir(args.loopmode):
                if fn.endswith('.torrent'):
                    fn_woext = os.path.splitext(fn)[0]
                    fn_scenename = re.search('-(.*)$', fn_woext).group(1).replace(' ', '.').lower()
                    print('!FOUND %s (%s)' % (fn_woext, fn_scenename))

                    at.populate_torrents_seeded_names()
                    print('!  There is currently %i seeded torrents in client' % len(at.torrents_seeded_names))

                    # Check if torrent exists
                    added = False
                    for thash, tname in at.torrents_seeded_names:
                        if tname == fn_scenename:
                            # If exists, check if seeding
                            seeding = at.get_complete(thash)
                            print('!  This release is already in the client and is %s' % ("seeding" if seeding else
                                                                                "downloading"))
                            # If seeding
                            if seeding:
                                # Add to cross-seed
                                print('!  Adding torrent in cross-seed mode')
                                addtfile(at, args.loopmode, set(fn), args.dry_run)
                                # delete torrent file
                                print('!  Deleting file')
                                os.remove(os.path.join(args.loopmode, fn))
                                added = True
                            else:
                                print('!  Skipping')
                    if not added:
                        # If not exists, add new
                        print("!  Adding torrent")
                        print('!  Adding new folders to database')
                        db.rebuild(config.get('general', 'store_path'))
                        # delete torrent file
                        print("!  Deleting file")
                        # os.remove(os.path.join(current_path, fn))

            time.sleep(5)

def addtfile(at, current_path, afiles, adry_run):
    print(current_path)
    print(afiles)
    dry_run = bool(adry_run)
    dry_run_data = []
    if not dry_run:
        at.populate_torrents_seeded()

    for torrent in afiles:
        result = at.handle_torrentfile(os.path.join(current_path, torrent), dry_run)
        if dry_run:
            dry_run_data.append({
                'torrent': torrent,
                'found_bytes': result[0],
                'missing_bytes': result[1],
                'would_add': not result[2],
                'local_files': result[3],
            })

    if dry_run:
        if dry_run == 'json':
            print(json.dumps(dry_run_data))
        elif dry_run == 'txt':
            for torrent in dry_run_data:
                print('Torrent: %s' % torrent['torrent'])
                print(' Found data: %s - Missing data: %s - Would add: %s' % (humanize_bytes(torrent['found_bytes']),
                                                                              humanize_bytes(torrent['missing_bytes']),
                                                                              torrent['would_add'] and 'Yes' or 'No'))
                print(' Local files used:')
                for f in torrent['local_files']:
                    print('  %s' % f)
                print('')

if __name__ == '__main__':
    commandline_handler()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
daily work, clean expired items and its place in queues in memory
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Aug. 22, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append("..")

from bson.objectid import ObjectId
from config import Collection, db
from config import rclient
import time
import cleaner

def clean_by_item(item_id):
    """
    remove item-related things in memory
    """
    if not item_id:
        return None
    if not rclient.exists(item_id):
        return None
    # it's possible item_id in queues be visited
    # success returns 1, else 0
    return rclient.delete(item_id)

def _is_zombie(item):
    """
    zombies are items in memory but not in database
    """
    if not item:
        return True

    document_name = item['language']
    document = Collection(db, document_name)
    item_id = ObjectId(item['_id'])
    found = document.find_one({'_id': item_id})
    if not found:
        return True
    else:
        return False

def clean_by_collection():
    """
    removed a group of expired items in memory
    """
    pass

def clean():
    """
    remove expired items from queues in memory
    walk through all redis content
    """
    print '... cleaning memory ...'
    news_lists = rclient.keys('news::*')
    for news_list in news_lists:
        # get the total number of a news list
        news_list_count = rclient.zcard(news_list)
        # get all the ids in a news list
        if news_list_count:
            news_ids = rclient.zrange(news_list, 0, news_list_count)
            for news_id in news_ids:
                # make sure every item is touched
                if not rclient.exists(news_id):
                    rclient.zrem(news_list, news_id)
                else:
                    news_item_string = rclient.get(news_id)
                    if news_item_string:
                        news_item = eval(news_item_string)
                        news_updated = float(news_item['updated'])

                        if cleaner.is_overdue(news_updated):  # WTF, remove it
                            rclient.zrem(news_list, news_id)
                            rclient.delete(news_id)
                        else:  # check if this is zombie
                            if _is_zombie(news_item):
                                rclient.zrem(news_list, news_id)
                                rclient.delete(news_id)
                    else:
                        rclient.zrem(news_list, news_id)
                        rclient.delete(news_id)

if __name__ == "__main__":
    clean()
<commit_msg>updated clean_memory.py<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
daily work, clean expired items and its place in queues in memory
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Aug. 22, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append("..")

from bson.objectid import ObjectId
from config import Collection, db
from config import rclient
import time
import cleaner

def _is_zombie(item):
    """
    zombies are items in memory but not in database
    """
    if not item:
        return True

    document_name = item['language']
    document = Collection(db, document_name)
    item_id = ObjectId(item['_id'])
    found = document.find_one({'_id': item_id})
    if not found:
        return True
    else:
        return False

def clean_by_item(item_id):
    """
    remove item-related things in memory
    """
    if not item_id:
        return None
    if not rclient.exists(item_id):
        return None
    # it's possible item_id in queues be visited
    # success returns 1, else 0
    return rclient.delete(item_id)

def clean_by_items():
    """
    removed a group of expired items in memory
    """
    pass

def clean():
    """
    remove expired items from queues in memory
    walk through all redis content
    """
    print '... cleaning memory ...'
    news_lists = rclient.keys('news::*')
    for news_list in news_lists:
        # get the total number of a news list
        news_list_count = rclient.zcard(news_list)
        # get all the ids in a news list
        if news_list_count:
            news_ids = rclient.zrange(news_list, 0, news_list_count)
            for news_id in news_ids:
                # make sure every item is touched
                if not rclient.exists(news_id):
                    rclient.zrem(news_list, news_id)
                else:
                    news_item_string = rclient.get(news_id)
                    if news_item_string:
                        news_item = eval(news_item_string)
                        news_updated = float(news_item['updated'])

                        if cleaner.is_overdue(news_updated):  # WTF, remove it
                            rclient.zrem(news_list, news_id)
                            rclient.delete(news_id)
                        else:  # check if this is zombie
                            if _is_zombie(news_item):
                                rclient.zrem(news_list, news_id)
                                rclient.delete(news_id)
                    else:
                        rclient.zrem(news_list, news_id)
                        rclient.delete(news_id)

if __name__ == "__main__":
    clean()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from tests.test_data import DummyCase

class DatetimeTestCase(DummyCase):
    def test_day_of_week(self):
        result = self.generic.datetime.day_of_week()
        self.assertIn(result, self.generic.datetime.data['day']['name'])

        result_abbr = self.generic.datetime.day_of_week(abbr=True)
        self.assertIn(result_abbr, self.generic.datetime.data['day']['abbr'])

    def test_month(self):
        result = self.generic.datetime.month() + '\n'
        self.assertGreater(len(result), 3)

        result_abbr = self.generic.datetime.month(abbr=True)
        self.assertIsInstance(result_abbr, str)

    def test_year(self):
        result = self.generic.datetime.year(minimum=2000, maximum=2016)
        self.assertTrue((result >= 2000) and (result <= 2016))

    def test_periodicity(self):
        result = self.generic.datetime.periodicity()
        self.assertIn(result, self.generic.datetime.data['periodicity'])

    def test_day_of_month(self):
        result = self.generic.datetime.day_of_month()
        self.assertTrue((result >= 1) or (result <= 31))

    def test_birthday(self):
        result = self.generic.datetime.birthday()
        self.assertIsInstance(result, str)

        not_readable = self.generic.datetime.birthday(readable=False)
        day, month, year = not_readable.split('-')
        self.assertTrue(int(day) <= 31)
        self.assertTrue(int(month) <= 12)
        self.assertTrue((int(year) >= 1980) and (int(year) <= 2000))

        fmt_year = self.generic.datetime.birthday(minimum=2015, maximum=2025, readable=False, fmt='%Y')
        self.assertGreaterEqual(int(fmt_year), 2015)
        self.assertLessEqual(int(fmt_year), 2025)

    def test_date(self):
        # Default values: date(sep='-', start=2000, end=2035, with_time=False)
        result = self.generic.datetime.date()
        d, m, y = result.split('-')
        self.assertTrue(int(d) <= 31)
        self.assertTrue(int(m) <= 12)
        self.assertTrue(int(y) >= 2000)
        self.assertTrue(int(y) <= 2035)

        result = self.generic.datetime.date(with_time=True)
        hour, minutes = result.split(' ')[1].split(':')
        self.assertTrue(int(hour) <= 24)
        self.assertTrue(int(minutes) <= 60)

    def test_time(self):
        result = self.generic.datetime.time()
        hour, minutes = result.split(':')
        self.assertTrue(int(hour) <= 24)
        self.assertTrue(int(minutes) <= 60)
<commit_msg>Fix issue<commit_after># -*- coding: utf-8 -*-

from tests.test_data import DummyCase

class DatetimeTestCase(DummyCase):
    def test_day_of_week(self):
        result = self.generic.datetime.day_of_week()
        self.assertIn(result, self.generic.datetime.data['day']['name'])

        result_abbr = self.generic.datetime.day_of_week(abbr=True)
        self.assertIn(result_abbr, self.generic.datetime.data['day']['abbr'])

    def test_month(self):
        result = self.generic.datetime.month()
        self.assertIsNotNone(result)

        result_abbr = self.generic.datetime.month(abbr=True)
        self.assertIsInstance(result_abbr, str)

    def test_year(self):
        result = self.generic.datetime.year(minimum=2000, maximum=2016)
        self.assertTrue((result >= 2000) and (result <= 2016))

    def test_periodicity(self):
        result = self.generic.datetime.periodicity()
        self.assertIn(result, self.generic.datetime.data['periodicity'])

    def test_day_of_month(self):
        result = self.generic.datetime.day_of_month()
        self.assertTrue((result >= 1) or (result <= 31))

    def test_birthday(self):
        result = self.generic.datetime.birthday()
        self.assertIsInstance(result, str)

        not_readable = self.generic.datetime.birthday(readable=False)
        day, month, year = not_readable.split('-')
        self.assertTrue(int(day) <= 31)
        self.assertTrue(int(month) <= 12)
        self.assertTrue((int(year) >= 1980) and (int(year) <= 2000))

        fmt_year = self.generic.datetime.birthday(minimum=2015, maximum=2025, readable=False, fmt='%Y')
        self.assertGreaterEqual(int(fmt_year), 2015)
        self.assertLessEqual(int(fmt_year), 2025)

    def test_date(self):
        # Default values: date(sep='-', start=2000, end=2035, with_time=False)
        result = self.generic.datetime.date()
        d, m, y = result.split('-')
        self.assertTrue(int(d) <= 31)
        self.assertTrue(int(m) <= 12)
        self.assertTrue(int(y) >= 2000)
        self.assertTrue(int(y) <= 2035)

        result = self.generic.datetime.date(with_time=True)
        hour, minutes = result.split(' ')[1].split(':')
        self.assertTrue(int(hour) <= 24)
        self.assertTrue(int(minutes) <= 60)

    def test_time(self):
        result = self.generic.datetime.time()
        hour, minutes = result.split(':')
        self.assertTrue(int(hour) <= 24)
        self.assertTrue(int(minutes) <= 60)
<|endoftext|>


<commit_before>#!/bin/python3
# -*- coding: utf-8 -*-
import json
import urllib.request
import github_api_util
import github_ubtue_util

ZEDER_URL_IXTHEO = 'http://www-ub.ub.uni-tuebingen.de/zeder/cgi-bin/zeder.cgi?action=get&Dimension=wert&Instanz=ixtheo&Bearbeiter='
ZEDER_URL_KRIMDOK = 'http://www-ub.ub.uni-tuebingen.de/zeder/cgi-bin/zeder.cgi?action=get&Dimension=wert&Instanz=krim&Bearbeiter='

def GetDataFromZeder(zeder_url):
    response = urllib.request.urlopen(zeder_url)
    jdata = json.load(response)
    return jdata

def HasZotAut(item, zota_number_code):
    return item.get('prodf') == zota_number_code # zota == 8 for IxTheo instance and zota == 6 for KrimDok instance

def GetZederZotaNumberCode(zeder_instance):
    for key in zeder_instance['meta']:
        if key['Kurz'] == "prodf":
            for option in key['Optionen']:
                if option['wert'] == "zota":
                   return str(option['id'])
    return "Unknown"

def GetZederZotAutStatusForISSN(issn, zeder_instances):
    for zeder_instance in zeder_instances:
        zota_number_code = GetZederZotaNumberCode(zeder_instance) # We have different number codes for zota depending on the instance
        if zota_number_code == "Unknown":
            raise Exception("Could not determine Id for Zeder zota")
        for item in zeder_instance['daten']:
             if 'essn' in item and item['essn'] == issn:
                 if HasZotAut(item, zota_number_code):
                     return True
             elif 'issn' in item and item['issn'] == issn:
                 if  HasZotAut(item, zota_number_code):
                     return True
    return False

def TagZoteroJournalStatusZotAutFromZeder():
    ixtheo_zeder = GetDataFromZeder(ZEDER_URL_IXTHEO)
    krimdok_zeder = GetDataFromZeder(ZEDER_URL_KRIMDOK)
    zotero_journal_status_issues = github_api_util.GetAllIssuesForUBTueRepository("zotero-journal-status")
    for issue in zotero_journal_status_issues:
        issn_matcher = github_api_util.GetISSNMatcher()
        issns = issn_matcher.findall(issue['title'])
        zotaut_status = False
        if len(issns):
            for issn in issns:
                zotaut_status = GetZederZotAutStatusForISSN(issn, [ixtheo_zeder, krimdok_zeder])
                if zotaut_status:
                    print("ZOTAUT_STATUS TRUE")
                    break
        if zotaut_status:
            print ("ZOTAT")
            new_labels = github_ubtue_util.AdjustZoteroStatusLabels(issue, [ github_ubtue_util.ZOTAUT ], [])
            if not github_ubtue_util.LabelsAreIdentical(issue, new_labels):
                issue_number = str(issue['number'])
                data = { "labels" : new_labels }
                github_api_util.UpdateIssueInRepository('zotero-journal-status', issue_number, data)

def Main():
    TagZoteroJournalStatusZotAutFromZeder()

if __name__ == "__main__":
    Main()
<commit_msg>Remove debug message<commit_after>#!/bin/python3
# -*- coding: utf-8 -*-
import json
import urllib.request
import github_api_util
import github_ubtue_util

ZEDER_URL_IXTHEO = 'http://www-ub.ub.uni-tuebingen.de/zeder/cgi-bin/zeder.cgi?action=get&Dimension=wert&Instanz=ixtheo&Bearbeiter='
ZEDER_URL_KRIMDOK = 'http://www-ub.ub.uni-tuebingen.de/zeder/cgi-bin/zeder.cgi?action=get&Dimension=wert&Instanz=krim&Bearbeiter='

def GetDataFromZeder(zeder_url):
    response = urllib.request.urlopen(zeder_url)
    jdata = json.load(response)
    return jdata

def HasZotAut(item, zota_number_code):
    return item.get('prodf') == zota_number_code # zota == 8 for IxTheo instance and zota == 6 for KrimDok instance

def GetZederZotaNumberCode(zeder_instance):
    for key in zeder_instance['meta']:
        if key['Kurz'] == "prodf":
            for option in key['Optionen']:
                if option['wert'] == "zota":
                   return str(option['id'])
    return "Unknown"

def GetZederZotAutStatusForISSN(issn, zeder_instances):
    for zeder_instance in zeder_instances:
        zota_number_code = GetZederZotaNumberCode(zeder_instance) # We have different number codes for zota depending on the instance
        if zota_number_code == "Unknown":
            raise Exception("Could not determine Id for Zeder zota")
        for item in zeder_instance['daten']:
             if 'essn' in item and item['essn'] == issn:
                 if HasZotAut(item, zota_number_code):
                     return True
             elif 'issn' in item and item['issn'] == issn:
                 if  HasZotAut(item, zota_number_code):
                     return True
    return False

def TagZoteroJournalStatusZotAutFromZeder():
    ixtheo_zeder = GetDataFromZeder(ZEDER_URL_IXTHEO)
    krimdok_zeder = GetDataFromZeder(ZEDER_URL_KRIMDOK)
    zotero_journal_status_issues = github_api_util.GetAllIssuesForUBTueRepository("zotero-journal-status")
    for issue in zotero_journal_status_issues:
        issn_matcher = github_api_util.GetISSNMatcher()
        issns = issn_matcher.findall(issue['title'])
        zotaut_status = False
        if len(issns):
            for issn in issns:
                zotaut_status = GetZederZotAutStatusForISSN(issn, [ixtheo_zeder, krimdok_zeder])
                if zotaut_status:
                    break
        if zotaut_status:
            print ("ZOTAT")
            new_labels = github_ubtue_util.AdjustZoteroStatusLabels(issue, [ github_ubtue_util.ZOTAUT ], [])
            if not github_ubtue_util.LabelsAreIdentical(issue, new_labels):
                issue_number = str(issue['number'])
                data = { "labels" : new_labels }
                github_api_util.UpdateIssueInRepository('zotero-journal-status', issue_number, data)

def Main():
    TagZoteroJournalStatusZotAutFromZeder()

if __name__ == "__main__":
    Main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from math import pi, sqrt
from numpy import sinc, exp, inf
from scipy.integrate import quad, quadrature
import numpy as np
from scipy.optimize import newton_krylov
from numpy import zeros_like, mgrid, zeros, exp
import matplotlib.pyplot as plt

import database

#Defining constants
kb = 8.61735E-5     # Boltzmann constant in eV/K
hb = 6.58212E-16    # Plank's constant(h) in eV*s
q0 = 1.60218E-19    # electron charge (C)
m0 = 9.10938E-31    # electron mass (kg)
m0 = m0/q0          # scale mass to eV
eps0 = 8.85419E-12  # vacuum permittivity in F/m

def bandgap(Eg0, Eg_alpha, Eg_betta, T):
    """Temperature dependence of energy bandgap"""
    return Eg0 - Eg_alpha*T**2/(T+ Eg_betta)

class Material(object):
    """Class for material properties.

    attributes:
    Eg0: energy band gap at 0K
    Eg_alpha: parameter  for Eg temperature dependence
    Eg_betta: parameter  for Eg temperature dependence
    m_e: conduction band effective mass (relative to electron mass)
    m_hh: heavy hole band effective mass
    m_lh: light hole band effective mass
    eps: dielectric constant
    Ea: ionization energy
    g: degeneracy factor
    """
    def __init__(self, material, dopand, database=database):
        self.material = material
        self.dopand = dopand
        self.material_property = database.materialproperty
        matprops = self.material_property[material]
        self.Eg0 =  matprops['Eg0']
        self.Eg_alpha =  matprops['Eg_alpha']
        self.Eg_betta =  matprops['Eg_betta']
        self.m_e = matprops['m_e']*m0
        self.m_hh = matprops['m_hh']*m0
        self.m_lh = matprops['m_lh']*m0
        self.eps = matprops['eps']
        self.Ea = matprops['impurity'][dopand]['Ea']
        self.g = matprops['impurity'][dopand]['g']
        self.dop_type = matprops['impurity'][dopand]['type']

class Structure(object):
    """Class for structure properties.

    attributes:
    T: temperate
    Nd: donor concentration
    Na: acceptor concentration
    Eg: energy band gap
    m_eff: effective mass
    eps: dielectric constant
    Ea: ionization energy
    g: degeneracy factor
    """
    def __init__(self, mat, T=300, Nd=0, Na=0):
        self.material = mat
        self.T = T
        self.Nd = Nd
        self.Na = Na
        self.Eg = bandgap(mat.Eg0, mat.Eg_alpha, mat.Eg_betta, self.T)
        self.eps = mat.eps
        self.Ea = mat.Ea
        self.g = mat.g
        if mat.dop_type == 'donor':
            self.m_eff = mat.m_e

    C = (2*self.m_eff/(hb**2))**(3./2)/(2*pi**2)
    L = hb/sqrt(2*self.m_eff*kb*self.T) # Fermi length
    alpha = (1-self.m_eff/m0)**2/self.Eg # nonparabolicity factor

    def cdos(E, Ec, z):
        """returns modified local density of states in conduction
        band"""
        if E >= Ec:
            E = E - Ec
            # correction factor
            f = 1 - sinc((2*z/L)*sqrt(E/kb/T)*sqrt(1+alpha*E)/pi)
            return C*sqrt(E)*sqrt(1+alpha*E)*(1+2*alpha*E)*f
        else:
            return 0
    <commit_msg>cdos function updated<commit_after># -*- coding: utf-8 -*-
from math import pi, sqrt
from numpy import sinc, exp, inf
from scipy.integrate import quad, quadrature
import numpy as np
from scipy.optimize import newton_krylov
from numpy import zeros_like, mgrid, zeros, exp
import matplotlib.pyplot as plt

import database

#Defining constants
kb = 8.61735E-5     # Boltzmann constant in eV/K
hb = 6.58212E-16    # Plank's constant(h) in eV*s
q0 = 1.60218E-19    # electron charge (C)
m0 = 9.10938E-31    # electron mass (kg)
m0 = m0/q0          # scale mass to eV
eps0 = 8.85419E-12  # vacuum permittivity in F/m

def bandgap(Eg0, Eg_alpha, Eg_betta, T):
    """Temperature dependence of energy bandgap"""
    return Eg0 - Eg_alpha*T**2/(T+ Eg_betta)

class Material(object):
    """Class for material properties.

    attributes:
    Eg0: energy band gap at 0K
    Eg_alpha: parameter  for Eg temperature dependence
    Eg_betta: parameter  for Eg temperature dependence
    m_e: conduction band effective mass (relative to electron mass)
    m_hh: heavy hole band effective mass
    m_lh: light hole band effective mass
    eps: dielectric constant
    Ea: ionization energy
    g: degeneracy factor
    """
    def __init__(self, material, dopand, database=database):
        self.material = material
        self.dopand = dopand
        self.material_property = database.materialproperty
        matprops = self.material_property[material]
        self.Eg0 =  matprops['Eg0']
        self.Eg_alpha =  matprops['Eg_alpha']
        self.Eg_betta =  matprops['Eg_betta']
        self.m_e = matprops['m_e']*m0
        self.m_hh = matprops['m_hh']*m0
        self.m_lh = matprops['m_lh']*m0
        self.eps = matprops['eps']
        self.Ea = matprops['impurity'][dopand]['Ea']
        self.g = matprops['impurity'][dopand]['g']
        self.dop_type = matprops['impurity'][dopand]['type']

class Structure(object):
    """Class for structure properties.

    attributes:
    T: temperate
    Nd: donor concentration
    Na: acceptor concentration
    Eg: energy band gap
    m_eff: effective mass
    eps: dielectric constant
    Ea: ionization energy
    g: degeneracy factor
    """
    def __init__(self, mat, T=300, Nd=0, Na=0):
        self.material = mat
        self.T = T
        self.Nd = Nd
        self.Na = Na
        self.Eg = bandgap(mat.Eg0, mat.Eg_alpha, mat.Eg_betta, self.T)
        self.eps = mat.eps
        self.Ea = mat.Ea
        self.g = mat.g
        if mat.dop_type == 'donor':
            self.m_eff = mat.m_e

    def cdos(self, E, Ec, z):
        """returns modified local density of states in conduction
        band"""
        C = (2*self.m_eff/(hb**2))**(3./2)/(2*pi**2)
        L = hb/sqrt(2*self.m_eff*kb*self.T) # Fermi length
        # nonparabolicity factor
        alpha = (1-self.m_eff/m0)**2/self.Eg
        if E >= Ec:
            E = E - Ec
            # correction factor
            f = 1 - sinc((2*z/L)*sqrt(E/kb/self.T)*sqrt(1+alpha*E)/pi)
            return C*sqrt(E)*sqrt(1+alpha*E)*(1+2*alpha*E)*f
        else:
            return 0

    def fd(self, E, Ef):
        """ Fermi-Dirac function"""
        return 1/(1+exp((E-Ef)/kb/self.T))

    def ced(Ec, z, Ef):
        """depth distribution of  conduction electron density"""
        <|endoftext|>


<commit_before># configuration steps import
import subprocess
import traceback

from nedgeBlockerException import NedgeBlockerException
from steps.baseConfigurationStep import BaseConfigurationStep
from steps.nedeployRCConfig import NedeployRCConfig
from steps.nedeployBashActivation import NedeployBashActivation
from steps.nedeployInstall import NedeployInstall
from steps.nedeployPrecheck import NedeployPrecheck
from steps.neadmRCConfig import NeadmRCConfig
from steps.neadmInitWait import NeadmInitWait
from steps.neadmSystemInit import NeadmSystemInit
from steps.neadmLicenseActivation import NeadmLicenseActivation
from steps.neadmOnlineNodesWait import NeadmOnlineNodesWait
from steps.neadmClusterCreation import NeadmClusterCreation
from steps.waitAuditService import WaitAuditService
from steps.waitNodeUUID import WaitNodeUUID
from steps.systemPreConfig import SystemPreConfig
from steps.systemPostConfig import SystemPostConfig

class NedgeBaseConfigurator:
    def __init__(self, environment={}, steps=[]):
        self.environment = environment
        self.steps = steps
        self.blockers = []

    def configure(self):
        print('Configuration started')

        #reset blockers
        self.blockers = []

        try:
            for step in self.steps:
                if isinstance(step, BaseConfigurationStep):
                    # configuration step virtual method
                    step.print_step_name()
                    step.process(self.environment)
                else:
                    print('WARNING: There is unknown object'
                          'in configuration steps!')

            return True
        except subprocess.CalledProcessError as cpe:
            print('Failed!\nMessage:\n{0}\nTrace:\n{1}\nOutput:\n{2}'
                  .format(cpe.message, traceback.format_exc(), cpe.output))
            return False
        except NedgeBlockerException as nbe:
            print('Got blocker configuration exception')
            self.blockers = nbe.blockers
            return False
        except Exception as e:
            print("Nedge configuration failed. Terminating")
            print('{}'.format(e.message))
            print('Traceback in {}'.format(traceback.format_exc()))
            return False

    def get_blockers():
        return self.blockers

class NedgeNodeConfigurator(NedgeBaseConfigurator):
    _steps = [SystemPreConfig(),
              NedeployRCConfig(),
              NedeployBashActivation(),
              NedeployPrecheck(),
              NedeployInstall(),
              WaitAuditService(),
              WaitNodeUUID(),
              SystemPostConfig()]

    def __init__(self, environment={}):
        environment['node_type'] = 'data'
        NedgeBaseConfigurator.__init__(self, environment,
                                       NedgeNodeConfigurator._steps)

class NedgeGatewayConfigurator(NedgeBaseConfigurator):
    _steps = [SystemPreConfig(),
              NedeployRCConfig(),
              NedeployBashActivation(),
              NedeployPrecheck(),
              NedeployInstall(),
              WaitAuditService(),
              WaitNodeUUID(),
              SystemPostConfig()]

    def __init__(self, environment={}):
        environment['node_type'] = 'gateway'
        NedgeBaseConfigurator.__init__(self, environment,
                                       NedgeNodeConfigurator._steps)

class NedgeMgmtConfigurator(NedgeBaseConfigurator):
    _steps = [
        SystemPreConfig(),
        NedeployRCConfig(),
        NedeployBashActivation(),
        NedeployPrecheck(),
        NedeployInstall(),
        WaitAuditService(),
        NeadmRCConfig(),
        NeadmInitWait(),
        NeadmSystemInit(),
        WaitNodeUUID(),
        NeadmLicenseActivation(),
        NeadmOnlineNodesWait(),
        NeadmClusterCreation(),
        WaitNodeUUID(),
        SystemPostConfig()
    ]

    def __init__(self, environment={}):
        environment['node_type'] = 'mgmt'
        NedgeBaseConfigurator.__init__(self, environment,
                                       NedgeMgmtConfigurator._steps)
<commit_msg>TA18271 fix method syntax<commit_after># configuration steps import
import subprocess
import traceback

from nedgeBlockerException import NedgeBlockerException
from steps.baseConfigurationStep import BaseConfigurationStep
from steps.nedeployRCConfig import NedeployRCConfig
from steps.nedeployBashActivation import NedeployBashActivation
from steps.nedeployInstall import NedeployInstall
from steps.nedeployPrecheck import NedeployPrecheck
from steps.neadmRCConfig import NeadmRCConfig
from steps.neadmInitWait import NeadmInitWait
from steps.neadmSystemInit import NeadmSystemInit
from steps.neadmLicenseActivation import NeadmLicenseActivation
from steps.neadmOnlineNodesWait import NeadmOnlineNodesWait
from steps.neadmClusterCreation import NeadmClusterCreation
from steps.waitAuditService import WaitAuditService
from steps.waitNodeUUID import WaitNodeUUID
from steps.systemPreConfig import SystemPreConfig
from steps.systemPostConfig import SystemPostConfig

class NedgeBaseConfigurator:
    def __init__(self, environment={}, steps=[]):
        self.environment = environment
        self.steps = steps
        self.blockers = []

    def configure(self):
        print('Configuration started')

        #reset blockers
        self.blockers = []

        try:
            for step in self.steps:
                if isinstance(step, BaseConfigurationStep):
                    # configuration step virtual method
                    step.print_step_name()
                    step.process(self.environment)
                else:
                    print('WARNING: There is unknown object'
                          'in configuration steps!')

            return True
        except subprocess.CalledProcessError as cpe:
            print('Failed!\nMessage:\n{0}\nTrace:\n{1}\nOutput:\n{2}'
                  .format(cpe.message, traceback.format_exc(), cpe.output))
            return False
        except NedgeBlockerException as nbe:
            print('Got blocker configuration exception')
            self.blockers = nbe.blockers
            return False
        except Exception as e:
            print("Nedge configuration failed. Terminating")
            print('{}'.format(e.message))
            print('Traceback in {}'.format(traceback.format_exc()))
            return False

    def get_blockers(self):
        return self.blockers

class NedgeNodeConfigurator(NedgeBaseConfigurator):
    _steps = [SystemPreConfig(),
              NedeployRCConfig(),
              NedeployBashActivation(),
              NedeployPrecheck(),
              NedeployInstall(),
              WaitAuditService(),
              WaitNodeUUID(),
              SystemPostConfig()]

    def __init__(self, environment={}):
        environment['node_type'] = 'data'
        NedgeBaseConfigurator.__init__(self, environment,
                                       NedgeNodeConfigurator._steps)

class NedgeGatewayConfigurator(NedgeBaseConfigurator):
    _steps = [SystemPreConfig(),
              NedeployRCConfig(),
              NedeployBashActivation(),
              NedeployPrecheck(),
              NedeployInstall(),
              WaitAuditService(),
              WaitNodeUUID(),
              SystemPostConfig()]

    def __init__(self, environment={}):
        environment['node_type'] = 'gateway'
        NedgeBaseConfigurator.__init__(self, environment,
                                       NedgeNodeConfigurator._steps)

class NedgeMgmtConfigurator(NedgeBaseConfigurator):
    _steps = [
        SystemPreConfig(),
        NedeployRCConfig(),
        NedeployBashActivation(),
        NedeployPrecheck(),
        NedeployInstall(),
        WaitAuditService(),
        NeadmRCConfig(),
        NeadmInitWait(),
        NeadmSystemInit(),
        WaitNodeUUID(),
        NeadmLicenseActivation(),
        NeadmOnlineNodesWait(),
        NeadmClusterCreation(),
        WaitNodeUUID(),
        SystemPostConfig()
    ]

    def __init__(self, environment={}):
        environment['node_type'] = 'mgmt'
        NedgeBaseConfigurator.__init__(self, environment,
                                       NedgeMgmtConfigurator._steps)
<|endoftext|>