<commit_before>from discord.ext import commands
import datetime
import asyncio
import json
import aiohttp

description = '''Animeforums bot Command List. '''
bot = commands.Bot(command_prefix='!', description=description)

@bot.event
async def on_ready():
    print('Logged in as')
    print(bot.user.name)
    print(bot.user.id)
    print('------')
    bot.uptime = datetime.datetime.utcnow()
    global Data_channel
    Data_channel = bot.get_channel(Config["Data Channel"])
    await timer()

async def timer():
    await latest()
    await asyncio.sleep(Config["Second"])
    await timer()

def get_bot_uptime():
    now = datetime.datetime.utcnow()
    delta = now - bot.uptime
    hours, remainder = divmod(int(delta.total_seconds()), 3600)
    minutes, seconds = divmod(remainder, 60)
    days, hours = divmod(hours, 24)
    if days:
        fmt = '**{d}** days, **{h}** hours, **{m}** minutes, and **{s}** seconds'
    else:
        fmt = '**{h}** hours, **{m}** minutes, and **{s}** seconds'

    return fmt.format(d=days, h=hours, m=minutes, s=seconds)

async def Readlinkjson(name): #read a link and then covert them into json
    with aiohttp.ClientSession() as session:
        async with session.get(website+name+".json") as resp:
            return(await resp.json())
    # print (website+name+".json")
    # r= requests.get(website+name+".json")
    # return json.loads(r.text)

async def Readfiles(files):
    with open(files,'r') as f:
        data = json.load(f)
    print(files, " Read")
    return (data)

async def InputFiles(data,files):  # Input data info into files Inside a folder
    with open(files, 'w') as f:
        json.dump(data, f, indent=2)
    print(files, " Updated")

async def latest():
    """
    Check if there is any new thread.
    If there is, then add them into a list
    using format of this way
    <Titles>  Author:<Creator>
    <link>

    Note: it will also ignore pinned one if pinned thread is only one post.
    It check if there is no replies, then it is as "new thread"
    """
    Data= []
    old_data= await Readfiles("Latest.json")
    json_data= await Readlinkjson("/latest")#calling function that will read link json
    for key in json_data["topic_list"]["topics"]:#for each of "latest" titles
        if key["title"] in old_data and key["id"] == old_data[key["title"]]: #if it already exist, then skip
            continue
        if key["posts_count"] == 1: #checking if post itself is only creator, which will show as "new thread"
            json_post=await Readlinkjson("/t/{}".format(key['id']))
            for post_data in json_post["post_stream"]["posts"]:
                if post_data["post_number"]==1: #Out of all post, it will get creator username
                    Data.append("{} \tAuthor:{} \n{}".format(key["title"],post_data["username"],website+"/t/"+key["slug"]+"/"+str(key['id'])))
                    old_data.update({key["title"]:key["id"]}) #It will write to files later on so it can check if it already exist and still no replies, poor original poster...
                    print(key["title"],post_data["username"],key["id"])
                    print("#############################################################")
                    break
    await InputFiles(old_data,"Latest.json") #Reason for Json/Dict, beacuse in case Titles are same name but creator may be different
    print("".join(Data))
    if len("".join(Data)) == 0:
        print ("return")
        return
    elif len("".join(Data)) >=5:
        print("send")
        await bot.send_message(Data_channel,"".join(Data))

@bot.command()
async def uptime(): #Showing Time that bot been total run
    """Tells you how long the bot has been up for."""
    await bot.say('I been up for {}'.format(get_bot_uptime()))

@bot.command(name = "time",brief="Allow to change timer for bot checking thread and post.",pass_context= True)
@commands.has_any_role("Mod","Owner")
async def Timer(msg):
    """
    Change time for bot to regular update.
    Note: Use a Second.
    Also it will take it effect after last timer up.
    For example, timer was 10 min
    now you enter !time 300
    which is 5 min, but you have to wait 10 min or w/e remain min since last post. Once it done, it will replace with 5 min.
    """
    second = (msg.message.content[len('!time'):].strip())
    if second == "":
        await bot.say("You didn't put how many second in!")
    else:
        Config.update({"Second":int(second)})
        await InputFiles(Config,"Config.json")
        if int(second) <=60:
            await bot.say("It is now updated. You have enter {} second".format(second))
        else:
            min = int(second)/60
            await bot.say("It is now updated. You have enter {} second, which is {} min".format(second, format(min,'.2f')))

@bot.command(name="summary",brief="Showing a summary of user",pass_context= True)
async def Summary_stat(msg): #Showing a summary stats of User
    '''
    Give a stat of summary of that username
    Topics Created:
    Posts Created:
    Likes Given:
    Likes Received:
    Days Visited:
    Posts Read:
    '''
    name = (msg.message.content[len('!summary'):].strip())
    if name == "":
        await bot.say("You didn't put name in!")
        return
    data = Readlinkjson("/users/{}/summary".format(name))
    if "errors" in data:
        await bot.say("{} is not found! Please double check case and spelling!".format(name))
        return
    summary=data["user_summary"]
    print_data= "Topics Created:{}\nPost Created:{}\nLikes Given:{}\nLikes Received:{}\nDays Visited:{}\nPosts Read:{}".format(summary["topic_count"],
                                                                                summary["post_count"],
                                                                                summary["likes_given"],
                                                                                summary["likes_received"],
                                                                                summary["days_visited"],
                                                                                summary["posts_read_count"])
    await bot.say("```py\n{}\n```".format(print_data))

@bot.command(name="stats",breif="Show a Site Statistics",pass_context=True)
async def Statictics(): #To show a stats of website of what have been total post, last 7 days, etc etc
    '''
    Show a table of Topics,Posts, New Users, Active Users, Likes for All Time, Last 7 Days and Lasts 30 Days
    '''
    data=await Readlinkjson("/about")
    stat=data["about"]["stats"]
    await bot.say("""
    ```py
|--------------|----------|--------------|--------------|
|              | All Time | Lasts 7 Days | Last 30 Days |
|--------------|----------|--------------|--------------|
| Topics       |    {0:<6}|      {1:<8}|     {2:<9}|
|--------------|----------|--------------|--------------|
| Posts        |    {3:<6}|      {4:<8}|     {5:<9}|
|--------------|----------|--------------|--------------|
| New Users    |    {6:<6}|      {7:<8}|     {8:<9}|
|--------------|----------|--------------|--------------|
| Active Users |    —     |      {9:<8}|     {10:<9}|
|--------------|----------|--------------|--------------|
| Likes        |    {11:<6}|      {12:<8}|     {13:<9}|
|--------------|----------|--------------|--------------|
```

    """.format(stat["topic_count"],stat["topics_7_days"],stat["topics_30_days"],
               stat["post_count"],stat["posts_7_days"],stat["posts_30_days"],
               stat["user_count"],stat["users_7_days"],stat["users_30_days"],
               stat["active_users_7_days"],stat["active_users_30_days"],
               stat["like_count"],stat["likes_7_days"],stat["likes_30_days"]))

loop = asyncio.get_event_loop()
Config=loop.run_until_complete(Readfiles("Config.json"))
website= Config["link"]
bot.run(Config['username'], Config['password'])<commit_msg>Fixed Summary<commit_after>from discord.ext import commands
import datetime
import asyncio
import json
import aiohttp

description = '''Animeforums bot Command List. '''
bot = commands.Bot(command_prefix='!', description=description)

@bot.event
async def on_ready():
    print('Logged in as')
    print(bot.user.name)
    print(bot.user.id)
    print('------')
    bot.uptime = datetime.datetime.utcnow()
    global Data_channel
    Data_channel = bot.get_channel(Config["Data Channel"])
    await timer()

async def timer():
    await latest()
    await asyncio.sleep(Config["Second"])
    await timer()

def get_bot_uptime():
    now = datetime.datetime.utcnow()
    delta = now - bot.uptime
    hours, remainder = divmod(int(delta.total_seconds()), 3600)
    minutes, seconds = divmod(remainder, 60)
    days, hours = divmod(hours, 24)
    if days:
        fmt = '**{d}** days, **{h}** hours, **{m}** minutes, and **{s}** seconds'
    else:
        fmt = '**{h}** hours, **{m}** minutes, and **{s}** seconds'

    return fmt.format(d=days, h=hours, m=minutes, s=seconds)

async def Readlinkjson(name): #read a link and then covert them into json
    with aiohttp.ClientSession() as session:
        async with session.get(website+name+".json") as resp:
            return(await resp.json())
    # print (website+name+".json")
    # r= requests.get(website+name+".json")
    # return json.loads(r.text)

async def Readfiles(files):
    with open(files,'r') as f:
        data = json.load(f)
    print(files, " Read")
    return (data)

async def InputFiles(data,files):  # Input data info into files Inside a folder
    with open(files, 'w') as f:
        json.dump(data, f, indent=2)
    print(files, " Updated")

async def latest():
    """
    Check if there is any new thread.
    If there is, then add them into a list
    using format of this way
    <Titles>  Author:<Creator>
    <link>

    Note: it will also ignore pinned one if pinned thread is only one post.
    It check if there is no replies, then it is as "new thread"
    """
    Data= []
    old_data= await Readfiles("Latest.json")
    json_data= await Readlinkjson("/latest")#calling function that will read link json
    for key in json_data["topic_list"]["topics"]:#for each of "latest" titles
        if key["title"] in old_data and key["id"] == old_data[key["title"]]: #if it already exist, then skip
            continue
        if key["posts_count"] == 1: #checking if post itself is only creator, which will show as "new thread"
            json_post=await Readlinkjson("/t/{}".format(key['id']))
            for post_data in json_post["post_stream"]["posts"]:
                if post_data["post_number"]==1: #Out of all post, it will get creator username
                    Data.append("{} \tAuthor:{} \n{}".format(key["title"],post_data["username"],website+"/t/"+key["slug"]+"/"+str(key['id'])))
                    old_data.update({key["title"]:key["id"]}) #It will write to files later on so it can check if it already exist and still no replies, poor original poster...
                    print(key["title"],post_data["username"],key["id"])
                    print("#############################################################")
                    break
    await InputFiles(old_data,"Latest.json") #Reason for Json/Dict, beacuse in case Titles are same name but creator may be different
    print("".join(Data))
    if len("".join(Data)) == 0:
        print ("return")
        return
    elif len("".join(Data)) >=5:
        print("send")
        await bot.send_message(Data_channel,"".join(Data))

@bot.command()
async def uptime(): #Showing Time that bot been total run
    """Tells you how long the bot has been up for."""
    await bot.say('I been up for {}'.format(get_bot_uptime()))

@bot.command(name = "time",brief="Allow to change timer for bot checking thread and post.",pass_context= True)
@commands.has_any_role("Mod","Owner")
async def Timer(msg):
    """
    Change time for bot to regular update.
    Note: Use a Second.
    Also it will take it effect after last timer up.
    For example, timer was 10 min
    now you enter !time 300
    which is 5 min, but you have to wait 10 min or w/e remain min since last post. Once it done, it will replace with 5 min.
    """
    second = (msg.message.content[len('!time'):].strip())
    if second == "":
        await bot.say("You didn't put how many second in!")
    else:
        Config.update({"Second":int(second)})
        await InputFiles(Config,"Config.json")
        if int(second) <=60:
            await bot.say("It is now updated. You have enter {} second".format(second))
        else:
            min = int(second)/60
            await bot.say("It is now updated. You have enter {} second, which is {} min".format(second, format(min,'.2f')))

@bot.command(name="summary",brief="Showing a summary of user",pass_context= True)
async def Summary_stat(msg): #Showing a summary stats of User
    '''
    Give a stat of summary of that username
    Topics Created:
    Posts Created:
    Likes Given:
    Likes Received:
    Days Visited:
    Posts Read:
    '''
    name = (msg.message.content[len('!summary'):].strip())
    if name == "":
        await bot.say("You didn't put name in!")
        return
    data = await Readlinkjson("/users/{}/summary".format(name))
    if "errors" in data:
        await bot.say("{} is not found! Please double check case and spelling!".format(name))
        return
    summary=data["user_summary"]
    print_data= "Topics Created:{}\nPost Created:{}\nLikes Given:{}\nLikes Received:{}\nDays Visited:{}\nPosts Read:{}".format(summary["topic_count"],
                                                                                summary["post_count"],
                                                                                summary["likes_given"],
                                                                                summary["likes_received"],
                                                                                summary["days_visited"],
                                                                                summary["posts_read_count"])
    await bot.say("```py\n{}\n```".format(print_data))

@bot.command(name="stats",breif="Show a Site Statistics",pass_context=True)
async def Statictics(): #To show a stats of website of what have been total post, last 7 days, etc etc
    '''
    Show a table of Topics,Posts, New Users, Active Users, Likes for All Time, Last 7 Days and Lasts 30 Days
    '''
    data=await Readlinkjson("/about")
    stat=data["about"]["stats"]
    await bot.say("""
    ```py
|--------------|----------|--------------|--------------|
|              | All Time | Lasts 7 Days | Last 30 Days |
|--------------|----------|--------------|--------------|
| Topics       |    {0:<6}|      {1:<8}|     {2:<9}|
|--------------|----------|--------------|--------------|
| Posts        |    {3:<6}|      {4:<8}|     {5:<9}|
|--------------|----------|--------------|--------------|
| New Users    |    {6:<6}|      {7:<8}|     {8:<9}|
|--------------|----------|--------------|--------------|
| Active Users |    —     |      {9:<8}|     {10:<9}|
|--------------|----------|--------------|--------------|
| Likes        |    {11:<6}|      {12:<8}|     {13:<9}|
|--------------|----------|--------------|--------------|
```

    """.format(stat["topic_count"],stat["topics_7_days"],stat["topics_30_days"],
               stat["post_count"],stat["posts_7_days"],stat["posts_30_days"],
               stat["user_count"],stat["users_7_days"],stat["users_30_days"],
               stat["active_users_7_days"],stat["active_users_30_days"],
               stat["like_count"],stat["likes_7_days"],stat["likes_30_days"]))

loop = asyncio.get_event_loop()
Config=loop.run_until_complete(Readfiles("Config.json"))
website= Config["link"]
bot.run(Config['username'], Config['password'])<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
from tornado import ioloop, gen
from tornado import web
from motor.motor_tornado import MotorClient
from entries.get_entries import entries_update
from entries.get_entries import entries_consumer, feeds_consumer

client = MotorClient(os.environ['MONGO_ENTRIES'])
db = client['entries-by-votes']

class MainHandler(web.RequestHandler):

    @gen.coroutine
    def get(self):
        db = self.settings['db']

        cursor = db.entries.find({"rank": {"$gt": 0}})\
            .sort('rank', -1).limit(100)
        entries = yield cursor.to_list(length=100)

        self.render('templates/index.html', entries=entries)

def make_app():
    return web.Application([
        (r'/', MainHandler),
    ], db=db)

if __name__ == "__main__":
    port = os.environ.get('TORNADO_PORT', 8888)
    print('Starting app in port {}'.format(port))
    app = make_app()
    app.listen(port)
    io_loop = ioloop.IOLoop.current()
    io_loop.spawn_callback(feeds_consumer)
    io_loop.spawn_callback(entries_consumer)
    io_loop.spawn_callback(entries_update)
    io_loop.start()
<commit_msg>rank as mongo aggregate<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
from tornado import ioloop, gen
from tornado import web
from motor.motor_tornado import MotorClient
from entries.get_entries import entries_update
from entries.get_entries import entries_consumer, feeds_consumer

client = MotorClient(os.environ['MONGO_ENTRIES'])
db = client['entries-by-votes']

class MainHandler(web.RequestHandler):

    @gen.coroutine
    def get(self):
        db = self.settings['db']

        cursor = db.entries.aggregate([
            {'$project': {
                '_id':0, 'link':1, 'title':1, 'published':1, 'total_votes':1, 'votes':1, 'rank': {
                '$divide': ["$total_votes", {
                    '$pow': [{
                        '$add': [{'$divide': [{'$subtract' : [datetime.now(), "$published"]}, 1000*60*60]}, 2]
                    }, 1.8]
                }]
            }}},
            {'$sort': {'rank': -1}},
            {'$limit': 100},
        ]);

        entries = yield cursor.to_list(length=100)

        self.render('templates/index.html', entries=entries)

def make_app():
    return web.Application([
        (r'/', MainHandler),
    ], db=db)

if __name__ == "__main__":
    port = os.environ.get('TORNADO_PORT', 8888)
    print('Starting app in port {}'.format(port))
    app = make_app()
    app.listen(port)
    io_loop = ioloop.IOLoop.current()
    io_loop.spawn_callback(feeds_consumer)
    io_loop.spawn_callback(entries_consumer)
    io_loop.spawn_callback(entries_update)
    io_loop.start()
<|endoftext|>


<commit_before>#our code here
from PIL import Image
from ntpath import basename
import os

class ImgFile(object):
	"""Image file class"""
	def __init__(self,path):
		self.path=path
		self.name=self.retrieve_name()

	def get_format(self):
		path=self.get_path()
		index=0
		for char in path:
			if char=='.':
				return path[index:]
			index+=1

	def valid_format(self):
		frmt=self.get_format()

	def get_name(self):
		return self.name

	def get_path(self):
		return self.path

	def retrieve_name(self):
		name=basename(self.path)
		index=0
		for char in name:
			if char=='.':
				return name[:index]
			index+=1

	def convert(self,trgt_format,trgt_path):
		try:
			im=Image.open(self.get_path())
			im.save("{}{}.{}".format(trgt_path, self.get_name(),trgt_format))
			print "done"
		except Exception, e:
			print e

def main():
	p="C://Users//Daniel//Desktop//shit//bodies.mp3"
	img=ImgFile(p)
	frmt="png"
	pth="C://Users//Daniel//Desktop//"
	img.convert(frmt, pth)

if __name__ == '__main__':
	main()<commit_msg>Minor get_format fix<commit_after>#our code here
from PIL import Image
from ntpath import basename
import os

class ImgFile(object):
	"""Image file class"""
	def __init__(self,path):
		self.path=path
		self.name=self.retrieve_name()

	def get_format(self):
		path=self.get_path()
		index=0
		for char in path:
			if char=='.':
				return path[index+1:]
			index+=1

	def valid_format(self):
		frmt=self.get_format()

	def get_name(self):
		return self.name

	def get_path(self):
		return self.path

	def retrieve_name(self):
		name=basename(self.path)
		index=0
		for char in name:
			if char=='.':
				return name[:index]
			index+=1

	def convert(self,trgt_format,trgt_path):
		try:
			im=Image.open(self.get_path())
			im.save("{}{}.{}".format(trgt_path, self.get_name(),trgt_format))
			print "done"
		except Exception, e:
			print e

def main():

	#testbed
	"""
	p="C://Users//Daniel//Desktop//shit//bodies.mp3"
	img=ImgFile(p)
	frmt="png"
	pth="C://Users//Daniel//Desktop//"
	img.convert(frmt, pth)
	"""

if __name__ == '__main__':
	main()<|endoftext|>


<commit_before>from __future__ import print_function
import io_utils
import scraper
import praw
import pandas as pd
from config import Config

def main():

    options = Config()
    reddit = praw.Reddit(user_agent = options.user_agent)

    post_generator = reddit.get_subreddit(options.network_hub).get_new(limit=options.post_limit)

    user_series = scraper.get_new_authors(post_generator)
    content_df = pd.DataFrame(columns=['downs', 'object_type', 'post_body',
    'post_id', 'post_timestamp', 'score', 'subreddit', 'thread_title',
    'thread_url', 'timestamp', 'ups', 'user_name'])
    log_df = io_utils.load_log()

    for user in user_series:
        if not (log_df.user_name.ix[log_df.object_type == 'comment'] == user.name).any():
            #Check if user already has a comment logged in the log dataframe
            content_df = scraper.get_user_comments(user, content_dataframe=content_df)

        if not (log_df.user_name.ix[log_df.object_type == 'submission'] == user.name).any():
            # Check if user already has a submission logged in the log dataframe
            content_df = scraper.get_user_submissions(user, content_dataframe=content_df)

    # Logging syntax needs to be steamlined...
    new_users_to_log = content_df[['user_name', 'object_type', 'post_id', 'post_timestamp']].ix[content_df.groupby(['user_name', 'object_type']).post_timestamp.idxmax()]
    log_df = log_df.append(new_users_to_log)
    log_df = log_df.reset_index(drop=True)
    io_utils.save_log(log_df)

    io_utils.dumps_like_a_truck(content_df)

    print("Logged %d new users!" % len(new_users_to_log.user_name.unique()))
    print("Logged %d new posts!" % len(content_df)

if __name__ == "__main__":
    main()
<commit_msg>Fix : missing parenthesis, main() works now<commit_after>from __future__ import print_function
import io_utils
import scraper
import praw
import pandas as pd
from config import Config

def main():

    options = Config()
    reddit = praw.Reddit(user_agent = options.user_agent)

    post_generator = reddit.get_subreddit(options.network_hub).get_new(limit=options.post_limit)

    user_series = scraper.get_new_authors(post_generator)
    content_df = pd.DataFrame(columns=['downs', 'object_type', 'post_body',
    'post_id', 'post_timestamp', 'score', 'subreddit', 'thread_title',
    'thread_url', 'timestamp', 'ups', 'user_name'])
    log_df = io_utils.load_log()

    for user in user_series:
        if not (log_df.user_name.ix[log_df.object_type == 'comment'] == user.name).any():
            #Check if user already has a comment logged in the log dataframe
            content_df = scraper.get_user_comments(user, content_dataframe=content_df)

        if not (log_df.user_name.ix[log_df.object_type == 'submission'] == user.name).any():
            # Check if user already has a submission logged in the log dataframe
            content_df = scraper.get_user_submissions(user, content_dataframe=content_df)

    # Logging syntax needs to be steamlined...
    new_users_to_log = content_df[['user_name', 'object_type', 'post_id', 'post_timestamp']].ix[content_df.groupby(['user_name', 'object_type']).post_timestamp.idxmax()]
    log_df = log_df.append(new_users_to_log)
    log_df = log_df.reset_index(drop=True)
    io_utils.save_log(log_df)

    io_utils.dumps_like_a_truck(content_df)

    print("Logged %d new users!" % len(new_users_to_log.user_name.unique()))
    print("Logged %d new posts!" % len(content_df))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#! /usr/bin/python

import json
import logging
import os
import time

import urlparse as up

import praw

from database import Database
from utils import delete_tmp_files
from dropbox import DropBox

def build_comment(imgur_url):
    head = '''Hi! I noticed that you posted an image from dropbox.com. I have
            rehosted your image to imgur because high traffic can break
            dropbox.com links.\n\n'''

    body = '[Here is the rehosted image](' + imgur_url + ')\n\n'

    tail = '''This action was performed by a bot. If there is an issue or
            problem, please report it below.\n\n'''

    foot = '''[^[Bot FAQ]](http://www.reddit.com/r/DropBox_Bot/wiki/index)
        [^[Report Problem]](http://www.reddit.com/message/compose/?to=DropBox_Bot&subject=Problem%20to%20Report)
        [^[Feedback]](http://www.reddit.com/r/DropBox_Bot/submit)
        [^[Source]](https://github.com/tzoch/dropbox-bot)'''

    return head + body + tail + foot

def scrape_domain_submissions(domain, config, r):
    '''
    This is the main work horse function of the dropbox bot. The bot gets the
    latest submissions to the dropbox.com domain and checks for two conditions.
        1. If it has already processed the submission, the bot skips it
        2. If it is able to be rehosted, we processes it
            - currently, the bot only supports the rehosting of images, but
              if there is a demand for other formats, and if there are
              convinient hosts to use, they will be implemented

    Currently, there is some basic logging to report the status of the bot.
    However, proper error handling should be added to make sure bad requests
    or 404'd dropbox pages are skipped and do not crash the bot.
    '''

    db = Database(config['database'])

    if config['test-mode']:
        submissions = r.get_subreddit('DropBox_Bot').get_new(limit=10)
    else:
        submissions = r.get_domain_listing(domain, sort='new', limit=100)

    for submission in submissions:
        name = submission.name # the thing_id - easier to reassign here
        drop = DropBox(submission.url, name)

        # skip deleted comments
        if not submission.author:
            logging.info('Skipped! [' + name + '] Submission has been deleted')
            continue

        # skip blacklisted subreddits
        if submission.subreddit.display_name in config['blacklist']:
            logging.info('Skipped! [' + name + '] in a blacklisted subreddit')
            continue

        if db.is_processed(name):
            logging.info('Skipped! [' + name + '] has already been processed')
            continue

        if drop.is_rehostable:
            drop.download_file()
            imgur_url = drop.rehost_image()

            # This will return false from the dropbox.py file
            # if there is an HTTP Exception uploading the file
            if imgur_url:
                comment = build_comment(imgur_url)
                # These try/excepts are to help get around the
                # commenting limit reddit has for new accounts
                try:
                    submission.add_comment(comment)
                    db.mark_as_processed(name)
                except praw.errors.RateLimitExceeded:
                    logging.error('ERROR! [' + name + '] RateLimtitExceeded')
                    logging.info('Trying to sleep off the RateLimit')
                    time.sleep(1200)
                    logging.info('AWAKE! Trying to comment again...')
                    try:
                        submission.add_coment(comment)
                        db.mark_as_processed(name)
                    except praw.errors.RateLimitExceeded:
                        logging.error('Skipped! [' + name + '] Rate Limited')
                        continue
                logging.info('Success! [' + name + '] rehosted')
            # This probably hapens due to an imgur API error
            # ex. the file is too large to be uploaded, so we
            # should skip and mark as processed
            else:
                logging.error('Failure! [' + name + '] error while uploading')
                db.mark_as_processed(name)
        else:
            db.mark_as_processed(name)
            logging.info('Skipped! [' + name + '] is not rehostable')

def main():
    fmt = '[%(asctime)-15s] (%(module)-15s) %(levelname)-8s : %(message)s'
    logging.basicConfig(filename='dropbox-bot.log',
                        format=fmt,
                        datefmt='%d-%b %H:%M:%S',
                        level=logging.INFO)

    logging.info('Bot Started')

    print '''DropBox Bot started...\n\tTo monitor the bots status check the log
             "dropbox-bot.log"\n\tTo stop the bot, use KeyboardInterrupt'''

    config = json.load(open('config.json'))

    r = praw.Reddit(config['user-agent'])
    r.login(config['username'], config['password'])

    while True:
        print 'Top of loop'
        try:
            print 'Scraping submissions'
            scrape_domain_submissions('dropbox.com',
                                      config, r)
            scrape_domain_submissions('dl.dropboxusercontent.com',
                                      config, r)
        except KeyboardInterrupt:
            import sys
            sys.exit(0)

        delete_tmp_files()
        print 'Bot done scraping...sleeping for 20 minutes'
        logging.info('Sleeping! No new submissions to process')
        time.sleep(1200)

if __name__ == '__main__':
    print '[INITIALIZED]'
    main()
<commit_msg>Moved the portion of code that checks if a submission has been processed to the top of the loop in scrape_domain_submissions(). Every other check doesn't matter if it's already been processed. Also, submissions will be marked as processed if they are deleted when the bot sees them.<commit_after>#! /usr/bin/python

import json
import logging
import os
import time

import urlparse as up

import praw

from database import Database
from utils import delete_tmp_files
from dropbox import DropBox

def build_comment(imgur_url):
    head = '''Hi! I noticed that you posted an image from dropbox.com. I have
            rehosted your image to imgur because high traffic can break
            dropbox.com links.\n\n'''

    body = '[Here is the rehosted image](' + imgur_url + ')\n\n'

    tail = '''This action was performed by a bot. If there is an issue or
            problem, please report it below.\n\n'''

    foot = '''[^[Bot FAQ]](http://www.reddit.com/r/DropBox_Bot/wiki/index)
        [^[Report Problem]](http://www.reddit.com/message/compose/?to=DropBox_Bot&subject=Problem%20to%20Report)
        [^[Feedback]](http://www.reddit.com/r/DropBox_Bot/submit)
        [^[Source]](https://github.com/tzoch/dropbox-bot)'''

    return head + body + tail + foot

def scrape_domain_submissions(domain, config, r):
    '''
    This is the main work horse function of the dropbox bot. The bot gets the
    latest submissions to the dropbox.com domain and checks for two conditions.
        1. If it has already processed the submission, the bot skips it
        2. If it is able to be rehosted, we processes it
            - currently, the bot only supports the rehosting of images, but
              if there is a demand for other formats, and if there are
              convinient hosts to use, they will be implemented

    Currently, there is some basic logging to report the status of the bot.
    However, proper error handling should be added to make sure bad requests
    or 404'd dropbox pages are skipped and do not crash the bot.
    '''

    db = Database(config['database'])

    if config['test-mode']:
        submissions = r.get_subreddit('DropBox_Bot').get_new(limit=10)
    else:
        submissions = r.get_domain_listing(domain, sort='new', limit=100)

    for submission in submissions:
        name = submission.name # the thing_id - easier to reassign here
        drop = DropBox(submission.url, name)

        # first, make sure the submission hasn't been processed already
        if db.is_processed(name):
            logging.info('Skipped! [' + name + '] has already been processed')
            continue

        # skip deleted comments
        if not submission.author:
            logging.info('Skipped! [' + name + '] Submission has been deleted')
            db.mark_as_processed(name)
            continue

        # skip blacklisted subreddits
        if submission.subreddit.display_name in config['blacklist']:
            logging.info('Skipped! [' + name + '] in a blacklisted subreddit')
            db.mark_as_processed(name)
            continue

        if drop.is_rehostable:
            drop.download_file()
            imgur_url = drop.rehost_image()

            # This will return false from the dropbox.py file
            # if there is an HTTP Exception uploading the file
            if imgur_url:
                comment = build_comment(imgur_url)
                # These try/excepts are to help get around the
                # commenting limit reddit has for new accounts
                try:
                    submission.add_comment(comment)
                    db.mark_as_processed(name)
                except praw.errors.RateLimitExceeded:
                    logging.error('ERROR! [' + name + '] RateLimtitExceeded')
                    logging.info('Trying to sleep off the RateLimit')
                    time.sleep(1200)
                    logging.info('AWAKE! Trying to comment again...')
                    try:
                        submission.add_coment(comment)
                        db.mark_as_processed(name)
                    except praw.errors.RateLimitExceeded:
                        logging.error('Skipped! [' + name + '] Rate Limited')
                        continue
                logging.info('Success! [' + name + '] rehosted')
            # This probably hapens due to an imgur API error
            # ex. the file is too large to be uploaded, so we
            # should skip and mark as processed
            else:
                logging.error('Failure! [' + name + '] error while uploading')
                db.mark_as_processed(name)
        else:
            db.mark_as_processed(name)
            logging.info('Skipped! [' + name + '] is not rehostable')

def main():
    fmt = '[%(asctime)-15s] (%(module)-15s) %(levelname)-8s : %(message)s'
    logging.basicConfig(filename='dropbox-bot.log',
                        format=fmt,
                        datefmt='%d-%b %H:%M:%S',
                        level=logging.INFO)

    logging.info('Bot Started')

    print '''DropBox Bot started...\n\tTo monitor the bots status check the log
             "dropbox-bot.log"\n\tTo stop the bot, use KeyboardInterrupt'''

    config = json.load(open('config.json'))

    r = praw.Reddit(config['user-agent'])
    r.login(config['username'], config['password'])

    while True:
        print 'Top of loop'
        try:
            print 'Scraping submissions'
            scrape_domain_submissions('dropbox.com',
                                      config, r)
            scrape_domain_submissions('dl.dropboxusercontent.com',
                                      config, r)
        except KeyboardInterrupt:
            import sys
            sys.exit(0)

        delete_tmp_files()
        print 'Bot done scraping...sleeping for 20 minutes'
        logging.info('Sleeping! No new submissions to process')
        time.sleep(1200)

if __name__ == '__main__':
    print '[INITIALIZED]'
    main()
<|endoftext|>


<commit_before>from flask import Flask, request
from flask_cors import CORS
import random
import func
import json
from db import Db

app = Flask(__name__)
app.debug = True
CORS(app)

# VARIABLES GLOBALES
db = Db()

@app.route('/reset',methods=['GET'])
def resetSimulation():
	""" Permet de reinitialiser la partie en cours
		...
	"""
	gameId = func.recupGameId()

	if gameId == "NoGame":
		func.creerGame()
	else:
		func.supprimerGame(gameId)
		func.creerGame()

	return func.makeJsonResponse(OK)

@app.route('/players',methods=['GET'])
def getPlayers():
	""" Retourne la liste des participants dans la partie en cours
		...
	"""
	# On regarde ceux present dans la partie (table Participate)
	playersInfo = db.select("SELECT pl.pl_pseudo FROM Player pl INNER JOIN Participate par ON par.pl_id = pl.pl_id WHERE par.present = true")

	return func.makeJsonResponse({ "players" : playersInfo },200)

@app.route('/players', methods=['POST'])
def rejoin():
	""" Permet de joindre la partie en cours
		...
	"""

	gameId = func.recupGameId()
	if gameId == "NoGame":
		func.creerGame()
		gameId = func.recupGameId()

	data = request.get_json()

	# Je verifie que le pseudo de ce joueur n'existe deja pas
	info = db.select("SELECT pl_pseudo FROM Player WHERE pl_pseudo = '"+ data['name'] +"'")

	if len(info) > 0 :
		return func.makeJsonResponse(data,400)

	longI = db.select("SELECT ga_longitude FROM Game WHERE ga_id = '"+ str(gameId) +"'")[0]["ga_longitude"]
	latI = db.select("SELECT ga_latitude FROM Game WHERE ga_id = '"+ str(gameId) +"'")[0]["ga_latitude"]

	# Coordonnees du stand generees aleatoirement
	data['longitude'] = random.uniform(0, longI)
	data['latitude'] = random.uniform(0, latI)

	# J'incris le joueur avec son nom et son budget initial (100, fixe)
	db.execute("""INSERT INTO Player(pl_pseudo, pl_budget_ini) VALUES (@(name), 100);""", data)

	# Je lui affilie un stand avec les coordonnees generees
	db.execute("""
					INSERT INTO stand(loc_longitude, loc_latitude, loc_rayon, pl_id)
		       		SELECT @(longitude), @(latitude),20, player.pl_id FROM Player player
					WHERE pl_pseudo = @(name);
			  """, data)
	# J'update la table Participate afin de dire que ce nouveau candidat participe a la partie courante
	db.execute("""
					INSERT INTO Participate(present, ga_id, pl_id)
					SELECT 'true', '"""+ str(gameId) +"""', player.pl_id FROM Player player
					WHERE pl_pseudo = @(name);
			  """, data)

	# J'appelle la fonction playerInfo qui cree une donne Json type PlayerInfo
	playerInfo =  func.makePlayerInfo(data['name'])

	return func.makeJsonResponse({ "name" : data['name'], "location" : { "latitude" : data['latitude'], "longitude" : data['longitude']}, "info" : playerInfo })

@app.route('/metrology',methods=['GET'])
def getMetrology():
	""" Retourne le timestamp, la meteo d'aujourd'hui et la meteo de demain
		...
	"""
	# On prend la meteo d'aujourd'hui
	weather = db.select("SELECT * FROM InfoDay ORDER BY da_id DESC LIMIT 1")

	# Mettre test de nullite des donnees

	if len(weather):
		wToday = weather[0]["da_weather"]
		wTomorrow = weather[0]["da_weather_tomorrow"]
		tStam = weather[0]["da_timestamp"]
		dDay = weather[0]["da_day"]

		outData = {
		"timestamp" : tStam+24*dDay,
		"weather" : [ {
		        "weather" : wToday,
		        "dfn" : 0,
		    },
		    {
		        "weather" : wTomorrow,
		        "dfn" : 1,
		    }]
		}
	else:
		return func.makeJsonResponse({},2000)

	return func.makeJsonResponse(outData)

@app.route('/metrology',methods=['POST'])
def setMetrology():
	""" Permet de poster le timestamp, la meteo d'aujourd'hui et la meteo de demain
		...
	"""

	data = request.get_json()
	dataSql = {}
	weatherToday = None
	weatherTomorrow =  None

	# Dernier jour du jeu
	result = db.select("SELECT da_id, da_day FROM InfoDay ORDER BY da_id DESC LIMIT 1")
	if len(result) == 0:
		lastDay = 1
	else:
		lastDay = result[0]['da_day']

	# Le Timestamp
	timestamp = int(data['timestamp'])

	# La meteo d'aujourd'hui et de demain
	for weather in data['weather']:
		if weather['dfn'] == '0':
			weatherToday = weather['weather']
		else:
			weatherTomorrow = weather['weather']

	dataSql['weatherToday'] = weatherToday
	dataSql['weatherTomorrow'] = weatherTomorrow
	dataSql['timestamp'] = timestamp%24
	dataSql['lastDay'] = lastDay
	dataSql['da_id'] = result[0]['da_id']

	dataSql['day'] = int(timestamp/24)
	if dataSql['day'] != result[0]['da_day']:
		# Insertion dans la base
		db.execute("""INSERT INTO InfoDay(da_day, da_weather, da_weather_tomorrow, da_timestamp)
			VALUES (@(day),@(weatherToday),@(weatherTomorrow),@(timestamp));""", dataSql)
	else:
		db.execute("""UPDATE InfoDay da SET da_timestamp = @(timestamp) WHERE da.da_day = @(lastDay) AND da.da_id = @(da_id);""", dataSql)

	return func.makeJsonResponse(data,200)

@app.route('/players/<playerName>', methods=['DELETE'])
def leave(playerName):
	""" Permet de supprimer un joueur dans la partie en cours
		...
	"""

	plId = func.recupIdFromName(playerName)

	# Je le passe a 'false' dans la table Participate
	db.execute("""UPDATE Participate par SET present = 'false' WHERE par.pl_id = '"""+ str(plId) +"""';""")

	return "",200

@app.route('/sales', methods=['POST'])
def simulCmd():
	""" Inserer les ventes du joueur apres simulation JAVA
		...
	"""

	day = func.getDayIdCurr()
	data = request.get_data()
	datas = json.loads(data)

	for rows in datas["sales"]:

		if len(rows['item']) > 0:

			# Alors le joueur n'a rien mit en vente : la table Transaction n'a pas ete remplit
			playerId = func.recupIdFromName(rows['name'])
			recId = func.recupIdRecFromName(rows['item'])
			qte = rows['quantity']

			db.execute("""
					UPDATE Transaction SET qte_sale = '"""+ str(qte) +"""' WHERE da_id = '"""+ str(day) +"""'
					AND pl_id = """+ str(playerId) +"""'
					AND rec_id = """+ str(recId) +"""';
				   """)

	return func.makeJsonResponse(OK)

@app.route('/actions/<playerName>',methods=['POST'])
def simulActions(playerName):
	""" Permet d'effectuer les actions possibles des joueurs
		...
	"""

	listRecipe = []
	tmp = {}
	data = request.get_data()
	datas = simplejson.loads(data)

	print (datas)

	# Si le player ne demande pas d'actions
	if not data['actions']:
		return '"Not find actions"', 412

	#if not data['simulated']:
	#	return '"Not find simulated"', 412

	# On recupere l'ID du player
	playerInfo = func.recupIdFromName(playerName)

	# Si le player n'existe pas
	if playerInfo == None:
		return '"Player ID Not Found"', 412

	tmp['playerId'] = playerInfo

	# On recupere le jour en cours
	dayInfo = func.getDayIdCurr()

	if dayInfo == None:
		return '"Current day Not Found"', 412

	tmp['dayId'] = dayInfo

	totalCost = 0
	if playerName :

		for action in data['actions']:

			# Action pour ajouter les recettes dans la base de donnees
			if action['kind'] == "drinks":

				for prepare in action['prepare']:
					for k, v in prepare.iteritems():
						listRecipe.append({"recipe":k,"quantity":v,"price":0})
					print prepare

				for price in action['price']:
					for k, v in price.iteritems():
						for re in listRecipe:
							if re['recipe'] == k:
								re['price'] = v
			print listRecipe

			# insertion dans la base de donnees
			for drinksOffered in listRecipe:

				tmp['recipe'] = func.recupIdRecFromName(drinksOffered['recipe'])
				tmp['quantity'] = drinksOffered['quantity']
				tmp['price'] = drinksOffered['price']

				totalCost += drinksOffered['quantity'] * drinksOffered['price']

				db.execute("""INSERT INTO Transaction(pl_id, rec_id, da_id, price, qte_prev)
					VALUES ( @(playerId), @(recipe), @(dayId), @(price), @(quantity) );""", tmp)

	if totalCost <= func.calculeMoneyInfo(playerName,1)['cash']:
		sufficientFunds = "true"
	else:
		sufficientFunds = "false"

	return func.makeJsonResponse({ "sufficientFunds" : sufficientFunds, "totalCost" : totalCost})

@app.route('/map',methods=['GET'])
def getMap():
	""" Retourne l'ensemble des informations relatives aux joueurs dans la partie
		...
	"""

	tmp = db.select("SELECT ga_id FROM Game WHERE ga_run = 'true'")
	if len(tmp) > 0 :
		game_id = tmp[0]["ga_id"]

		itemByPlayer = {}
		playerInfo = {}
		drinkByPlayer = {}

		players_actifs_id = db.select("""SELECT par.pl_id FROM Participate par
										 INNER JOIN Game ga ON par.ga_id = ga.ga_id
										 WHERE par.ga_id = '"""+ str(game_id) +"""' AND par.present = 'true'""")

		for players in players_actifs_id:
			pseudo = db.select("SELECT pl_pseudo FROM Player WHERE pl_id = '"+ str(players["pl_id"]) +"'")[0]["pl_pseudo"]
			itemByPlayer[pseudo] = func.makeMapItem(players["pl_id"])
			playerInfo[pseudo] = func.makePlayerInfo(pseudo)
			drinkByPlayer[pseudo] = func.makeDrinkEveryTime(pseudo)

		ranking = func.rankingPlayer(game_id)
		region = func.makeRegion(game_id)

	else:
		return '"Game ID Not Found"', 412

	map = { "region" : region, "ranking" : ranking, "itemsByPlayer" : itemByPlayer , "playerInfo" : playerInfo, "drinksByPlayer" : drinkByPlayer }
	return func.makeJsonResponse({ "map" : map })

@app.route('/map/<playerName>',methods=['GET'])
def getPlayerMap(playerName):
	""" Retourne l'ensemble des informations relatives a un joueur
		...
	"""

	player_id = func.recupIdFromName(playerName)
	game_id = func.recupGameId()

	plInfo = func.makePlayerInfo(playerName)
	region = func.makeRegion(game_id)
	itemPlayer = func.makeMapItem(player_id)

	availableIngredients = func.getAvailableIngredients(playerName)

	map = { "availableIngredients" : availableIngredients, "region" : region, "itemPlayer" : itemPlayer }
	return "Obtenir les details d une partie (Client Web)"

@app.route('/ingredients',methods=['GET'])
def getIngredients():
	""" Retourne l'ensemble des ingredients disponibles
		...
	"""

	ingredient_list = db.select("SELECT ing_nom, ing_prix, ing_alcohol, ing_cold FROM Ingredient")

	return func.makeJsonResponse({ "ingredients": ingredient_list })

@app.route('/recipes',methods=['GET'])
def getRecipes():
	""" Retourne l'ensemble des recettes disponibles
		...
	"""

	recipes_List = db.select("SELECT * FROM Recipe")

	return func.makeJsonResponse({ "recipes": recipes_List })

@app.route('/recipe/<int:rc_id>',methods=['GET'])
def getRecipeById(rc_id):
	""" Retourne une recette
		...
	"""

	recipe = db.select("SELECT * FROM Recipe WHERE rec_id = '"+ str(rc_id) +"'")
	ingredient_list = db.select("""SELECT ing.* FROM Ingredient ing
								   INNER JOIN IngInRec inrec ON ing.ing_id = inrec.ing_id
								   INNER JOIN Recipe rec ON rec.rec_id = inrec.rec_id
								   WHERE rec.rec_id = '"""+ str(rc_id) +"""'""")

	if len(recipe) > 0:
		return func.makeJsonResponse({ "recipe": recipe, "ingredients": ingredient_list } )
	else:
		return '"Recipe Not Found"', 412

@app.route('/player/info/<playerName>', methods=['GET'])
def getInfoPlayer(playerName):

	playerInfo = func.makePlayerInfo(playerName)
	return func.makeJsonResponse(playerInfo)

if __name__ == "__main__":
    app.run()
<commit_msg>/actions<commit_after>from flask import Flask, request
from flask_cors import CORS
import random
import func
import json
from db import Db

app = Flask(__name__)
app.debug = True
CORS(app)

# VARIABLES GLOBALES
db = Db()

@app.route('/reset',methods=['GET'])
def resetSimulation():
	""" Permet de reinitialiser la partie en cours
		...
	"""
	gameId = func.recupGameId()

	if gameId == "NoGame":
		func.creerGame()
	else:
		func.supprimerGame(gameId)
		func.creerGame()

	return func.makeJsonResponse(OK)

@app.route('/players',methods=['GET'])
def getPlayers():
	""" Retourne la liste des participants dans la partie en cours
		...
	"""
	# On regarde ceux present dans la partie (table Participate)
	playersInfo = db.select("SELECT pl.pl_pseudo FROM Player pl INNER JOIN Participate par ON par.pl_id = pl.pl_id WHERE par.present = true")

	return func.makeJsonResponse({ "players" : playersInfo },200)

@app.route('/players', methods=['POST'])
def rejoin():
	""" Permet de joindre la partie en cours
		...
	"""

	gameId = func.recupGameId()
	if gameId == "NoGame":
		func.creerGame()
		gameId = func.recupGameId()

	data = request.get_json()

	# Je verifie que le pseudo de ce joueur n'existe deja pas
	info = db.select("SELECT pl_pseudo FROM Player WHERE pl_pseudo = '"+ data['name'] +"'")

	if len(info) > 0 :
		return func.makeJsonResponse(data,400)

	longI = db.select("SELECT ga_longitude FROM Game WHERE ga_id = '"+ str(gameId) +"'")[0]["ga_longitude"]
	latI = db.select("SELECT ga_latitude FROM Game WHERE ga_id = '"+ str(gameId) +"'")[0]["ga_latitude"]

	# Coordonnees du stand generees aleatoirement
	data['longitude'] = random.uniform(0, longI)
	data['latitude'] = random.uniform(0, latI)

	# J'incris le joueur avec son nom et son budget initial (100, fixe)
	db.execute("""INSERT INTO Player(pl_pseudo, pl_budget_ini) VALUES (@(name), 100);""", data)

	# Je lui affilie un stand avec les coordonnees generees
	db.execute("""
					INSERT INTO stand(loc_longitude, loc_latitude, loc_rayon, pl_id)
		       		SELECT @(longitude), @(latitude),20, player.pl_id FROM Player player
					WHERE pl_pseudo = @(name);
			  """, data)
	# J'update la table Participate afin de dire que ce nouveau candidat participe a la partie courante
	db.execute("""
					INSERT INTO Participate(present, ga_id, pl_id)
					SELECT 'true', '"""+ str(gameId) +"""', player.pl_id FROM Player player
					WHERE pl_pseudo = @(name);
			  """, data)

	# J'appelle la fonction playerInfo qui cree une donne Json type PlayerInfo
	playerInfo =  func.makePlayerInfo(data['name'])

	return func.makeJsonResponse({ "name" : data['name'], "location" : { "latitude" : data['latitude'], "longitude" : data['longitude']}, "info" : playerInfo })

@app.route('/metrology',methods=['GET'])
def getMetrology():
	""" Retourne le timestamp, la meteo d'aujourd'hui et la meteo de demain
		...
	"""
	# On prend la meteo d'aujourd'hui
	weather = db.select("SELECT * FROM InfoDay ORDER BY da_id DESC LIMIT 1")

	# Mettre test de nullite des donnees

	if len(weather):
		wToday = weather[0]["da_weather"]
		wTomorrow = weather[0]["da_weather_tomorrow"]
		tStam = weather[0]["da_timestamp"]
		dDay = weather[0]["da_day"]

		outData = {
		"timestamp" : tStam+24*dDay,
		"weather" : [ {
		        "weather" : wToday,
		        "dfn" : 0,
		    },
		    {
		        "weather" : wTomorrow,
		        "dfn" : 1,
		    }]
		}
	else:
		return func.makeJsonResponse({},2000)

	return func.makeJsonResponse(outData)

@app.route('/metrology',methods=['POST'])
def setMetrology():
	""" Permet de poster le timestamp, la meteo d'aujourd'hui et la meteo de demain
		...
	"""

	data = request.get_json()
	dataSql = {}
	weatherToday = None
	weatherTomorrow =  None

	# Dernier jour du jeu
	result = db.select("SELECT da_id, da_day FROM InfoDay ORDER BY da_id DESC LIMIT 1")
	if len(result) == 0:
		lastDay = 1
	else:
		lastDay = result[0]['da_day']

	# Le Timestamp
	timestamp = int(data['timestamp'])

	# La meteo d'aujourd'hui et de demain
	for weather in data['weather']:
		if weather['dfn'] == '0':
			weatherToday = weather['weather']
		else:
			weatherTomorrow = weather['weather']

	dataSql['weatherToday'] = weatherToday
	dataSql['weatherTomorrow'] = weatherTomorrow
	dataSql['timestamp'] = timestamp%24
	dataSql['lastDay'] = lastDay
	dataSql['da_id'] = result[0]['da_id']

	dataSql['day'] = int(timestamp/24)
	if dataSql['day'] != result[0]['da_day']:
		# Insertion dans la base
		db.execute("""INSERT INTO InfoDay(da_day, da_weather, da_weather_tomorrow, da_timestamp)
			VALUES (@(day),@(weatherToday),@(weatherTomorrow),@(timestamp));""", dataSql)
	else:
		db.execute("""UPDATE InfoDay da SET da_timestamp = @(timestamp) WHERE da.da_day = @(lastDay) AND da.da_id = @(da_id);""", dataSql)

	return func.makeJsonResponse(data,200)

@app.route('/players/<playerName>', methods=['DELETE'])
def leave(playerName):
	""" Permet de supprimer un joueur dans la partie en cours
		...
	"""

	plId = func.recupIdFromName(playerName)

	# Je le passe a 'false' dans la table Participate
	db.execute("""UPDATE Participate par SET present = 'false' WHERE par.pl_id = '"""+ str(plId) +"""';""")

	return "",200

@app.route('/sales', methods=['POST'])
def simulCmd():
	""" Inserer les ventes du joueur apres simulation JAVA
		...
	"""

	day = func.getDayIdCurr()
	data = request.get_data()
	datas = json.loads(data)

	for rows in datas["sales"]:

		if len(rows['item']) > 0:

			# Alors le joueur n'a rien mit en vente : la table Transaction n'a pas ete remplit
			playerId = func.recupIdFromName(rows['name'])
			recId = func.recupIdRecFromName(rows['item'])
			qte = rows['quantity']

			db.execute("""
					UPDATE Transaction SET qte_sale = '"""+ str(qte) +"""' WHERE da_id = '"""+ str(day) +"""'
					AND pl_id = """+ str(playerId) +"""'
					AND rec_id = """+ str(recId) +"""';
				   """)

	return func.makeJsonResponse(OK)

@app.route('/actions/<playerName>',methods=['POST'])
def simulActions(playerName):
	""" Permet d'effectuer les actions possibles des joueurs
		...
	"""

	listRecipe = []
	tmp = {}
	data = request.get_json()

	# Si le player ne demande pas d'actions
	if not data['actions']:
		return '"Not find actions"', 412

	#if not data['simulated']:
	#	return '"Not find simulated"', 412

	# On recupere l'ID du player
	playerInfo = func.recupIdFromName(playerName)

	# Si le player n'existe pas
	if playerInfo == None:
		return '"Player ID Not Found"', 412

	tmp['playerId'] = playerInfo

	# On recupere le jour en cours
	dayInfo = func.getDayIdCurr()

	if dayInfo == None:
		return '"Current day Not Found"', 412

	tmp['dayId'] = dayInfo

	totalCost = 0
	if playerName :

		for action in data['actions']:

			# Action pour ajouter les recettes dans la base de donnees
			if action['kind'] == "drinks":

				for prepare in action['prepare']:
					for k, v in prepare.iteritems():
						listRecipe.append({"recipe":k,"quantity":v,"price":0})
					print prepare

				for price in action['price']:
					for k, v in price.iteritems():
						for re in listRecipe:
							if re['recipe'] == k:
								re['price'] = v
			print listRecipe

			# insertion dans la base de donnees
			for drinksOffered in listRecipe:

				tmp['recipe'] = func.recupIdRecFromName(drinksOffered['recipe'])
				tmp['quantity'] = drinksOffered['quantity']
				tmp['price'] = drinksOffered['price']

				totalCost += drinksOffered['quantity'] * drinksOffered['price']

				db.execute("""INSERT INTO Transaction(pl_id, rec_id, da_id, price, qte_prev)
					VALUES ( @(playerId), @(recipe), @(dayId), @(price), @(quantity) );""", tmp)

	if totalCost <= func.calculeMoneyInfo(playerName,1)['cash']:
		sufficientFunds = "true"
	else:
		sufficientFunds = "false"

	return func.makeJsonResponse({ "sufficientFunds" : sufficientFunds, "totalCost" : totalCost})

@app.route('/map',methods=['GET'])
def getMap():
	""" Retourne l'ensemble des informations relatives aux joueurs dans la partie
		...
	"""

	tmp = db.select("SELECT ga_id FROM Game WHERE ga_run = 'true'")
	if len(tmp) > 0 :
		game_id = tmp[0]["ga_id"]

		itemByPlayer = {}
		playerInfo = {}
		drinkByPlayer = {}

		players_actifs_id = db.select("""SELECT par.pl_id FROM Participate par
										 INNER JOIN Game ga ON par.ga_id = ga.ga_id
										 WHERE par.ga_id = '"""+ str(game_id) +"""' AND par.present = 'true'""")

		for players in players_actifs_id:
			pseudo = db.select("SELECT pl_pseudo FROM Player WHERE pl_id = '"+ str(players["pl_id"]) +"'")[0]["pl_pseudo"]
			itemByPlayer[pseudo] = func.makeMapItem(players["pl_id"])
			playerInfo[pseudo] = func.makePlayerInfo(pseudo)
			drinkByPlayer[pseudo] = func.makeDrinkEveryTime(pseudo)

		ranking = func.rankingPlayer(game_id)
		region = func.makeRegion(game_id)

	else:
		return '"Game ID Not Found"', 412

	map = { "region" : region, "ranking" : ranking, "itemsByPlayer" : itemByPlayer , "playerInfo" : playerInfo, "drinksByPlayer" : drinkByPlayer }
	return func.makeJsonResponse({ "map" : map })

@app.route('/map/<playerName>',methods=['GET'])
def getPlayerMap(playerName):
	""" Retourne l'ensemble des informations relatives a un joueur
		...
	"""

	player_id = func.recupIdFromName(playerName)
	game_id = func.recupGameId()

	plInfo = func.makePlayerInfo(playerName)
	region = func.makeRegion(game_id)
	itemPlayer = func.makeMapItem(player_id)

	availableIngredients = func.getAvailableIngredients(playerName)

	map = { "availableIngredients" : availableIngredients, "region" : region, "itemPlayer" : itemPlayer }
	return "Obtenir les details d une partie (Client Web)"

@app.route('/ingredients',methods=['GET'])
def getIngredients():
	""" Retourne l'ensemble des ingredients disponibles
		...
	"""

	ingredient_list = db.select("SELECT ing_nom, ing_prix, ing_alcohol, ing_cold FROM Ingredient")

	return func.makeJsonResponse({ "ingredients": ingredient_list })

@app.route('/recipes',methods=['GET'])
def getRecipes():
	""" Retourne l'ensemble des recettes disponibles
		...
	"""

	recipes_List = db.select("SELECT * FROM Recipe")

	return func.makeJsonResponse({ "recipes": recipes_List })

@app.route('/recipe/<int:rc_id>',methods=['GET'])
def getRecipeById(rc_id):
	""" Retourne une recette
		...
	"""

	recipe = db.select("SELECT * FROM Recipe WHERE rec_id = '"+ str(rc_id) +"'")
	ingredient_list = db.select("""SELECT ing.* FROM Ingredient ing
								   INNER JOIN IngInRec inrec ON ing.ing_id = inrec.ing_id
								   INNER JOIN Recipe rec ON rec.rec_id = inrec.rec_id
								   WHERE rec.rec_id = '"""+ str(rc_id) +"""'""")

	if len(recipe) > 0:
		return func.makeJsonResponse({ "recipe": recipe, "ingredients": ingredient_list } )
	else:
		return '"Recipe Not Found"', 412

@app.route('/player/info/<playerName>', methods=['GET'])
def getInfoPlayer(playerName):

	playerInfo = func.makePlayerInfo(playerName)
	return func.makeJsonResponse(playerInfo)

if __name__ == "__main__":
    app.run()
<|endoftext|>


<commit_before>import os
import scipy.misc
import numpy as np

from model import DCGAN
from utils import pp, visualize, to_json

import tensorflow as tf

flags = tf.app.flags
flags.DEFINE_string("activation", "relu", "nonlinearity")
flags.DEFINE_integer("epoch", 100, "Epoch to train [25]")
flags.DEFINE_float("learning_rate_g", 0.0002, "Learning rate of for adam [0.0002]")
flags.DEFINE_float("learning_rate_d", 0.0002, "Learning rate of for adam [0.0002]")
flags.DEFINE_float("beta1", 0.5, "Momentum term of adam [0.5]")
flags.DEFINE_integer("train_size", np.inf, "The size of train images [np.inf]")
flags.DEFINE_integer("batch_size", 64, "The size of batch images [64]")
flags.DEFINE_integer("image_size", 108, "The size of image to use (will be center cropped) [108]")
flags.DEFINE_integer("output_height", 64, "The height of the output images to produce [64]")
flags.DEFINE_integer("output_width", 64, "The width of the output images (2nd dim) to produce [64]")
flags.DEFINE_integer("c_dim", 3, "Dimension of image color. [3]")
flags.DEFINE_integer("z_dim", 100, "Dimension of z. [100]")
flags.DEFINE_integer("gmm_cluster", None, "number of gmm cluster")
flags.DEFINE_integer("gmm_dim", 2, "dimention of gmm data")
flags.DEFINE_float("gmm_var", 0.02, "variance of generated gmm clusters")
flags.DEFINE_float("gmm_scale", 2, "scale of generated gmm clusters")
flags.DEFINE_string("dataset", "celebA", "The name of dataset [celebA, mnist, lsun]")
flags.DEFINE_string("network", "DCGAN", "D G architecture choice")
flags.DEFINE_string("checkpoint_dir", "checkpoint", "Directory name to save the checkpoints [checkpoint]")
flags.DEFINE_string("sample_dir", "samples", "Directory name to save the image samples [samples]")
flags.DEFINE_string("log_dir", "logs", "Directory name to save the image samples [samples]")
flags.DEFINE_string("cluster_est", "gmm", "method to estimate cluster #, gmm or jump")
flags.DEFINE_string("init_type", "orthogonal", "How to initialize linear layer weights")
flags.DEFINE_boolean("is_train", False, "True for training, False for testing [False]")
flags.DEFINE_boolean("is_crop", False, "True for training, False for testing [False]")
flags.DEFINE_boolean("visualize", False, "True for visualizing, False for nothing [False]")
flags.DEFINE_integer("visualize_interval", 5, "True for visualizing, False for nothing [False]")
flags.DEFINE_integer("g_heruistic", 0, "True for -log(D) g loss ")
flags.DEFINE_integer("g_update", 1, "Two generator update for 1 discriminator update")
FLAGS = flags.FLAGS

def main(_):
    pp.pprint(flags.FLAGS.__flags)

    if not os.path.exists(FLAGS.checkpoint_dir):
        os.makedirs(FLAGS.checkpoint_dir)
    if not os.path.exists(FLAGS.sample_dir):
        os.makedirs(FLAGS.sample_dir)

    with tf.Session() as sess:
        if FLAGS.dataset == 'mnist':
            dcgan = DCGAN(sess,
                          image_size=FLAGS.image_size,
                          batch_size=FLAGS.batch_size,
                          y_dim=10,
                          output_height=28,
                          output_width=28,
                          c_dim=1,
                          z_dim=100,
                          sample_size=FLAGS.batch_size,
                          dataset_name=FLAGS.dataset,
                          is_crop=FLAGS.is_crop,
                          checkpoint_dir=FLAGS.checkpoint_dir,
                          sample_dir=FLAGS.sample_dir,
                          flags=FLAGS)
        else:
            dcgan = DCGAN(sess,
                          image_size=FLAGS.image_size,
                          batch_size=FLAGS.batch_size,
                          output_width=FLAGS.output_width,
                          output_height=FLAGS.output_height,
                          c_dim=FLAGS.c_dim,
                          z_dim=FLAGS.z_dim,
                          sample_size=FLAGS.batch_size,
                          dataset_name=FLAGS.dataset,
                          is_crop=FLAGS.is_crop,
                          checkpoint_dir=FLAGS.checkpoint_dir,
                          sample_dir=FLAGS.sample_dir,
                          flags=FLAGS)

        if FLAGS.is_train:
            dcgan.train(FLAGS)
        else:
            dcgan.load(FLAGS.checkpoint_dir)

        # to_json("./web/js/layers.js", [dcgan.h0_w, dcgan.h0_b, dcgan.g_bn0],
        #                               [dcgan.h1_w, dcgan.h1_b, dcgan.g_bn1],
        #                               [dcgan.h2_w, dcgan.h2_b, dcgan.g_bn2],
        #                               [dcgan.h3_w, dcgan.h3_b, dcgan.g_bn3],
        #                               [dcgan.h4_w, dcgan.h4_b, None])

        # Below is codes for visualization
        OPTION = 1
        #visualize(sess, dcgan, FLAGS, OPTION)

if __name__ == '__main__':
    tf.app.run()
<commit_msg>change flags typo<commit_after>import os
import scipy.misc
import numpy as np

from model import DCGAN
from utils import pp, visualize, to_json

import tensorflow as tf

flags = tf.app.flags
flags.DEFINE_string("activation", "relu", "nonlinearity")
flags.DEFINE_integer("epoch", 100, "Epoch to train [100]")
flags.DEFINE_float("learning_rate_g", 0.0002, "Learning rate of for adam [0.0002]")
flags.DEFINE_float("learning_rate_d", 0.0002, "Learning rate of for adam [0.0002]")
flags.DEFINE_float("beta1", 0.5, "Momentum term of adam [0.5]")
flags.DEFINE_integer("train_size", np.inf, "The size of train images [np.inf]")
flags.DEFINE_integer("batch_size", 64, "The size of batch images [64]")
flags.DEFINE_integer("image_size", 108, "The size of image to use (will be center cropped) [108]")
flags.DEFINE_integer("output_height", 64, "The height of the output images to produce [64]")
flags.DEFINE_integer("output_width", 64, "The width of the output images (2nd dim) to produce [64]")
flags.DEFINE_integer("c_dim", 3, "Dimension of image color. [3]")
flags.DEFINE_integer("z_dim", 100, "Dimension of z. [100]")
flags.DEFINE_integer("gmm_cluster", None, "number of gmm cluster")
flags.DEFINE_integer("gmm_dim", 2, "dimention of gmm data")
flags.DEFINE_float("gmm_var", 0.02, "variance of generated gmm clusters")
flags.DEFINE_float("gmm_scale", 2, "scale of generated gmm clusters")
flags.DEFINE_string("dataset", "celebA", "The name of dataset [celebA, mnist, lsun]")
flags.DEFINE_string("network", "DCGAN", "D G architecture choice")
flags.DEFINE_string("checkpoint_dir", "checkpoint", "Directory name to save the checkpoints [checkpoint]")
flags.DEFINE_string("sample_dir", "samples", "Directory name to save the image samples [samples]")
flags.DEFINE_string("log_dir", "logs", "Directory name to save the image samples [samples]")
flags.DEFINE_string("cluster_est", "gmm", "method to estimate cluster #, gmm or jump")
flags.DEFINE_string("init_type", "orthogonal", "How to initialize linear layer weights")
flags.DEFINE_boolean("is_train", False, "True for training, False for testing [False]")
flags.DEFINE_boolean("is_crop", False, "True for training, False for testing [False]")
flags.DEFINE_boolean("visualize", False, "True for visualizing, False for nothing [False]")
flags.DEFINE_integer("visualize_interval", 5, "True for visualizing, False for nothing [False]")
flags.DEFINE_integer("g_heruistic", 0, "True for -log(D) g loss ")
flags.DEFINE_integer("g_update", 1, "Two generator update for 1 discriminator update")
FLAGS = flags.FLAGS

def main(_):
    pp.pprint(flags.FLAGS.__flags)

    if not os.path.exists(FLAGS.checkpoint_dir):
        os.makedirs(FLAGS.checkpoint_dir)
    if not os.path.exists(FLAGS.sample_dir):
        os.makedirs(FLAGS.sample_dir)

    with tf.Session() as sess:
        if FLAGS.dataset == 'mnist':
            dcgan = DCGAN(sess,
                          image_size=FLAGS.image_size,
                          batch_size=FLAGS.batch_size,
                          y_dim=10,
                          output_height=28,
                          output_width=28,
                          c_dim=1,
                          z_dim=100,
                          sample_size=FLAGS.batch_size,
                          dataset_name=FLAGS.dataset,
                          is_crop=FLAGS.is_crop,
                          checkpoint_dir=FLAGS.checkpoint_dir,
                          sample_dir=FLAGS.sample_dir,
                          flags=FLAGS)
        else:
            dcgan = DCGAN(sess,
                          image_size=FLAGS.image_size,
                          batch_size=FLAGS.batch_size,
                          output_width=FLAGS.output_width,
                          output_height=FLAGS.output_height,
                          c_dim=FLAGS.c_dim,
                          z_dim=FLAGS.z_dim,
                          sample_size=FLAGS.batch_size,
                          dataset_name=FLAGS.dataset,
                          is_crop=FLAGS.is_crop,
                          checkpoint_dir=FLAGS.checkpoint_dir,
                          sample_dir=FLAGS.sample_dir,
                          flags=FLAGS)

        if FLAGS.is_train:
            dcgan.train(FLAGS)
        else:
            dcgan.load(FLAGS.checkpoint_dir)

        # to_json("./web/js/layers.js", [dcgan.h0_w, dcgan.h0_b, dcgan.g_bn0],
        #                               [dcgan.h1_w, dcgan.h1_b, dcgan.g_bn1],
        #                               [dcgan.h2_w, dcgan.h2_b, dcgan.g_bn2],
        #                               [dcgan.h3_w, dcgan.h3_b, dcgan.g_bn3],
        #                               [dcgan.h4_w, dcgan.h4_b, None])

        # Below is codes for visualization
        OPTION = 1
        #visualize(sess, dcgan, FLAGS, OPTION)

if __name__ == '__main__':
    tf.app.run()
<|endoftext|>


<commit_before># encoding: utf-8

"""
DIM: Docker Image Manager
A set of convenience classes to create and manage Docker Images and Containers
"""

from __future__ import unicode_literals, print_function
from collections import OrderedDict
from contextlib import contextmanager
import os
ROOT = os.path.abspath(os.path.dirname(__file__))

import docker

from utils import wait, random_hex, chain_temp_files

docker_client = docker.Client(base_url='unix://var/run/docker.sock', version='auto')

class betterdict(dict):

    def __add__(self, other):
        self.update(other)
        return self

class DockerRunParameters(betterdict):
    """
    A class to define parameters for the docker create/run commands.
    Is also used to enrich Dockerfile EXPOSE and VOLUME instructions.
    """
    allowed = ()

    def __init__(self, **options):
        dict.__init__(self)
        self.log = options.pop('log', log)
        volumes = options.pop('volumes', ())
        ports = options.pop('ports', ())
        links = options.pop('links', ())
        self.update(options)
        self.kwargs = {}
        for vol in volumes:
            self.add_volume(vol)
        for port in ports:
            self.add_port(port)
        for link in links:
            self.add_link(link)
        self.finalize()

    def clone(self):
        # TODO provide true clone
        return betterdict(self)

    def add_volume(self, vol):
        binds = self.kwargs.setdefault('binds', {})
        volumes = self.setdefault('volumes', set())
        # host can be void (volume has no mapping)
        if ':' in vol:
            host, guest = vol.split(':', 1)
        else:
            host, guest = None, vol
        # default mode is 'rw', you can specify 'ro' instead
        if ':' in guest:
            guest, mode = guest.split(':')
        else:
            mode = 'rw'
        if host:
            host = os.path.abspath(os.path.expanduser(host))
            binds[host] = {'bind': guest, 'mode': mode}
        volumes.add(guest)

    def add_port(self, port):
        port_bindings = self.kwargs.setdefault('port_bindings', {})
        ports = self.setdefault('ports', set())
        if isinstance(port, basestring):
            if ':' in port:
                # TODO does not accept format host_ip:host_port:guest_port yet
                host, guest = port.rsplit(':', 1)
                guest = int(guest)
                port_bindings[guest] = int(host) if host else None
                ports.add(guest)
            elif '-' in port:
                start, end = port.split('-')
                for p in xrange(int(start), int(end) + 1):
                    port_bindings[p] = None
                    ports.add(p)
            else:
                port = int(port)
                port_bindings[port] = None
                ports.add(port)
        else:
            port_bindings[port] = None
            ports.add(port)

    def add_link(self, link):
        self.kwargs.setdefault('links', []).append((link, link))

    def finalize(self):
        if self.kwargs:
            self['host_config'] = docker.utils.create_host_config(**self.kwargs)
        if 'ports' in self:
            self['ports'] = list(self['ports'])
            self['ports'].sort()
        if 'volumes' in self:
            self['volumes'] = list(self['volumes'])

    def __add__(self, other):
        res = self.clone()
        res.update(other)
        return res

class DockerFile(object):
    """
    A class to manage Dockerfile like objects
    """
    def __init__(self, *files, **options):
        self.log = options.pop('log', log)
        # this is the global context used by every template involved in the build process
        # templates can be Dockerfile, configuration files, ...
        self.template_context = options.pop('template_context', None) or {}
        self.parameters = options.pop('parameters', None) or {}
        self.add_ons = options.pop('add_ons', None) or ()
        # a file can be a real file, specified by a path,
        # or a pseudo file, specified via a (name, content) pair
        self.files = OrderedDict()
        for file in files:
            if isinstance(file, tuple):
                name, content = file
            else:
                name = os.path.basename(file)
                with open(os.path.expanduser(file), 'rb') as f:
                    content = f.read()
            self.files[name] = content
        self.files.update(options)
        if not 'Dockerfile' in self.files:
            raise ValueError("No Dockerfile specified")
        # Dockerfile must be the last element
        self.files['Dockerfile'] = self.files.pop('Dockerfile')

    def process_parameters(self):
        dockerfile = self.files['Dockerfile']
        if 'ports' in self.parameters and not '\nEXPOSE' in dockerfile:
            extension = '\nEXPOSE ' + ' '.join(str(p) for p in self.parameters['ports']) + '\n'
            self.files['Dockerfile'] += extension
        if 'volumes' in self.parameters and not '\nVOLUME' in dockerfile:
            extension = '\nVOLUME ["' + '" "'.join(str(p) for p in self.parameters['volumes']) + '"]\n'
            self.files['Dockerfile'] += extension
        return self

    def process_addons(self):
        for add_on in self.add_ons:
            with open(os.path.join(ROOT, 'addons', add_on)) as f:
                self.files['Dockerfile'] += f.read()
        return self

    def get_dockerfile(self, process=True):
        with self.get_docker_context(process=process) as context:
            with open(os.path.join(context, 'Dockerfile'), 'rb') as f:
                return f.read()

    @contextmanager
    def get_docker_context(self, process=True):
        if process:
            self.process_addons().process_parameters()
        with chain_temp_files(self.files, self.template_context) as context:
            yield context

class DockerImageManager(object):
    """
    A class to manage Docker images, wich features:
    - Creating a new image out of an existing image.
    - Creating a new image out of a Dockerfile that can be inline or file,
      raw or template.
    - Running an image with a set of parameters.
    - Pushing an image to dockerhub, provided the user is logged in.
    An instance manages a single image.
    """

    def __init__(self, source, **kwargs):
        self.source = source
        kwargs['image'] = self.image_name = kwargs.pop('image_name', None) or self.random_name()
        self.log = kwargs.pop('log', log)
        self.log.debug("New {}(image_name='{}')".format(self.__class__.__name__, self.image_name))
        self.parameters = kwargs.pop('parameters', None)
        if self.parameters:
            self.parameters.update(kwargs)
        else:
            self.parameters = DockerRunParameters(**kwargs)

    @staticmethod
    def random_name():
        return 'nav_' + random_hex()

    @property
    def exists(self):
        return bool(self.inspect())

    @property
    def untagged_name(self):
        return self.image_name.split(':')[0]

    def pull(self):
        wait(docker_client.pull(self.source, stream=True))
        return self

    def build(self, fail_if_exists=True):
        if self.exists:
            if fail_if_exists:
                raise RuntimeError("Image '{}' already exists".format(self.image_name))
        else:
            self.log.info("Building image {}".format(self.image_name))
            with self.source.get_docker_context() as context:
                wait(docker_client.build(path=context, tag=self.image_name, rm=True))
        return self

    def create_image(self):
        if isinstance(self.source, DockerFile):
            return self.build()
        return self.pull()

    def inspect(self, field=None, image_name=None):
        try:
            config = docker_client.inspect_image(image_name or self.image_name)
            if field:
                config = config.get(field, '')
            return config
        except docker.errors.NotFound:
            return None

    def resolve_tag(self, image_name, tag):
        image_name = image_name or self.image_name
        try:
            image_name, _tag = image_name.split(':')
            tag = tag or _tag
        except ValueError:
            pass
        return image_name, tag

    def push(self, image_name=None, tag=None):
        image_name, tag = self.resolve_tag(image_name, tag)
        if tag:
            self.log.info("Pushing image '{}:{}'".format(image_name, tag))
            docker_client.push(image_name, tag)
        else:
            self.log.info("Pushing image '{}'".format(image_name))
            docker_client.push(image_name)
        return self

    def remove_image(self, image_name=None, tag=None):
        image_name, tag = self.resolve_tag(image_name, tag)
        try:
            docker_client.remove_image(image=image_name)
            self.log.info("Removing image '{}'".format(image_name))
        except docker.errors.APIError:
            self.log.warning("Can't remove image '{}': not found".format(image_name))
        return self

    def get_container(self, container_name=None):
        return DockerContainerManager(self, container_name=container_name)

    def create_container(self, container_name=None, **kwargs):
        return DockerContainerManager(self, container_name=container_name).create(**kwargs)

    def print_command(self, image_name, container_name):
        cmd = "docker run -d"
        for k, v in self.parameters.kwargs['port_bindings'].iteritems():
            cmd += ' -p {}:{}'.format(k, v)
        for k, v in self.parameters.kwargs['binds'].iteritems():
            cmd += ' -v {}:{}'.format(k, v['bind'])
        cmd += ' --name {}'.format(container_name)
        cmd += ' {}'.format(image_name)
        return cmd

class DockerContainerManager(object):
    """
    A class to manage docker containers
    """
    def __init__(self, image=None, **kwargs):
        kwargs['name'] = self.container_name = kwargs.pop('container_name', None) or self.random_name()
        if image:
            self.image, self.image_name = image, image.image_name
            self.parameters = self.image.parameters + kwargs
            self.log = image.log
        else:
            self.image, self.image_name = None, None
            self.log = kwargs.pop('log', log)
        self.log.debug("New {}(container_name='{}')".format(self.__class__.__name__, self.container_name))

    def random_name(self):
        if self.image:
            return self.image_name[:12] + '_' + random_hex(11)
        else:
            return random_hex()

    @property
    def exists(self):
        return bool(self.inspect('State'))

    @property
    def is_running(self):
        return self.inspect('State.Running')

    def create(self, start=False, if_exist='fail'):
        """
        Create a container
        :param if_exist: what to do if container already exists
               'fail': raise exception, 'remove': remove container, 'silent': silently do nothing
        """
        if self.exists:
            if if_exist == 'fail':
                raise RuntimeError("Container '{}' already exists".format(self.container_name))
            elif if_exist == 'remove':
                self.remove_container()
        if not self.exists:
            docker_client.create_container(**self.parameters)
        if start:
            self.start()
        return self

    def start(self):
        if not self.is_running:
            self.log.debug("Starting container {}".format(self.container_name))
            docker_client.start(self.container_name)
        return self

    def inspect(self, field='NetworkSettings.IPAddress', container_name=None):
        try:
            config = docker_client.inspect_container(container_name or self.container_name)
            if field:
                for x in field.split('.'):
                    if x and config:
                        config = config.get(x, '')
            return config
        except docker.errors.NotFound:
            return None

    def exec_container(self, cmd, stream=True):
        id = docker_client.exec_create(self.container_name, cmd)
        if stream:
            wait(docker_client.exec_start(exec_id=id, stream=True))
        else:
            docker_client.exec_start(exec_id=id)
        return self

    def copy_from_container(self, guest_path, host_path=None):
        data = docker_client.copy(self.container_name, guest_path)
        if host_path:
            with open(os.path.abspath(os.path.expanduser(host_path)), 'wb') as f:
                f.write(data)
        else:
            return data

    def stop(self):
        if self.is_running:
            self.log.debug("Stopping container {}".format(self.container_name))
            docker_client.stop(self.container_name)
        return self

    def commit(self, image_name, version=''):
        params = (self.container_name, image_name)
        message = "Commiting container {} as {}".format(*params)
        if version:
            params += (version,)
            message += ':' + version
        self.log.debug(message)
        docker_client.commit(*params)
        return self

    def tag(self, image_name, version=''):
        tagged_name = '{}:{}'.format(image_name, version) if version else image_name
        image_name = image_name.split(':')[0]
        if image_name == tagged_name:
            self.log.warning("Trying to tag image_name == tagged_name, nothing to do")
            return self
        self.log.debug("Tagging image {} as last version".format(tagged_name))
        docker_client.tag(tagged_name, image_name)
        return self

    def remove_container(self):
        self.stop()
        if self.exists:
            self.log.debug("Removing container {}".format(self.container_name))
            docker_client.remove_container(self.container_name)
        return self
<commit_msg>force tagging (especially when rebuilding images)<commit_after># encoding: utf-8

"""
DIM: Docker Image Manager
A set of convenience classes to create and manage Docker Images and Containers
"""

from __future__ import unicode_literals, print_function
from collections import OrderedDict
from contextlib import contextmanager
import os
ROOT = os.path.abspath(os.path.dirname(__file__))

import docker

from utils import wait, random_hex, chain_temp_files

docker_client = docker.Client(base_url='unix://var/run/docker.sock', version='auto')

class betterdict(dict):

    def __add__(self, other):
        self.update(other)
        return self

class DockerRunParameters(betterdict):
    """
    A class to define parameters for the docker create/run commands.
    Is also used to enrich Dockerfile EXPOSE and VOLUME instructions.
    """
    allowed = ()

    def __init__(self, **options):
        dict.__init__(self)
        self.log = options.pop('log', log)
        volumes = options.pop('volumes', ())
        ports = options.pop('ports', ())
        links = options.pop('links', ())
        self.update(options)
        self.kwargs = {}
        for vol in volumes:
            self.add_volume(vol)
        for port in ports:
            self.add_port(port)
        for link in links:
            self.add_link(link)
        self.finalize()

    def clone(self):
        # TODO provide true clone
        return betterdict(self)

    def add_volume(self, vol):
        binds = self.kwargs.setdefault('binds', {})
        volumes = self.setdefault('volumes', set())
        # host can be void (volume has no mapping)
        if ':' in vol:
            host, guest = vol.split(':', 1)
        else:
            host, guest = None, vol
        # default mode is 'rw', you can specify 'ro' instead
        if ':' in guest:
            guest, mode = guest.split(':')
        else:
            mode = 'rw'
        if host:
            host = os.path.abspath(os.path.expanduser(host))
            binds[host] = {'bind': guest, 'mode': mode}
        volumes.add(guest)

    def add_port(self, port):
        port_bindings = self.kwargs.setdefault('port_bindings', {})
        ports = self.setdefault('ports', set())
        if isinstance(port, basestring):
            if ':' in port:
                # TODO does not accept format host_ip:host_port:guest_port yet
                host, guest = port.rsplit(':', 1)
                guest = int(guest)
                port_bindings[guest] = int(host) if host else None
                ports.add(guest)
            elif '-' in port:
                start, end = port.split('-')
                for p in xrange(int(start), int(end) + 1):
                    port_bindings[p] = None
                    ports.add(p)
            else:
                port = int(port)
                port_bindings[port] = None
                ports.add(port)
        else:
            port_bindings[port] = None
            ports.add(port)

    def add_link(self, link):
        self.kwargs.setdefault('links', []).append((link, link))

    def finalize(self):
        if self.kwargs:
            self['host_config'] = docker.utils.create_host_config(**self.kwargs)
        if 'ports' in self:
            self['ports'] = list(self['ports'])
            self['ports'].sort()
        if 'volumes' in self:
            self['volumes'] = list(self['volumes'])

    def __add__(self, other):
        res = self.clone()
        res.update(other)
        return res

class DockerFile(object):
    """
    A class to manage Dockerfile like objects
    """
    def __init__(self, *files, **options):
        self.log = options.pop('log', log)
        # this is the global context used by every template involved in the build process
        # templates can be Dockerfile, configuration files, ...
        self.template_context = options.pop('template_context', None) or {}
        self.parameters = options.pop('parameters', None) or {}
        self.add_ons = options.pop('add_ons', None) or ()
        # a file can be a real file, specified by a path,
        # or a pseudo file, specified via a (name, content) pair
        self.files = OrderedDict()
        for file in files:
            if isinstance(file, tuple):
                name, content = file
            else:
                name = os.path.basename(file)
                with open(os.path.expanduser(file), 'rb') as f:
                    content = f.read()
            self.files[name] = content
        self.files.update(options)
        if not 'Dockerfile' in self.files:
            raise ValueError("No Dockerfile specified")
        # Dockerfile must be the last element
        self.files['Dockerfile'] = self.files.pop('Dockerfile')

    def process_parameters(self):
        dockerfile = self.files['Dockerfile']
        if 'ports' in self.parameters and not '\nEXPOSE' in dockerfile:
            extension = '\nEXPOSE ' + ' '.join(str(p) for p in self.parameters['ports']) + '\n'
            self.files['Dockerfile'] += extension
        if 'volumes' in self.parameters and not '\nVOLUME' in dockerfile:
            extension = '\nVOLUME ["' + '" "'.join(str(p) for p in self.parameters['volumes']) + '"]\n'
            self.files['Dockerfile'] += extension
        return self

    def process_addons(self):
        for add_on in self.add_ons:
            with open(os.path.join(ROOT, 'addons', add_on)) as f:
                self.files['Dockerfile'] += f.read()
        return self

    def get_dockerfile(self, process=True):
        with self.get_docker_context(process=process) as context:
            with open(os.path.join(context, 'Dockerfile'), 'rb') as f:
                return f.read()

    @contextmanager
    def get_docker_context(self, process=True):
        if process:
            self.process_addons().process_parameters()
        with chain_temp_files(self.files, self.template_context) as context:
            yield context

class DockerImageManager(object):
    """
    A class to manage Docker images, wich features:
    - Creating a new image out of an existing image.
    - Creating a new image out of a Dockerfile that can be inline or file,
      raw or template.
    - Running an image with a set of parameters.
    - Pushing an image to dockerhub, provided the user is logged in.
    An instance manages a single image.
    """

    def __init__(self, source, **kwargs):
        self.source = source
        kwargs['image'] = self.image_name = kwargs.pop('image_name', None) or self.random_name()
        self.log = kwargs.pop('log', log)
        self.log.debug("New {}(image_name='{}')".format(self.__class__.__name__, self.image_name))
        self.parameters = kwargs.pop('parameters', None)
        if self.parameters:
            self.parameters.update(kwargs)
        else:
            self.parameters = DockerRunParameters(**kwargs)

    @staticmethod
    def random_name():
        return 'nav_' + random_hex()

    @property
    def exists(self):
        return bool(self.inspect())

    @property
    def untagged_name(self):
        return self.image_name.split(':')[0]

    def pull(self):
        wait(docker_client.pull(self.source, stream=True))
        return self

    def build(self, fail_if_exists=True):
        if self.exists:
            if fail_if_exists:
                raise RuntimeError("Image '{}' already exists".format(self.image_name))
        else:
            self.log.info("Building image {}".format(self.image_name))
            with self.source.get_docker_context() as context:
                wait(docker_client.build(path=context, tag=self.image_name, rm=True))
        return self

    def create_image(self):
        if isinstance(self.source, DockerFile):
            return self.build()
        return self.pull()

    def inspect(self, field=None, image_name=None):
        try:
            config = docker_client.inspect_image(image_name or self.image_name)
            if field:
                config = config.get(field, '')
            return config
        except docker.errors.NotFound:
            return None

    def resolve_tag(self, image_name, tag):
        image_name = image_name or self.image_name
        try:
            image_name, _tag = image_name.split(':')
            tag = tag or _tag
        except ValueError:
            pass
        return image_name, tag

    def push(self, image_name=None, tag=None):
        image_name, tag = self.resolve_tag(image_name, tag)
        if tag:
            self.log.info("Pushing image '{}:{}'".format(image_name, tag))
            docker_client.push(image_name, tag)
        else:
            self.log.info("Pushing image '{}'".format(image_name))
            docker_client.push(image_name)
        return self

    def remove_image(self, image_name=None, tag=None):
        image_name, tag = self.resolve_tag(image_name, tag)
        try:
            docker_client.remove_image(image=image_name)
            self.log.info("Removing image '{}'".format(image_name))
        except docker.errors.APIError:
            self.log.warning("Can't remove image '{}': not found".format(image_name))
        return self

    def get_container(self, container_name=None):
        return DockerContainerManager(self, container_name=container_name)

    def create_container(self, container_name=None, **kwargs):
        return DockerContainerManager(self, container_name=container_name).create(**kwargs)

    def print_command(self, image_name, container_name):
        cmd = "docker run -d"
        for k, v in self.parameters.kwargs['port_bindings'].iteritems():
            cmd += ' -p {}:{}'.format(k, v)
        for k, v in self.parameters.kwargs['binds'].iteritems():
            cmd += ' -v {}:{}'.format(k, v['bind'])
        cmd += ' --name {}'.format(container_name)
        cmd += ' {}'.format(image_name)
        return cmd

class DockerContainerManager(object):
    """
    A class to manage docker containers
    """
    def __init__(self, image=None, **kwargs):
        kwargs['name'] = self.container_name = kwargs.pop('container_name', None) or self.random_name()
        if image:
            self.image, self.image_name = image, image.image_name
            self.parameters = self.image.parameters + kwargs
            self.log = image.log
        else:
            self.image, self.image_name = None, None
            self.log = kwargs.pop('log', log)
        self.log.debug("New {}(container_name='{}')".format(self.__class__.__name__, self.container_name))

    def random_name(self):
        if self.image:
            return self.image_name[:12] + '_' + random_hex(11)
        else:
            return random_hex()

    @property
    def exists(self):
        return bool(self.inspect('State'))

    @property
    def is_running(self):
        return self.inspect('State.Running')

    def create(self, start=False, if_exist='fail'):
        """
        Create a container
        :param if_exist: what to do if container already exists
               'fail': raise exception, 'remove': remove container, 'silent': silently do nothing
        """
        if self.exists:
            if if_exist == 'fail':
                raise RuntimeError("Container '{}' already exists".format(self.container_name))
            elif if_exist == 'remove':
                self.remove_container()
        if not self.exists:
            docker_client.create_container(**self.parameters)
        if start:
            self.start()
        return self

    def start(self):
        if not self.is_running:
            self.log.debug("Starting container {}".format(self.container_name))
            docker_client.start(self.container_name)
        return self

    def inspect(self, field='NetworkSettings.IPAddress', container_name=None):
        try:
            config = docker_client.inspect_container(container_name or self.container_name)
            if field:
                for x in field.split('.'):
                    if x and config:
                        config = config.get(x, '')
            return config
        except docker.errors.NotFound:
            return None

    def exec_container(self, cmd, stream=True):
        id = docker_client.exec_create(self.container_name, cmd)
        if stream:
            wait(docker_client.exec_start(exec_id=id, stream=True))
        else:
            docker_client.exec_start(exec_id=id)
        return self

    def copy_from_container(self, guest_path, host_path=None):
        data = docker_client.copy(self.container_name, guest_path)
        if host_path:
            with open(os.path.abspath(os.path.expanduser(host_path)), 'wb') as f:
                f.write(data)
        else:
            return data

    def stop(self):
        if self.is_running:
            self.log.debug("Stopping container {}".format(self.container_name))
            docker_client.stop(self.container_name)
        return self

    def commit(self, image_name, version=''):
        params = (self.container_name, image_name)
        message = "Commiting container {} as {}".format(*params)
        if version:
            params += (version,)
            message += ':' + version
        self.log.debug(message)
        docker_client.commit(*params)
        return self

    def tag(self, image_name, version=''):
        tagged_name = '{}:{}'.format(image_name, version) if version else image_name
        image_name = image_name.split(':')[0]
        if image_name == tagged_name:
            self.log.warning("Trying to tag image_name == tagged_name, nothing to do")
            return self
        self.log.debug("Tagging image {} as last version".format(tagged_name))
        docker_client.tag(tagged_name, image_name, force=True)
        return self

    def remove_container(self):
        self.stop()
        if self.exists:
            self.log.debug("Removing container {}".format(self.container_name))
            docker_client.remove_container(self.container_name)
        return self
<|endoftext|>