source: feed2telegram/trunk/feed2telegram.py

#-*- coding: utf-8 -*-
import datetime
import email
import json
import os
import sys
import threading
import time
import traceback
import urlparse

import feedparser
import requests

class Telegram:
    class API:
        URL = 'https://api.telegram.org/bot<token>/<method>'

        def __init__(self, token, method, http_method='POST'):
            self.token = token
            self.method = method
            self.http_method = http_method.upper()
            self.param = {}

        def setParam(self, **kwargs):
            self.param.update(kwargs)
            return self

        @property
        def url(self):
            ret = Telegram.API.URL.replace('<token>', self.token).replace('<method>', self.method)
            return ret

        @property
        def header(self):
            header = {
                'accept': 'application/x-www-form-urlencoded',
            }
            return header

    def __init__(self, token, chat_id):
        self.token = token
        self.chatId = chat_id

    def request(self, api):
        if api.http_method == 'GET':
            req = requests.get(api.url, headers=api.header, params=api.param)
        else:# if api.http_method == 'POST':
            req = requests.get(api.url, headers=api.header, data=api.param)

        response = req.json()

        if not response['ok']:
            raise Exception(response['description'])

        return response

    def getUpdates(self, limit=None):
        api = Telegram.API(token=self.token, method='getUpdates', http_method='GET')

        if limit:
            api.setParam(limit=limit)

        return self.request(api)

    def sendMessage(self, text):
        api = Telegram.API(token=self.token, method='sendMessage').setParam(chat_id=self.chatId, text=text)
        return self.request(api)

class Datetime:
    def __init__(self):
        self.datetime = None

    def setRfc2822(self, value):
        parsed = email.utils.parsedate(value)
        self.datetime = datetime.datetime(*parsed[:6])
        return self

    def toTimestamp(self):
        timetuple = self.datetime.timetuple()
        return time.mktime(timetuple)

class Cloudflare:
    @staticmethod
    def check(response):
        return response.status_code == 503 and 'DDoS protection by Cloudflare' in response.content

    @staticmethod
    def get(*args, **kwargs):
        import cfscrape
        return cfscrape.create_scraper().get(*args, **kwargs)

class Feed:
    def __init__(self, url):
        self.url = url
        self.etag = None
        self.modified = None
        self.feed = None

        self.save_dir = '.changes'

        if os.path.isfile(self.filepath):
            self.load()

    @property
    def filepath(self):
        domain = urlparse.urlparse(self.url).netloc
        return os.path.join(self.save_dir, domain)

    def save(self):
        data = {
            'url': self.url,
            'etag': self.etag,
            'modified': self.modified
        }
        jsonstr = json.dumps(data, indent=2)

        if not os.path.isdir(self.save_dir):
            os.makedirs(self.save_dir)

        with open(self.filepath, 'w') as f:
            f.write(jsonstr)

    def load(self):
        with open(self.filepath, 'r') as f:
            jsonobj = json.load(f)

        self.etag = jsonobj['etag']
        self.modified = jsonobj['modified']

    def get(self):
        try:
            resp = requests.get(self.url, timeout=10.0)
            if Cloudflare.check(resp):
                resp = Cloudflare.get(self.url, timeout=10.0)
        except requests.ReadTimeout as e:
            raise

        # self.feed = feedparser.parse(self.url)# , etag=self.etag, modified=self.modified)
        self.feed = feedparser.parse(resp.content)

        try:
            self.etag = self.feed.etag
        except:
            self.etag = None

        try:
            self.modified = self.feed.modified
        except:
            convert = lambda rfc2822: Datetime().setRfc2822(rfc2822).toTimestamp()
            mtimes = [convert(entry.published) for entry in self.feed.entries]

            if len(mtimes) > 0:
                self.modified = self.feed.entries[mtimes.index(max(mtimes))].published

        return self.feed

class Feed2Telegram:
    def __init__(self,
                 feed_url,              # required
                 telegram_token,        # required
                 chat_id,               # required
                 callback_get_message,  # required
                 callback_get_entries=lambda entries: reversed(entries),
                 check_interval=60*60,
                 new_entries_only=True,
                 send_error=True,
                 continue_on_error=False,
                 threaded=False):
        self.url  = feed_url
        self.feed = Feed(self.url)

        self.token    = telegram_token
        self.chat_id  = chat_id
        self.telegram = Telegram(self.token, self.chat_id)

        self.get_message = callback_get_message
        self.get_entries = callback_get_entries

        self.new_entries_only = new_entries_only
        self.send_error = send_error
        self.continue_on_error = continue_on_error

        self.thread = None
        if threaded:
            self.thread = threading.Thread(target=self.run)

        self.stop_event = threading.Event()
        self.last_modified = None

    def isSendingEntry(self, entry):
        if self.new_entries_only:
            if self.last_modified:
                published = Datetime().setRfc2822(entry.published).toTimestamp()
                last      = Datetime().setRfc2822(self.last_modified).toTimestamp()

                if published > last:
                    return True
            else:
                return True

        return False

    def once(self):
        self.last_modified = self.feed.modified
        feed = self.feed.get()

        for entry in self.get_entries(feed.entries):
            if self.stop_event.is_set():
                break

            if not self.isSendingEntry(entry):
                continue

            message = self.get_message(entry)
            self.telegram.sendMessage(message)

        self.feed.save()

    def run(self):
        while not self.stop_event.is_set():

            try:
                self.once()

            except:
                if self.send_error:
                    e = traceback.format_exc()
                    sys.stderr.write(e)
#                   self.telegram.sendMessage(e)

                if not self.continue_on_error:
                    raise

            finally:
                if not self.stop_event.is_set() and self.continue_on_error:
                    self.stop_event.wait(60 * 60)

    def start(self):
        self.stop_event.clear()

        if self.thread:
            self.thread.run()
        else:
            self.run()

    def join(self, timeout=None):
        if self.thread:
            self.thread.join(timeout)

    def stop(self):
        self.stop_event.set()