Python NNTP.xover方法代码示例

# 需要导入模块: from nntplib import NNTP [as 别名]
# 或者: from nntplib.NNTP import xover [as 别名]
class NNTPConnector(BaseConnector):  
    @logit(log,'fetch')
    def fetch(self):
        """
        Fetches all the messages for a given news group uri and return Fetched staus depending 
        on the success and faliure of the task
        """
        try:
            #eg. self.currenturi = nntp://msnews.microsoft.com/microsoft.public.exchange.setup
            #nntp_server = 'msnews.microsoft.com'
            #nntp_group = 'microsoft.public.exchange.setup'
            self.genre = 'review'
            try:
                nntp_server = urlparse(self.currenturi)[1]
            except:
                log.exception(self.log_msg("Exception occured while connecting to NNTP server %s"%self.currenturi))
                return False
            nntp_group =  urlparse(self.currenturi)[2][1:]
            self.server = NNTP(nntp_server)
            try:
                self.__updateParentSessionInfo()
                resp, count, first, last, name = self.server.group(nntp_group)
                last_id = int(last)
                first_id = self.__getMaxCrawledId(last_id)+1
                log.debug("first_id is %d:"%first_id)
                log.debug("last_id is %d:"%last_id)
                if last_id >= first_id:
                    resp, items = self.server.xover(str(first_id), str(last_id))
                    log.debug(self.log_msg("length of items:%s"%str(len(items))))
                    for self.id, self.subject, self.author, self.date, self.message_id,\
                            self.references, size, lines in items:
                        self.__getMessages(self.task.instance_data['uri'])
                self.server.quit()
                return True
            except:
                log.exception(self.log_msg("Exception occured in fetch()"))
                self.server.quit()
                return False
        except Exception,e:
            log.exception(self.log_msg("Exception occured in fetch()"))
            return False

# 需要导入模块: from nntplib import NNTP [as 别名]
# 或者: from nntplib.NNTP import xover [as 别名]
sql = sqlite3.connect('usenet.db')

print "About to start dumping articles from " + group
serv = NNTP('news.astraweb.com', 119, 'arealmuto', 'stock1114')

resp = serv.group(group)
count = int(resp[1])
first = int(resp[2])
last = int(resp[3])

print "There are " + str(count) + " articles to get"
print "First: " + str(first)
print "Last: " + str(last)
print "Using chunks size of: " + str(chunk_size)
print "It should take " + str(count/chunk_size) + " requests to finish"

id = int(first)

i = 0
master_list = []
while id < last:
	print str(i) +": Getting id's " + str(id) + " - " + str(id + chunk_size)
	resp, list = serv.xover(str(id), str(id+ chunk_size))
	print "Done fetching"
	print "Adding to master list"
	for line in list:
		article = (line[0], line[1], line[2], line[3], line[4])
		master_list.append(article)			
	id += chunk_size + 1
	i += 1

# 需要导入模块: from nntplib import NNTP [as 别名]
# 或者: from nntplib.NNTP import xover [as 别名]
class Archive(object):

    @staticmethod
    def is_diff(body):
        return bool([line for line in body if line.startswith("diff ")])

    def __init__(self, group, server):
        self.conn = NNTP(server)
        resp, count, first, last, name = self.conn.group(group)

        self.group = group
        self.server = server
        self.first = int(first)
        self.last = int(last)

    def get_number_from_user(self, msg_id):
        """
            Convert something the user might input into a message id.

            These are:
            # An NNTP message number
            # A gmane link that includes the NNTP message number
            # The original Message-Id header of the message.

            NOTE: gmane's doesn't include the message number in STAT requests
            that involve only the Message-Id (hence the convolution of getting
            all the headers).
        """
        msg_id = re.sub(r".*gmane.org/gmane.comp.version-control.git/([0-9]+).*", r"\1", str(msg_id))
        _, n, id, result = self.conn.head(msg_id)

        for header in result:
            m = re.match(r"Xref: .*:([0-9]+)\s*$", header, re.I)
            if m:
                return int(m.group(1))
        else:
            raise FatalError("No (or bad) Xref header for message '%s'" % msg_id)

    def get_patch_series(self, user_input, search_limit=100):
        """
            Given an NNTP message number or a Message-Id header return
            an mbox containing the patches introduced by the author of that message.

            This handles the case where the threading is right *and* the patches
            are numbered in a simple scheme:

            [PATCH] this patch has no replies and stands on its own

            [PATCH 0/2] this is an introduction to the series
              |- [PATCH 1/2] the first commit
              |- [PATCH 2/2] the second commit

            [PATCH 1/3] this is the first commit
              |- [PATCH 2/3] and this is the second
                   |- [PATCH 3/3] and this is the third

            TODO: it would be nice to make the search more efficient, we can
            use the numbers in [PATCH <foo>/<bar>] to stop early.
        """

        start_id = self.get_number_from_user(user_input)

        messages = limit(self.messages_starting_from(start_id), search_limit)
        try:
            thread = Thread(messages.next())
        except StopIteration:
            raise FatalError("No message at id '%s' using XOVER")

        n_since_last = 0
        for message in messages:
            if n_since_last > 5:
                break

            elif thread.should_include(message):
                n_since_last = 0
                thread.append(message)

            else:
                n_since_last += 1

        else:
            raise FatalError('did not find end of series within %s messages', search_limit)

        for message in self.xover(start_id - 5, start_id -1):
            if thread.should_include(message):
                thread.append(message)

        return self.mboxify(thread)

    def mboxify(self, thread):
        """
            Convert a thread into an mbox for application via git-am.
        """
        lines = []

        for message in thread.in_order():
            _, number, msg_id, body = self.conn.body(str(message.number))

            # git-am doesn't like empty patches very much, and the 0/X'th patch is
            # often not a patch, we skip it here. (TODO, warn the user about this)
#.........这里部分代码省略.........