Python nntplib.NNTP类代码示例

 def gmane(self):
     try:
         gmane = NNTP("news.gmane.org")
     except:
         return self.gmane
     gmane.group(self.group)
     return gmane

def download_group(name):

    if not os.path.exists(name):
        os.mkdir(name)

    s = NNTP('news.gmane.org')
    resp, count, first, last, name = s.group(name)
    print 'Group', name, 'has', count, 'articles, range', first, 'to', last
    resp, subs = s.xhdr('subject', first + '-' + last)
    for id, sub in subs: 
        print id
        with open(os.path.join(name, str(id)), 'wb') as fp:
            pprint.pprint(s.article(id), stream=fp)

    def __init__(self, group, server):
        self.conn = NNTP(server)
        resp, count, first, last, name = self.conn.group(group)

        self.group = group
        self.server = server
        self.first = int(first)
        self.last = int(last)

def add_new_jobs(group_name):
    db = Connection().usenet
    max_h = db.control.find({"group": group_name}).sort('end', DESCENDING).limit(1)[0]['end'] + 1
    if not max_h: max_h = 0

    news = NNTP('eu.news.astraweb.com', user='vasc', password='dZeZlO89hY6F')
    group = dict_group(news, group_name)
    news.quit()

    i = max(group['first'], max_h)

    if max_h > group['last']: return
    while(i+100000 < group['last']):
        db.control.insert({'init': i, 'end': i+99999, 'done':False, "group": group_name})
        i += 100000

    db.control.insert({'init': i, 'end': group['last'], 'done':False, "group": group_name})

    def getItems(self):
        start = localtime(time()-self.window*day)
        date = strftime('%y%m%d',start)
        hour = strftime('%H%M%S',start)
        server = NNTP(self.servername)
        ids = server.newnews(self.group,date,hour)[1]

        for id in ids:
            lines = server.article(id)[3]
            message = message_from_string('\n'.join(lines))

            title = memssage['subject']
            body = message.get_payload()
            if message.is_multipart():
                body = body[0]

            yield NewsItem(title,body)
        server.quit()

class NewsGrep:
    def __init__(self,server,username,password):
        self.server = NNTP(server, 119,username,password)
    def __del__(self):
        pass

    def __str__(self):
        pass

    def list(self):
        resp, groups = self.server.list()
        for group, last, first, flag in groups:
            print group

    def ls(self,group_name):
        resp, count, first, last, name = self.server.group(group_name)
        print 'Group', name, 'has', count, 'articles, range', first, 'to', last

        resp, subs = self.server.xhdr('subject', first + '-' + last)
        for id, sub in subs[-10:]:
            print id, sub

 def start(self):
     port = self.port
     if not port:
         port = NNTP_PORT
     self.nntp = NNTP(self.server, port, self.user, self.pw,
                      readermode=1)
     # only look for messages that appear after we start. Usenet is big.
     if not self.last: # only do this the first time
         for g in self.groups:
             resp, count, first, last, name = self.nntp.group(g)
             self.last[g] = int(last)
             if self.debug: print "last[%s]: %d" % (g, self.last[g])
     self.timeout = gtk.timeout_add(self.pollInterval*1000,
                                    self.doTimeout)

def main():
  s = NNTP(settings.nntp_hostname, settings.nntp_port)
  resp, groups = s.list()
  # check all of the groups, just in case
  for group_name, last, first, flag in groups:
      resp, count, first, last, name = s.group(group_name)
      print "\nGroup", group_name, 'has', count, 'articles, range', first, 'to', last
      resp, subs = s.xhdr('subject', first + '-' + last)
      for id, sub in subs[-10:]:
          print id, sub
  s.quit()

def main():
    tree = {}

    # Check that the output directory exists
    checkopdir(pagedir)

    try:
        print 'Connecting to %s...' % newshost
        if sys.version[0] == '0':
            s = NNTP.init(newshost)
        else:
            s = NNTP(newshost)
        connected = True
    except (nntplib.error_temp, nntplib.error_perm), x:
        print 'Error connecting to host:', x
        print 'I\'ll try to use just the local list.'
        connected = False

 def get_items(self):
     server = NNTP(self.servername)
     resp, count, first, last, name = server.group(self.group)
     start = last - self.howmany + 1
     resp, overviews = server.over((start, last))
     for id, over in overviews:
         title = decode_header(over['subject'])
         resp, info = server.body(id)
         body = '\n'.join(line.decode('latin')
                          for line in info.lines) + '\n\n'
         yield NewsItem(title, body)
     server.quit()

    def __init__(self, article_cache_size=300, cache_file=None):
        """Initialize local variables"""
        # Connect to news.gmane.org
        self.nntp = NNTP('news.gmane.org')
        # Setting the group returns information, which right now we ignore
        self.nntp.group('gmane.comp.internationalization.dansk')

        # Keep a local cache in an OrderedDict, transferred across session
        # in a pickled version in a file
        self.article_cache_size = article_cache_size
        self.cache_file = cache_file
        if cache_file and path.isfile(cache_file):
            with open(cache_file, 'rb') as file_:
                self.article_cache = pickle.load(file_)
            logging.info('Loaded %i items from file cache',
                         len(self.article_cache))
        else:
            self.article_cache = OrderedDict()

    def __init__(self):
        try:
            self.news = NNTP(NEWS_SERVER, NNTP_PORT, NNTP_USERNAME,
                             NNTP_PASSWORD)
        except NNTPTemporaryError as e:
            raise SpotError('NNTP', e)
        except socket.error as e:
            raise SpotError('Connection', e)

        self.conn = sqlite3.connect(NEWS_SERVER + '.db')
        self.conn.row_factory = sqlite3.Row

        self.cur = self.conn.cursor()
        self.cur.executescript('''\
            PRAGMA synchronous = OFF;
            PRAGMA journal_mode = MEMORY;
            PRAGMA temp_store = MEMORY;
            PRAGMA count_changes = OFF;
        ''')

	def getItems(self):
	#	yesterday = localtime(time()-self.window*day)
	#	date = strftime('%y%m%d',yesterday)
	#	time = strftime('%H%M%S',yesterday)
		# create a nntp server
		s = NNTP(self.servername)	
		resp,count,first,last,name = s.group(self.groupname)
		resp,overviews = s.over((last-1,last))
		for num,over in overviews:
			title = over.get('subject')
			resp,body = s.body(num)
			# create a generator to iterate news 
			if title and body:
				yield NewsItem(title,body)
		s.quit()	

 def connect(self):
     address = net.format_addr((self.hostname, self.port))
     self.log.write("Connecting to {}\n".format(address))
     if self.port is None:
         port = ()
     else:
         port = (self.port,)
     self.connect_time = time.monotonic()
     self.nntp = NNTP(self.hostname, *port, **self.timeout)
     with ExitStack() as cleanup:
         cleanup.push(self)
         if self.debuglevel is not None:
             self.nntp.set_debuglevel(self.debuglevel)
         self.log.write("{}\n".format(self.nntp.getwelcome()))
         
         if self.username is not None:
             self.log.write("Logging in as {}\n".format(self.username))
             with self.handle_abort():
                 self.nntp.login(self.username, self.password)
             self.log.write("Logged in\n")
         cleanup.pop_all()

 def fetch(self):
     """
     Fetches all the messages for a given news group uri and return Fetched staus depending 
     on the success and faliure of the task
     """
     try:
         #eg. self.currenturi = nntp://msnews.microsoft.com/microsoft.public.exchange.setup
         #nntp_server = 'msnews.microsoft.com'
         #nntp_group = 'microsoft.public.exchange.setup'
         self.genre = 'review'
         try:
             nntp_server = urlparse(self.currenturi)[1]
         except:
             log.exception(self.log_msg("Exception occured while connecting to NNTP server %s"%self.currenturi))
             return False
         nntp_group =  urlparse(self.currenturi)[2][1:]
         self.server = NNTP(nntp_server)
         try:
             self.__updateParentSessionInfo()
             resp, count, first, last, name = self.server.group(nntp_group)
             last_id = int(last)
             first_id = self.__getMaxCrawledId(last_id)+1
             log.debug("first_id is %d:"%first_id)
             log.debug("last_id is %d:"%last_id)
             if last_id >= first_id:
                 resp, items = self.server.xover(str(first_id), str(last_id))
                 log.debug(self.log_msg("length of items:%s"%str(len(items))))
                 for self.id, self.subject, self.author, self.date, self.message_id,\
                         self.references, size, lines in items:
                     self.__getMessages(self.task.instance_data['uri'])
             self.server.quit()
             return True
         except:
             log.exception(self.log_msg("Exception occured in fetch()"))
             self.server.quit()
             return False
     except Exception,e:
         log.exception(self.log_msg("Exception occured in fetch()"))
         return False