Python models.MPageFetchHistory类代码示例

def calculate_metrics():
    import datetime
    from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory
    
    last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1)
    
    return {
        'feed_errors': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
        'feed_success': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
        'page_errors': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
        'page_success': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
    }

def load_feed_statistics(request, feed_id):
    user = get_user(request)
    stats = dict()
    feed = get_object_or_404(Feed, pk=feed_id)
    feed.count_subscribers()
    feed.set_next_scheduled_update(verbose=True, skip_scheduling=True)
    feed.save_feed_story_history_statistics()
    feed.save_classifier_counts()
    
    # Dates of last and next update
    stats['active'] = feed.active
    stats['last_update'] = relative_timesince(feed.last_update)
    if feed.is_push:
        stats['next_update'] = "real-time..."
    else:
        stats['next_update'] = relative_timeuntil(feed.next_scheduled_update)

    # Minutes between updates
    update_interval_minutes, _ = feed.get_next_scheduled_update(force=True, verbose=False)
    if feed.is_push:
        stats['update_interval_minutes'] = 0
    else:
        stats['update_interval_minutes'] = update_interval_minutes
    original_active_premium_subscribers = feed.active_premium_subscribers
    original_premium_subscribers = feed.premium_subscribers
    feed.active_premium_subscribers = max(feed.active_premium_subscribers+1, 1)
    feed.premium_subscribers += 1
    premium_update_interval_minutes, _ = feed.get_next_scheduled_update(force=True, verbose=False)
    feed.active_premium_subscribers = original_active_premium_subscribers
    feed.premium_subscribers = original_premium_subscribers
    if feed.is_push:
        stats['premium_update_interval_minutes'] = 0
    else:
        stats['premium_update_interval_minutes'] = premium_update_interval_minutes
    
    # Stories per month - average and month-by-month breakout
    average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history
    stats['average_stories_per_month'] = average_stories_per_month
    stats['story_count_history'] = story_count_history and json.decode(story_count_history)
    
    # Subscribers
    stats['subscriber_count'] = feed.num_subscribers
    stats['stories_last_month'] = feed.stories_last_month
    stats['last_load_time'] = feed.last_load_time
    stats['premium_subscribers'] = feed.premium_subscribers
    stats['active_subscribers'] = feed.active_subscribers
    stats['active_premium_subscribers'] = feed.active_premium_subscribers
    
    # Classifier counts
    stats['classifier_counts'] = json.decode(feed.data.feed_classifier_counts)
    
    # Fetch histories
    timezone = user.profile.timezone
    stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id, timezone=timezone)
    stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id, timezone=timezone)
    stats['feed_push_history'] = MFeedPushHistory.feed_history(feed_id, timezone=timezone)
    
    logging.user(request, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (feed, feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers,))

    return stats

def load_feed_statistics(request):
    stats = dict()
    feed_id = request.GET['feed_id']
    feed = get_object_or_404(Feed, pk=feed_id)
    feed.save_feed_story_history_statistics()
    
    # Dates of last and next update
    stats['last_update'] = relative_timesince(feed.last_update)
    stats['next_update'] = relative_timeuntil(feed.next_scheduled_update)
    
    # Minutes between updates
    update_interval_minutes, random_factor = feed.get_next_scheduled_update()
    stats['update_interval_minutes'] = update_interval_minutes
    
    # Stories per month - average and month-by-month breakout
    average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history
    stats['average_stories_per_month'] = average_stories_per_month
    stats['story_count_history'] = story_count_history and json.decode(story_count_history)
    
    # Subscribers
    stats['subscriber_count'] = feed.num_subscribers
    stats['stories_last_month'] = feed.stories_last_month
    stats['last_load_time'] = feed.last_load_time
    stats['premium_subscribers'] = feed.premium_subscribers
    stats['active_subscribers'] = feed.active_subscribers
    
    # Fetch histories
    stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id)
    stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id)
    
    logging.user(request.user, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (feed, feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers,))

    return stats

def load_feed_settings(request, feed_id):
    stats = dict()
    feed = get_object_or_404(Feed, pk=feed_id)
    
    stats['duplicate_addresses'] = feed.duplicate_addresses.all()
    stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id)
    stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id)
    
    return stats

def load_feed_settings(request, feed_id):
    stats = dict()
    feed = get_object_or_404(Feed, pk=feed_id)
    user = get_user(request)
    timezone = user.profile.timezone
    
    stats['duplicate_addresses'] = feed.duplicate_addresses.all()
    stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id, timezone=timezone)
    stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id, timezone=timezone)
    
    return stats

 def delete_old_history():
     print "Deleting old history."
     feed_fetch_last_day = MFeedFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304])
     page_fetch_last_day = MPageFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304])
     feed_fetch_last_month = MFeedFetchHistory.objects(fetch_date__lt=last_month)
     page_fetch_last_month = MPageFetchHistory.objects(fetch_date__lt=last_month)
     push_last_month = MFeedPushHistory.objects(push_date__lt=last_month)
     print "Found %s/%s/%s/%s/%s (feed/page day, feed/page month, push month)" % (
         feed_fetch_last_day.count(),
         page_fetch_last_day.count(),
         feed_fetch_last_month.count(),
         page_fetch_last_month.count(),
         push_last_month.count(),
     )
     
     feed_fetch_last_day.delete()
     page_fetch_last_day.delete()
     feed_fetch_last_month.delete()
     page_fetch_last_month.delete()
     push_last_month.delete()

 def collect_statistics_feeds_fetched(cls, last_day=None):
     if not last_day:
         last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
     
     feeds_fetched = MFeedFetchHistory.objects(fetch_date__gte=last_day).count()
     cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched)
     
     old_fetch_histories = MFeedFetchHistory.objects(fetch_date__lte=last_day)
     for history in old_fetch_histories:
         history.delete()
     old_page_histories = MPageFetchHistory.objects(fetch_date__lte=last_day)
     for history in old_page_histories:
         history.delete()
     
     return feeds_fetched

#!/usr/bin/env python 

from utils.munin.base import MuninGraph
from apps.rss_feeds.models import MFeedFetchHistory, MPageFetchHistory
import datetime


graph_config = {
    'graph_category' : 'NewsBlur',
    'graph_title' : 'NewsBlur Fetching History',
    'graph_vlabel' : 'errors',
    'feed_errors.label': 'Feed Errors',
    'feed_success.label': 'Feed Success',
    'page_errors.label': 'Page Errors',
    'page_success.label': 'Page Success',
}

last_day = datetime.datetime.utcnow() - datetime.timedelta(days=1)

metrics = {
    'feed_errors': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
    'feed_success': MFeedFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
    'page_errors': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__nin=[200, 304]).count(),
    'page_success': MPageFetchHistory.objects(fetch_date__gte=last_day, status_code__in=[200, 304]).count(),
}

if __name__ == '__main__':
    MuninGraph(graph_config, metrics).run()

def exception_change_feed_link(request):
    feed_id = request.POST['feed_id']
    feed = get_object_or_404(Feed, pk=feed_id)
    original_feed = feed
    feed_link = request.POST['feed_link']
    timezone = request.user.profile.timezone
    code = -1
    
    if feed.has_page_exception or feed.has_feed_exception:
        # Fix broken feed
        logging.user(request, "~FRFixing feed exception by link: ~SB%s~SN to ~SB%s" % (feed.feed_link, feed_link))
        feed_address = feedfinder.feed(feed_link)
        if feed_address:
            code = 1
            feed.has_page_exception = False
            feed.active = True
            feed.fetched_once = False
            feed.feed_link = feed_link
            feed.feed_address = feed_address
            duplicate_feed = feed.schedule_feed_fetch_immediately()
            if duplicate_feed:
                new_feed = Feed.objects.get(pk=duplicate_feed.pk)
                feed = new_feed
                new_feed.schedule_feed_fetch_immediately()
                new_feed.has_page_exception = False
                new_feed.active = True
                new_feed.save()
    else:
        # Branch good feed
        logging.user(request, "~FRBranching feed by link: ~SB%s~SN to ~SB%s" % (feed.feed_link, feed_link))
        feed, _ = Feed.objects.get_or_create(feed_address=feed.feed_address, feed_link=feed_link)
        code = 1
        if feed.pk != original_feed.pk:
            try:
                feed.branch_from_feed = original_feed.branch_from_feed or original_feed
            except Feed.DoesNotExist:
                feed.branch_from_feed = original_feed
            feed.feed_link_locked = True
            feed.save()

    feed = feed.update()
    feed = Feed.get_by_id(feed.pk)

    try:
        usersub = UserSubscription.objects.get(user=request.user, feed=feed)
    except UserSubscription.DoesNotExist:
        usersubs = UserSubscription.objects.filter(user=request.user, feed=original_feed)
        if usersubs:
            usersub = usersubs[0]
            usersub.switch_feed(feed, original_feed)
        else:
            return {
                'code': -1,
                'feed_fetch_history': MFeedFetchHistory.feed_history(feed_id, timezone=timezone),
                'page_fetch_history': MPageFetchHistory.feed_history(feed_id, timezone=timezone),
            }
        
    usersub.calculate_feed_scores(silent=False)
    
    feed.update_all_statistics()
    classifiers = get_classifiers_for_user(usersub.user, feed_id=usersub.feed_id)
    
    if feed and feed.has_feed_exception:
        code = -1
    
    feeds = {
        original_feed.pk: usersub.canonical(full=True, classifiers=classifiers), 
    }
    return {
        'code': code, 
        'feeds': feeds, 
        'new_feed_id': usersub.feed_id,
        'feed_fetch_history': MFeedFetchHistory.feed_history(feed_id, timezone=timezone),
        'page_fetch_history': MPageFetchHistory.feed_history(feed_id, timezone=timezone),
    }

 def delete_old_history():
     MFeedFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete()
     MPageFetchHistory.objects(fetch_date__lt=last_day, status_code__in=[200, 304]).delete()
     MFeedFetchHistory.objects(fetch_date__lt=last_month).delete()
     MPageFetchHistory.objects(fetch_date__lt=last_month).delete()
     MFeedPushHistory.objects(push_date__lt=last_month).delete()

 def delete_old_history():
     MFeedFetchHistory.objects(fetch_date__lt=last_day).delete()
     MPageFetchHistory.objects(fetch_date__lt=last_day).delete()