Python Database.list_tasks方法代码示例

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def index(request):
    db = Database()
    tasks_files = db.list_tasks(limit=50, category="file", not_status=TASK_PENDING)
    tasks_urls = db.list_tasks(limit=50, category="url", not_status=TASK_PENDING)

    analyses_files = []
    analyses_urls = []

    if tasks_files:
        for task in tasks_files:
            new = task.to_dict()
            new["sample"] = db.view_sample(new["sample_id"]).to_dict()
            if db.view_errors(task.id):
                new["errors"] = True

            analyses_files.append(new)

    if tasks_urls:
        for task in tasks_urls:
            new = task.to_dict()

            if db.view_errors(task.id):
                new["errors"] = True

            analyses_urls.append(new)

    return render_to_response("analysis/index.html",
                              {"files": analyses_files, "urls": analyses_urls},
                              context_instance=RequestContext(request))

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def cuckoo_clean_bson_suri_logs():
    """Clean up raw suri log files probably not needed if storing in mongo. Does not remove extracted files
    """
    # Init logging.
    # This need to init a console logger handler, because the standard
    # logger (init_logging()) logs to a file which will be deleted.
    create_structure()
    init_console_logging()
    from glob import glob
    # Initialize the database connection.
    db = Database()
    failed_tasks_a = db.list_tasks(status=TASK_FAILED_ANALYSIS)
    failed_tasks_p = db.list_tasks(status=TASK_FAILED_PROCESSING)
    failed_tasks_r = db.list_tasks(status=TASK_FAILED_REPORTING)
    failed_tasks_rc = db.list_tasks(status=TASK_RECOVERED)
    tasks_rp = db.list_tasks(status=TASK_REPORTED)
    for e in failed_tasks_a,failed_tasks_p,failed_tasks_r,failed_tasks_rc,tasks_rp:
        for el2 in e:
            new = el2.to_dict()
            id = new["id"]
            path = os.path.join(CUCKOO_ROOT, "storage", "analyses","%s" % id)
            if os.path.exists(path):
                jsonlogs=glob("%s/logs/*json*" % (path))
                bsondata=glob("%s/logs/*.bson" % (path))
                filesmeta=glob("%s/logs/files/*.meta" %(path))
                for f in jsonlogs,bsondata,filesmeta:
                    for fe in f:
                        try:
                            print "removing %s" % (fe)
                            os.remove(fe)
                        except Exception as Err:
                            print "failed to remove sorted_pcap from disk %s" % (Err)

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def cuckoo_clean_failed_tasks():
    """Clean up failed tasks 
    It deletes all stored data from file system and configured databases (SQL
    and MongoDB for failed tasks.
    """
    # Init logging.
    # This need to init a console logger handler, because the standard
    # logger (init_logging()) logs to a file which will be deleted.
    create_structure()
    init_console_logging()

    # Initialize the database connection.
    db = Database()

    # Check if MongoDB reporting is enabled and drop that if it is.
    cfg = Config("reporting")
    if cfg.mongodb and cfg.mongodb.enabled:
        from pymongo import MongoClient
        host = cfg.mongodb.get("host", "127.0.0.1")
        port = cfg.mongodb.get("port", 27017)
        mdb = cfg.mongodb.get("db", "cuckoo")
        try:
            results_db = MongoClient(host, port)[mdb]
        except:
            log.warning("Unable to connect to MongoDB database: %s", mdb)
            return 

        failed_tasks_a = db.list_tasks(status=TASK_FAILED_ANALYSIS)
        failed_tasks_p = db.list_tasks(status=TASK_FAILED_PROCESSING)
        failed_tasks_r = db.list_tasks(status=TASK_FAILED_REPORTING)
        failed_tasks_rc = db.list_tasks(status=TASK_RECOVERED)
        for e in failed_tasks_a,failed_tasks_p,failed_tasks_r,failed_tasks_rc:
            for el2 in e:
                new = el2.to_dict()
                remove_task(new["id"])

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def index(request):
    db = Database()
    tasks_files = db.list_tasks(limit=50, category="file", not_status=TASK_PENDING)
    tasks_urls = db.list_tasks(limit=50, category="url", not_status=TASK_PENDING)

    analyses_files = []
    analyses_urls = []

    if tasks_files:
        for task in tasks_files:
            new = task.to_dict()
            new["sample"] = db.view_sample(new["sample_id"]).to_dict()

            filename = os.path.basename(new["target"])
            new.update({"filename": filename})

            if db.view_errors(task.id):
                new["errors"] = True

            analyses_files.append(new)

    if tasks_urls:
        for task in tasks_urls:
            new = task.to_dict()

            if db.view_errors(task.id):
                new["errors"] = True

            analyses_urls.append(new)

    return render(request, "analysis/index.html", {
        "files": analyses_files,
        "urls": analyses_urls,
    })

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def index(request):
    db = Database()
    dbmx = DatabaseMX()

    report = dict(
        total_mails=dbmx.count_mails(),
        total_malwares=dbmx.count_malwares(),
        total_attachments=dbmx.count_attachments(),
        total_urls=dbmx.count_urls(),
        mails_have_malwares=dbmx.get_mails_have_malwares(),

        total_samples=db.count_samples(),
        total_tasks=db.count_tasks(),
        states_count={},
        estimate_hour=None,
        estimate_day=None
    )
    
    states = (
        TASK_PENDING,
        TASK_RUNNING,
        TASK_COMPLETED,
        TASK_RECOVERED,
        TASK_REPORTED,
        TASK_FAILED_ANALYSIS,
        TASK_FAILED_PROCESSING,
        TASK_FAILED_REPORTING
    )

    for state in states:
        report["states_count"][state] = db.count_tasks(state)

    offset = None

    # For the following stats we're only interested in completed tasks.
    tasks = db.list_tasks(offset=offset, status=TASK_COMPLETED)
    tasks += db.list_tasks(offset=offset, status=TASK_REPORTED)

    if tasks:
        # Get the time when the first task started.
        started = min(timestamp(task.started_on) for task in tasks)

        # Get the time when the last task completed.
        completed = max(timestamp(task.completed_on) for task in tasks)

        # Get the amount of tasks that actually completed.
        finished = len(tasks)

        hourly = 60 * 60 * finished / (completed - started)

        report["estimate_hour"] = int(hourly)
        report["estimate_day"] = int(24 * hourly)

    return render_to_response("cuckoomx/index.html",
                              {"report" : report},
                              context_instance=RequestContext(request))

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def index(request):
    db = Database()

    report = dict(
        total_samples=db.count_samples(),
        total_tasks=db.count_tasks(),
        states_count={},
        estimate_hour=None,
        estimate_day=None
    )

    states = (
        TASK_PENDING,
        TASK_RUNNING,
        TASK_COMPLETED,
        TASK_RECOVERED,
        TASK_REPORTED,
        TASK_FAILED_ANALYSIS,
        TASK_FAILED_PROCESSING,
        TASK_FAILED_REPORTING
    )

    for state in states:
        report["states_count"][state] = db.count_tasks(state)

    offset = None

    # For the following stats we're only interested in completed tasks.
    tasks = db.list_tasks(offset=offset, status=TASK_COMPLETED)
    tasks += db.list_tasks(offset=offset, status=TASK_REPORTED)

    if tasks:
        tasksubset = [x for x in tasks if timestamp(x.started_on) >= timestamp(datetime.datetime.today() - timedelta(days=30))]

        # Get the time when the first task started.
        started = min(timestamp(task.started_on) for task in tasksubset)

        # Get the time when the last task completed.
        completed = max(timestamp(task.completed_on) for task in tasksubset)

        # Get the amount of tasks that actually completed.
        finished = len(tasksubset)

        # It has happened that for unknown reasons completed and started were
        # equal in which case an exception is thrown, avoid this.
        if int(completed - started):
            hourly = 60 * 60 * finished / (completed - started)
        else:
            hourly = 0

        report["estimate_hour"] = int(hourly)
        report["estimate_day"] = int(24 * hourly)

    return render_to_response("dashboard/index.html",
                              {"report" : report},
                              context_instance=RequestContext(request))

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def main():
    db = Database()

    print("%d samples in db" % db.count_samples())
    print("%d tasks in db" % db.count_tasks())

    states = (
        TASK_PENDING,
        TASK_RUNNING,
        TASK_COMPLETED,
        TASK_RECOVERED,
        TASK_REPORTED,
        TASK_FAILED_ANALYSIS,
        TASK_FAILED_PROCESSING,
        TASK_FAILED_REPORTING,
    )

    for state in states:
        print("%s %d tasks" % (state, db.count_tasks(state)))

    # Later on we might be interested in only calculating stats for all
    # tasks starting at a certain offset, because the Cuckoo daemon may
    # have been restarted at some point in time.
    offset = None

    # For the following stats we're only interested in completed tasks.
    tasks = db.list_tasks(offset=offset, status=TASK_COMPLETED)
    tasks += db.list_tasks(offset=offset, status=TASK_REPORTED)

    if tasks:
        # Get the time when the first task started.
        started = min(timestamp(task.started_on) for task in tasks)

        # Get the time when the last task completed.
        stamps = []
        for task in tasks:
            try:
                stamps.append(timestamp(task.completed_on))
            except AttributeError:
                pass

        completed = max(stamps)

        # Get the amount of tasks that actually completed.
        finished = len(tasks)

        hourly = 60 * 60 * finished / (completed - started)

        print("roughly %d tasks an hour" % int(hourly))
        print("roughly %d tasks a day" % int(24 * hourly))

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def cuckoo_clean_failed_tasks():
    """Clean up failed tasks 
    It deletes all stored data from file system and configured databases (SQL
    and MongoDB for failed tasks.
    """
    # Init logging.
    # This need to init a console logger handler, because the standard
    # logger (init_logging()) logs to a file which will be deleted.
    create_structure()
    init_console_logging()

    # Initialize the database connection.
    db = Database()

    # Check if MongoDB reporting is enabled and drop that if it is.
    cfg = Config("reporting")
    if cfg.mongodb and cfg.mongodb.enabled:
        from pymongo import MongoClient
        host = cfg.mongodb.get("host", "127.0.0.1")
        port = cfg.mongodb.get("port", 27017)
        mdb = cfg.mongodb.get("db", "cuckoo")
        try:
            results_db = MongoClient(host, port)[mdb]
        except:
            log.warning("Unable to connect to MongoDB database: %s", mdb)
            return 

        failed_tasks_a = db.list_tasks(status=TASK_FAILED_ANALYSIS)
        failed_tasks_p = db.list_tasks(status=TASK_FAILED_PROCESSING)
        failed_tasks_r = db.list_tasks(status=TASK_FAILED_REPORTING)

        for e in failed_tasks_a,failed_tasks_p,failed_tasks_r:
            for el2 in e:
                new = el2.to_dict()
                print int(new["id"])
                try:
                    results_db.suricata.remove({"info.id": int(new["id"])})
                except:
                    print "failed to remove suricata info (may not exist) %s" % (int(new["id"]))
                try:
                    results_db.analysis.remove({"info.id": int(new["id"])})
                except:
                    print "failed to remove analysis info (may not exist) %s" % (int(new["id"]))
                if db.delete_task(new["id"]):
                    delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses",
                            "%s" % int(new["id"])))
                else:
                    print "failed to remove faile task %s from DB" % (int(new["id"]))

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def remove_pending(request):
    db = Database()    
    tasks = db.list_tasks(status=TASK_PENDING)
    for task in tasks:
        db.delete_task(task.id)
        
    return redirect("analysis.views.pending")

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def attempt_to_start_analysis(binary):
    print 'starting ana'
    db = Database()           
    tasks = db.list_tasks()
    
    filename = ntpath.basename(binary.file_path)
    output = ntpath.join('/tmp/', filename)                
    
    for task in tasks:
        if task.to_dict()['target'] == output:
            return
        else:
            with open(output, "wb") as handle:
                handle.write(binary.data)                        
                task_id = db.add_path(file_path=output,
                                      package="",
                                      timeout=120,
                                      options="",
                                      priority=1,
                                      machine="",
                                      custom="",
                                      memory=False,
                                      enforce_timeout=False,
                                      tags=None)
                if not task_id:
                    print 'asd'
                    err = "Failed adding sandbox analysis for %s" % filename                                
                    raise Exception(err)    

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def main():
    """main function for standalone usage"""
    usage = "usage: %prog [options]"
    parser = OptionParser(usage=usage)
    parser.add_option('-a', '--archive-toplevel-dir', default='/mnt/cuckoo_archive',
                      help='Archive top-level directory [default: %default]')
    parser.add_option('-m', '--local-machine-dir', default=socket.gethostname(),
                      help='Machine-specific directory [default: $HOST]')

    (options, args) = parser.parse_args()

    if len(args) != 0:
        parser.print_help()
        return 2

    # do stuff
    archive_dir = os.path.join(options.archive_toplevel_dir, options.local_machine_dir)
    try:
        os.mkdir(archive_dir)
    except OSError: # already exists
        pass

    db = Database()

    for task in db.list_tasks(status=TASK_REPORTED):
        task_path_src = _analysis_dir(task.id)

        if not os.path.islink(task_path_src):
            task_path_dst = os.path.join(archive_dir, str(task.id))
            move(task_path_src, task_path_dst)
            os.symlink(task_path_dst, task_path_src)
            print(bold(green('Successfully')) + ' archived %s' % task_path_dst)

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def autoprocess(parallel=1):
    maxcount = cfg.cuckoo.max_analysis_count
    count = 0
    db = Database()
    pool = multiprocessing.Pool(parallel)
    pending_results = []

    # CAUTION - big ugly loop ahead.
    while count < maxcount or not maxcount:

        # Pending_results maintenance.
        for ar, tid, target, copy_path in list(pending_results):
            if ar.ready():
                if ar.successful():
                    log.info("Task #%d: reports generation completed", tid)
                else:
                    try:
                        ar.get()
                    except:
                        log.exception("Exception when processing task ID %u.", tid)
                        db.set_status(tid, TASK_FAILED_PROCESSING)

                pending_results.remove((ar, tid, target, copy_path))

        # If still full, don't add more (necessary despite pool).
        if len(pending_results) >= parallel:
            time.sleep(1)
            continue

        # If we're here, getting parallel tasks should at least
        # have one we don't know.
        tasks = db.list_tasks(status=TASK_COMPLETED, limit=parallel,
                              order_by="completed_on asc")

        # For loop to add only one, nice.
        for task in tasks:
            # Not-so-efficient lock.
            if task.id in [tid for ar, tid, target, copy_path
                           in pending_results]:
                continue

            log.info("Processing analysis data for Task #%d", task.id)

            sample = db.view_sample(task.sample_id)

            copy_path = os.path.join(CUCKOO_ROOT, "storage",
                                     "binaries", sample.sha256)

            args = task.id, task.target, copy_path
            kwargs = dict(report=True, auto=True)
            result = pool.apply_async(process, args, kwargs)

            pending_results.append((result, task.id, task.target, copy_path))

            count += 1
            break

        # If there wasn't anything to add, sleep tight.
        if not tasks:
            time.sleep(5)

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def init_tasks():
    """Check tasks and reschedule uncompleted ones."""
    db = Database()
    cfg = Config()

    log.debug("Checking for locked tasks..")

    if cfg.cuckoo.reschedule:
        tasks = db.list_tasks(status=TASK_RUNNING)

        for task in tasks:
            db.reschedule(task.id)
            log.info("Rescheduled task with ID {0} and "
                     "target {1}".format(task.id, task.target))
    else:
        for task in db.list_tasks(status=TASK_RUNNING):
            db.set_status(task.id, TASK_FAILED_ANALYSIS)
            log.info("Set task #%d to failed analysis", task.id)

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def pending(request):
    db = Database()
    tasks = db.list_tasks(status=TASK_PENDING)

    pending = []
    for task in tasks:
        pending.append(task.to_dict())

    return render_to_response("analysis/pending.html", {"tasks": pending}, context_instance=RequestContext(request))

# 需要导入模块: from lib.cuckoo.core.database import Database [as 别名]
# 或者: from lib.cuckoo.core.database.Database import list_tasks [as 别名]
def pending(request):
    db = Database()
    tasks = db.list_tasks(status=TASK_PENDING)

    pending = []
    for task in tasks:
        pending.append(task.to_dict())

    return render(request, "analysis/pending.html", {"tasks": pending})