Python File.get_or_create方法代码示例

# 需要导入模块: from models import File [as 别名]
# 或者: from models.File import get_or_create [as 别名]
def ingest(filepath):
    '''Ingest file into database'''

    print "Ingesting %s" % filepath
    rows_in_file = parse_fec_file(filepath)
    myfile = File.get_or_create(name=filepath)
    myfile_id = myfile.id

    with db.transaction(): # TODO: More sane error handling
        for idx in range(0, len(rows_in_file), 500):

            # Ingest 500 rows at a time
            print "Inserting row %d of %s" % (idx, filepath)
            rows_subset = rows_in_file[idx:idx+500]
            rows_to_insert = []

            for row in rows_subset:
                unsaved_new_contribution = Contribution(**row_to_dict(row))
                import pdb; pdb.set_trace()
                # If the row isn't already there, insert it
                if :
                    pass
                # If the row is there, check for modifications
                elif:
                    # If it has not been modified, simply add a ContributionHistory object
                    if:
                    # If it has been modified, create a new object and give the new object a contribution history
                    else:
                        pass

            Contribution.insert_many(rows_subset).execute()

# 需要导入模块: from models import File [as 别名]
# 或者: from models.File import get_or_create [as 别名]
def ingest(filepath):
    '''Ingest file into database'''
    print "Ingesting %s" % filepath
    rows = parse_fec_file(filepath)
    # check history table to see if this file is done

    with db.transaction():
        for idx, row in enumerate(rows):
            print "Checking row %d of %d from %s" % (idx, len(rows), filepath)

            try:
                contribution_in_db = Contribution.get(cycle=row['cycle'], sub_id=row['sub_id'])
            except Contribution.DoesNotExist:
                contribution_in_db = None

            # If the row isn't already there, insert it
            if not contribution_in_db:
                print t.cyan("\tInserting new row %d of %s" % (idx, filepath))
                new_contribution = Contribution.create(**row)
                ContributionHistory.create(contribution=new_contribution.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id'])

            # If the row is there, check for modifications
            else:
                # If it has not been modified, simply add a ContributionHistory object
                contribution_in_db_dict = get_dictionary_from_model(contribution_in_db)

                # x = {k:v for k,v in contribution_in_db_dict.iteritems() if k not in ["date", "id"]}
                # y = {k:v for k,v in row.iteritems() if k != "date"}

                if {k:v for k,v in contribution_in_db_dict.iteritems() if k not in ["date", "id"]} == {k:v for k,v in row.iteritems() if k != "date"}:
                    print t.white("\tNo changes found in row %d of %s" % (idx, filepath))
                    ContributionHistory.create(contribution=contribution_in_db.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id'])
                # If it has been modified, create a new object and give the new object a contribution history
                else:
                    print t.magenta("\tDetected change in row %d of %s" % (idx, filepath))
                    # print diff(x,y)

                    # import pdb; pdb.set_trace()

                    ContributionChanges.create(contribution=contribution_in_db.id, **{k:v for k,v in contribution_in_db_dict.iteritems() if k != "id"})

                    for k,v in row.iteritems():
                        if v != getattr(contribution_in_db, k):
                            setattr(contribution_in_db, k, v)

                    contribution_in_db.save()

                    ContributionHistory.create(contribution=contribution_in_db.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id'])

        myfile, _ = File.get_or_create(
                name = os.path.basename(filepath),
                years=next(re.finditer(r'\d{4}_\d{4}', os.path.basename(filepath))),
                sha1 = sha1OfFile(filepath),
                updated = dateparse(os.path.dirname(filepath).split("/")[-1].replace("downloaded_", "").replace("_", "-")).date(),
                ingested = True
                )

# 需要导入模块: from models import File [as 别名]
# 或者: from models.File import get_or_create [as 别名]
def seed_from(filepath):
    '''Ingest file into sqlite database'''

    print "Ingesting %s" % filepath
    rows = parse_fec_file(filepath)

    with db.transaction():
        myfile, _ = File.get_or_create(
                    name = os.path.basename(filepath),
                    years=next(re.finditer(r'\d{4}_\d{4}', os.path.basename(filepath))),
                    sha1 = sha1OfFile(filepath),
                    updated = dateparse(os.path.dirname(filepath).split("/")[-1].replace("downloaded_", "").replace("_", "-")).date(),
                    ingested = False
                    )

        for idx in range(0, len(rows), 500):
            print "Inserting row %d of %d from %s" % (idx, len(rows), filepath)
            rows_subset = rows[idx:idx+500]
            Contribution.insert_many(rows_subset).execute()

        myfile.update(ingested=True)