Python Salesforce.query_more方法代码示例

# 需要导入模块: from simple_salesforce import Salesforce [as 别名]
# 或者: from simple_salesforce.Salesforce import query_more [as 别名]
with open("failures.log", "w") as fail_log:

    sf = Salesforce(username=username, password=password, security_token=security_token, sandbox=False)

    result = sf.query(soql % (comma_separated_column_names, table))
    done = False
    while not done:
        for r in result["records"]:
            line_number = line_number + 1
            # Here's where we actually print...
            try:
                row = [dig_out_value(c, r, table) for c in columns]
                writer.writerow(row)
                printed_lines = printed_lines + 1
            except TypeError:
                fail_log.write("# %d\n" % line_number)
                fail_log.write(str(r))
                fail_log.write("\n")
                skipped_lines = skipped_lines + 1
        if result['done']:
            done = True
        else:
            result = sf.query_more(result['nextRecordsUrl'], identifier_is_url=True)

if skipped_lines > 0:
    sys.stderr.write("WARNING: Skipped %d lines.\n" % skipped_lines)
    sys.exit(0)
else:
    sys.stderr.write("OK: wrote all lines successfully.\n")
    sys.exit(0)

# 需要导入模块: from simple_salesforce import Salesforce [as 别名]
# 或者: from simple_salesforce.Salesforce import query_more [as 别名]
class SF2MySQLCopy:

    def __init__(self):
        v = mysql.connector.version.VERSION
        if v[0] < 2 or v[1] < 1 or v[2] < 3:
            raise Exception('Minimum required MySQL driver version is 2.1.3')
        self.salesforce = Salesforce(username=sf_credentials.SF_USER, password=sf_credentials.SF_PASSWD, security_token=sf_credentials.SF_SEC_TOKEN, sandbox=False, sf_version='34.0')
        self.salesforce.headers['Accept-Encoding'] = 'gzip'
        # https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/headers_queryoptions.htm
        self.salesforce.headers['Sforce-Query-Options'] = 'batchSize=2000'
        self.mysql_connection = mysql.connector.connect(user='root', password='root',
              host=mysql_config.MYSQL_HOST,
              port=mysql_config.MYSQL_PORT,
              database=mysql_config.MYSQL_SCHEMA,
              charset=mysql_config.MYSQL_CHARSET,
              use_unicode=True)

    def copy_sobjects(self, sobjects_names):
        for sobject in sobjects_names:
            self._actually_copy_sobject(sobject);

    def copy_sobject(self, sobject_name):
        self._actually_copy_sobject(self.mysql_connection, sobject_name)

    def _actually_copy_sobject(self, mysql_connection, sobject_name):
        # Step 1: Obtain SObject's metadata and instantiate helper objects
        stype = self.salesforce.__getattr__(sobject_name)
        print 'Retrieving metadata for SObject ' + sobject_name + '...'
        sobject_desc = stype.describe()
        sobject_fields_map = self._sobject_fields_map(sobject_desc['fields'])
        table_creator = MySQLTableCreator(sobject_desc)
        soql_helper = SOQLHelper(sobject_desc)
        cursor = mysql_connection.cursor()
        # Step 2: Drop table if table already exists
        drop_table_statement = table_creator.generate_drop_table()
        try:
            cursor.execute(drop_table_statement)
            print 'Dropped MySQL table: ' + table_creator.mysql_table_name
        except mysql.connector.Error as err:
            print 'MySQL Error when executing: ' + drop_table_statement
        # Step 3: Create table
        create_table_statement = table_creator.generate_create_table()
        create_index_statements = table_creator.generate_indexes_for_references()
        table_name = table_creator.mysql_table_name
        try:
            cursor.execute(create_table_statement)
            for create_index_statement in create_index_statements:
                cursor.execute(create_index_statement)
            print 'Created MySQL table: ' + table_name
        except mysql.connector.Error as err:
            print 'MySQL Error when executing: ' + create_table_statement
            print 'Skipping SObject copy ...'
            return

        # Step 4: Start to pump data from Salesforce
        soql_query = soql_helper.generate_soql_select()
        print 'Querying SObject ' + sobject_desc['name']
        timer = Timer()
        timer.start()
        results = self.salesforce.query(soql_query)
        total_size = results['totalSize'] # Int
        print 'Salesforce query retrieved ' + str(total_size) + ' records for SObject ' + sobject_desc['name']
        done = False
        paged = False
        page_number = 0
        pages_count = 0
        insert_statement = table_creator.insert_statement()
        while not done:
            done = results['done']
            if not done and not paged:
                paged = True
            sf_records = results['records']
            page_size = len(sf_records)
            if pages_count == 0:
                pages_count = total_size / page_size
                pages_count = int(math.ceil(pages_count))
            if not done:
                print 'Query result is paged'
            if paged:
                page_number = page_number + 1
                print 'Page #' + str(page_number) + ': ' + str(page_size) + ' records'
            mysql_records = []
            for sf_record in sf_records:
                mysql_record = table_creator.sf_record2mysql_record(sf_record)
                mysql_records.append(mysql_record)
            # Step 5: Insert records into MySQL
            print 'Inserting ' + str(len(mysql_records)) + ' records into the ' + table_name + ' table'
            try:
                cursor.executemany(insert_statement, mysql_records)
                mysql_connection.commit()
            except mysql.connector.Error as err:
                print 'MySQL Error when executing: ' + insert_statement
                print 'Aborting ...'
                return
            if not done:
                if 'nextRecordsUrl' in results:
                    next_records = results['nextRecordsUrl']
                    print 'Retrieving page ' + str(page_number + 1) + ' of ' + str(pages_count) + ' ...'
                    print 'ETC: ' + timer.formatted_etc(page_number, pages_count)
                    results = self.salesforce.query_more(next_records, True)
#.........这里部分代码省略.........