Golang JobConfigurationQuery.AllowLargeResults方法代码示例

func (q *QueryConfig) populateJobQueryConfig(conf *bq.JobConfigurationQuery) error {
	conf.Query = q.Q

	if len(q.TableDefinitions) > 0 {
		conf.TableDefinitions = make(map[string]bq.ExternalDataConfiguration)
	}
	for name, data := range q.TableDefinitions {
		conf.TableDefinitions[name] = data.externalDataConfig()
	}

	if q.DefaultProjectID != "" || q.DefaultDatasetID != "" {
		conf.DefaultDataset = &bq.DatasetReference{
			DatasetId: q.DefaultDatasetID,
			ProjectId: q.DefaultProjectID,
		}
	}

	if tier := int64(q.MaxBillingTier); tier > 0 {
		conf.MaximumBillingTier = &tier
	}
	conf.CreateDisposition = string(q.CreateDisposition)
	conf.WriteDisposition = string(q.WriteDisposition)
	conf.AllowLargeResults = q.AllowLargeResults
	conf.Priority = string(q.Priority)

	f := false
	if q.DisableQueryCache {
		conf.UseQueryCache = &f
	}
	if q.DisableFlattenedResults {
		conf.FlattenResults = &f
		// DisableFlattenResults implies AllowLargeResults.
		conf.AllowLargeResults = true
	}
	if q.MaxBytesBilled >= 1 {
		conf.MaximumBytesBilled = q.MaxBytesBilled
	}
	if q.UseStandardSQL || len(q.Parameters) > 0 {
		conf.UseLegacySql = false
		conf.ForceSendFields = append(conf.ForceSendFields, "UseLegacySql")
	}

	if q.Dst != nil && !q.Dst.implicitTable() {
		conf.DestinationTable = q.Dst.tableRefProto()
	}
	for _, p := range q.Parameters {
		qp, err := p.toRaw()
		if err != nil {
			return err
		}
		conf.QueryParameters = append(conf.QueryParameters, qp)
	}
	return nil
}

func (opt allowLargeResults) customizeQuery(conf *bq.JobConfigurationQuery) {
	conf.AllowLargeResults = true
}

func (opt allowLargeResults) customizeQuery(conf *bq.JobConfigurationQuery, projectID string) {
	conf.AllowLargeResults = true
}

// largeDataPagedQuery builds a job and inserts it into the job queue allowing the flexibility to set the custom AllowLargeResults flag for the job
func (c *Client) largeDataPagedQuery(service *bigquery.Service, pageSize int, dataset, project, queryStr string, dataChan chan Data) ([][]interface{}, []string, error) {
	c.printDebug("largeDataPagedQuery starting")
	ts := time.Now()
	// start query
	tableRef := bigquery.TableReference{DatasetId: dataset, ProjectId: project, TableId: c.tempTableName}
	jobConfigQuery := bigquery.JobConfigurationQuery{}

	datasetRef := &bigquery.DatasetReference{
		DatasetId: dataset,
		ProjectId: project,
	}

	jobConfigQuery.AllowLargeResults = true
	jobConfigQuery.Query = queryStr
	jobConfigQuery.DestinationTable = &tableRef
	jobConfigQuery.DefaultDataset = datasetRef
	if !c.flattenResults {
		c.printDebug("setting FlattenResults to false")
		// need a pointer to bool
		f := false
		jobConfigQuery.FlattenResults = &f
	}
	jobConfigQuery.WriteDisposition = "WRITE_TRUNCATE"
	jobConfigQuery.CreateDisposition = "CREATE_IF_NEEDED"

	jobConfig := bigquery.JobConfiguration{}

	jobConfig.Query = &jobConfigQuery

	job := bigquery.Job{}
	job.Configuration = &jobConfig

	jobInsert := service.Jobs.Insert(project, &job)
	runningJob, jerr := jobInsert.Do()

	if jerr != nil {
		c.printDebug("Error inserting job!", jerr)
		if dataChan != nil {
			dataChan <- Data{Err: jerr}
		}
		return nil, nil, jerr
	}

	qr, err := service.Jobs.GetQueryResults(project, runningJob.JobReference.JobId).Do()

	if err != nil {
		c.printDebug("Error loading query: ", err)
		if dataChan != nil {
			dataChan <- Data{Err: err}
		}
		return nil, nil, err
	}

	var headers []string
	rows := [][]interface{}{}

	// if query is completed process, otherwise begin checking for results
	if qr.JobComplete {
		c.printDebug("job complete, got rows", len(qr.Rows))
		headers, rows = c.headersAndRows(qr.Schema, qr.Rows)
		if dataChan != nil {
			dataChan <- Data{Headers: headers, Rows: rows}
		}
	}

	if !qr.JobComplete {
		resultChan := make(chan [][]interface{})
		headersChan := make(chan []string)

		go c.pageOverJob(len(rows), runningJob.JobReference, qr.PageToken, resultChan, headersChan)

	L:
		for {
			select {
			case h, ok := <-headersChan:
				if ok {
					c.printDebug("got headers")
					headers = h
				}
			case newRows, ok := <-resultChan:
				if !ok {
					break L
				}
				if dataChan != nil {
					c.printDebug("got rows", len(newRows))
					dataChan <- Data{Headers: headers, Rows: newRows}
				} else {
					rows = append(rows, newRows...)
				}
			}
		}
	}

	if dataChan != nil {
		close(dataChan)
	}
	c.printDebug("largeDataPagedQuery completed in ", time.Now().Sub(ts).Seconds(), "s")
	return rows, headers, nil
}