Golang v2.Job类代码示例

// setJobRef initializes job's JobReference if given a non-empty jobID.
// projectID must be non-empty.
func setJobRef(job *bq.Job, jobID, projectID string) {
	if jobID == "" {
		return
	}
	// We don't check whether projectID is empty; the server will return an
	// error when it encounters the resulting JobReference.

	job.JobReference = &bq.JobReference{
		JobId:     jobID,
		ProjectId: projectID,
	}
}

func (opt jobID) customizeJob(job *bq.Job, projectID string) {
	job.JobReference = &bq.JobReference{
		JobId:     string(opt),
		ProjectId: projectID,
	}
}

func main() {
	flag.Parse()
	// setEnvVars()

	client, err := GoogleClient()

	if err == nil {
		bq, _ := bigquery.New(client)
		dsr := new(bigquery.DatasetReference)
		dsr.DatasetId = *bqSourceDataset
		dsr.ProjectId = *bqSourceProject

		// request := new(bigquery.QueryRequest)
		// request.DefaultDataset = dsr
		// request.Query = "SELECT count(*) FROM []"

		// call := bq.Jobs.Query("", request)

		// resp, err := call.Do()

		// jobs := new(bigquery.JobsService)
		// job := jobs.Query("sapient-catbird-547", request)
		// resp, err := job.Do()
		// fmt.Print(resp.CacheHit, resp.JobReference, err)

		// jobId := resp.JobReference.JobId

		// s, _ := bq.Jobs.GetQueryResults("", jobId).Do()

		// buf, _ := json.Marshal(s)
		// fmt.Println(s, string(buf), "\n\n\n")

		tabr := new(bigquery.TableReference)
		tabr.DatasetId = *bqDestDataset
		tabr.ProjectId = *bqDestProject
		tabr.TableId = "temp_grouped_v2"

		// jcq := new(bigquery.JobConfigurationQuery)
		// jcq.DestinationTable = tabr
		// jcq.Priority = "BATCH"
		// jcq.WriteDisposition = "WRITE_TRUNCATE"
		// jcq.Query = "SELECT ap_mac, COUNT(DISTINCT(client_mac)), DATE(TIMESTAMP(first_seen)) date FROM [dev_sense_v1.sensev4_ct] GROUP BY ap_mac, date"

		// jc := new(bigquery.JobConfiguration)
		// jc.Query = jcq

		// job := new(bigquery.Job)
		// job.Configuration = jc

		// aa, err := bq.Jobs.Insert(*bqSourceProject, job).Do()
		// if err == nil {
		// 	fmt.Print(aa.Id)
		// } else {
		// 	fmt.Print(err)
		// }

		jce := new(bigquery.JobConfigurationExtract)
		jce.DestinationFormat = "csv"
		jce.DestinationUri = "gs://ct_temp/151028.csv"
		jce.SourceTable = tabr

		extractJc := new(bigquery.JobConfiguration)
		extractJc.Extract = jce
		extractJob := new(bigquery.Job)
		extractJob.Configuration = extractJc

		aa, err := bq.Jobs.Insert(*bqSourceProject, extractJob).Do()
		if err == nil {
			fmt.Print(aa.Id)
		} else {
			fmt.Print(err)
		}

	}

}

// largeDataPagedQuery builds a job and inserts it into the job queue allowing the flexibility to set the custom AllowLargeResults flag for the job
func (c *Client) largeDataPagedQuery(service *bigquery.Service, pageSize int, dataset, project, queryStr string, dataChan chan Data) ([][]interface{}, []string, error) {
	c.printDebug("largeDataPagedQuery starting")
	ts := time.Now()
	// start query
	tableRef := bigquery.TableReference{DatasetId: dataset, ProjectId: project, TableId: c.tempTableName}
	jobConfigQuery := bigquery.JobConfigurationQuery{}

	datasetRef := &bigquery.DatasetReference{
		DatasetId: dataset,
		ProjectId: project,
	}

	jobConfigQuery.AllowLargeResults = true
	jobConfigQuery.Query = queryStr
	jobConfigQuery.DestinationTable = &tableRef
	jobConfigQuery.DefaultDataset = datasetRef
	if !c.flattenResults {
		c.printDebug("setting FlattenResults to false")
		// need a pointer to bool
		f := false
		jobConfigQuery.FlattenResults = &f
	}
	jobConfigQuery.WriteDisposition = "WRITE_TRUNCATE"
	jobConfigQuery.CreateDisposition = "CREATE_IF_NEEDED"

	jobConfig := bigquery.JobConfiguration{}

	jobConfig.Query = &jobConfigQuery

	job := bigquery.Job{}
	job.Configuration = &jobConfig

	jobInsert := service.Jobs.Insert(project, &job)
	runningJob, jerr := jobInsert.Do()

	if jerr != nil {
		c.printDebug("Error inserting job!", jerr)
		if dataChan != nil {
			dataChan <- Data{Err: jerr}
		}
		return nil, nil, jerr
	}

	qr, err := service.Jobs.GetQueryResults(project, runningJob.JobReference.JobId).Do()

	if err != nil {
		c.printDebug("Error loading query: ", err)
		if dataChan != nil {
			dataChan <- Data{Err: err}
		}
		return nil, nil, err
	}

	var headers []string
	rows := [][]interface{}{}

	// if query is completed process, otherwise begin checking for results
	if qr.JobComplete {
		c.printDebug("job complete, got rows", len(qr.Rows))
		headers, rows = c.headersAndRows(qr.Schema, qr.Rows)
		if dataChan != nil {
			dataChan <- Data{Headers: headers, Rows: rows}
		}
	}

	if !qr.JobComplete {
		resultChan := make(chan [][]interface{})
		headersChan := make(chan []string)

		go c.pageOverJob(len(rows), runningJob.JobReference, qr.PageToken, resultChan, headersChan)

	L:
		for {
			select {
			case h, ok := <-headersChan:
				if ok {
					c.printDebug("got headers")
					headers = h
				}
			case newRows, ok := <-resultChan:
				if !ok {
					break L
				}
				if dataChan != nil {
					c.printDebug("got rows", len(newRows))
					dataChan <- Data{Headers: headers, Rows: newRows}
				} else {
					rows = append(rows, newRows...)
				}
			}
		}
	}

	if dataChan != nil {
		close(dataChan)
	}
	c.printDebug("largeDataPagedQuery completed in ", time.Now().Sub(ts).Seconds(), "s")
	return rows, headers, nil
}