Golang wcg.NewLogger函数代码示例

func main() {
	app := cli.NewApp()
	app.Name = "speedland-agent"
	app.Usage = "A agent command for speedland.net"
	app.Version = "1.0.0"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			"config, c",
			"wcg.ini",
			"configuration file",
			"WCG_INI_FILE",
		},
	}
	app.Before = func(c *cli.Context) error {
		wcg.ConfigureProcess(c.String("config"))
		// normalize path
		lib.Config.Endpoint.Path = ""
		wcg.NewLogger(nil).Info("Used configurn file: %q", c.String("config"))
		wcg.NewLogger(nil).Info("Target Endpoint: %q", lib.Config.Endpoint)
		wcg.NewLogger(nil).Debug("Token: %q", lib.Config.Token)
		return nil
	}
	app.Commands = commands.AllCommands()
	app.Run(os.Args)
	wcg.WaitLogs()
}

func TestDelChannelApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		// prepare
		d := NewTvChannelDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		ent1 := &tv.TvChannel{"c1", "s1", "foo", "bar"}
		ent2 := &tv.TvChannel{"c2", "s2", "hoge", "piyo"}
		d.Put(d.NewKey(ent1.Key(), 0, nil), ent1)
		d.Put(d.NewKey(ent2.Key(), 0, nil), ent2)

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvChannel entities has been stored within a timeout window.")
		p := app.Api.Path("/channels/c1/s1.json")
		req := ts.Delete(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)

		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "DELETE %s Confirm TvChannel entities has been deleted via API within a timeout window.", p)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "DELETE %s should invalidate the cache", p)

	})
}

func TestAddKeywordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		p := app.Api.Path("/keywords/")
		req := ts.PostForm(p, url.Values{
			"keyword":  []string{"モーニング娘。'15"},
			"category": []string{"モーニング娘。"},
			"scope":    []string{"1"},
		})
		lib.SetApiTokenForTest(req, lib.Admin)
		var got map[string]interface{}
		res := req.RouteTo(app.Routes())
		res.Json(&got)
		assert.HttpStatus(201, res)
		assert.EqStr(
			"http://localhost:8080/api/pt/keywords/モーニング娘。'15.json",
			got["location"].(string),
			"POST %s location",
			p,
		)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "POST %s should invalidate the cache", p)

	})
}

func TestChannel_AddAndDelCrawlerConfig(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := wcg.NewAssert(t)

		// prepare
		d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		d.Add(&tv.CrawlerConfig{
			Keyword:  "キーワード1",
			Category: "カテゴリー1",
			Scope:    1,
		})
		d.Add(&tv.CrawlerConfig{
			Keyword:  "キーワード2",
			Category: "カテゴリー2",
			Scope:    1,
		})

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")

		d.Delete("キーワード1")
		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm CrawlerConfig entities has been deleted within a timeout window.")

		err = d.Delete("Not Exists")
		assert.Nil(err, "Delete should not return even trying to delete the unexising keyword.")
	})
}

func TestAddChannelApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		p := app.Api.Path("/channels/")
		req := ts.PostForm(p, url.Values{
			"cid":             []string{"c1"},
			"sid":             []string{"s1"},
			"name":            []string{"foo"},
			"iepg_station_id": []string{"bar"},
		})
		lib.SetApiTokenForTest(req, lib.Admin)
		var got map[string]interface{}
		res := req.RouteTo(app.Routes())
		res.Json(&got)
		assert.HttpStatus(201, res)
		assert.EqStr(
			"http://localhost:8080/api/pt/channels/c1/s1.json",
			got["location"].(string),
			"POST %s location",
			p,
		)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_CHANNELS), "POST %s should invalidate the cache", p)

	})
}

func NewAppContextFromTestServer(app *lib.App, ts *test.TestServer) *lib.AppContext {
	return &lib.AppContext{
		app,
		ts.Context,
		wcg.NewLogger(nil),
	}
}

func TestGetOverlaps(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		var found []*tv.TvRecord
		const timeForm = "2006/01/02 15:04:05"
		assert := wcg.NewAssert(t)
		d := NewTvRecordDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		base := genTestRecord()
		base.StartAt, _ = time.Parse(timeForm, "2014/01/01 12:00:00")
		base.EndAt, _ = time.Parse(timeForm, "2014/01/01 13:00:00")
		d.Save(base)
		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvRecord entities has been stored within a timeout window.")

		// case1: no overlaps
		r := genTestRecord()
		r.StartAt, _ = time.Parse(timeForm, "2014/01/01 10:50:00")
		r.EndAt, _ = time.Parse(timeForm, "2014/01/01 11:10:00")
		found, err = d.GetOverlaps(r)
		assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case1)")
		assert.EqInt(0, len(found), "TvRecordDriver#GetOverlaps should return empty array (case1)")

		// case 2: base.end < r.start
		r = genTestRecord()
		r.StartAt, _ = time.Parse(timeForm, "2014/01/01 13:50:00")
		r.EndAt, _ = time.Parse(timeForm, "2014/01/01 14:10:00")
		found, err = d.GetOverlaps(r)
		assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case2)")
		assert.EqInt(0, len(found), "TvRecordDriver#GetOverlaps should return empty array (case2)")

		// case 3: base.start < r.start < base.end
		r = genTestRecord()
		r.StartAt, _ = time.Parse(timeForm, "2014/01/01 11:50:00")
		r.EndAt, _ = time.Parse(timeForm, "2014/01/01 12:10:00")
		found, err = d.GetOverlaps(r)
		assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case3)")
		assert.EqInt(1, len(found), "TvRecordDriver#GetOverlaps should return an array with an entity (case3)")

		// case 4: r.start < base.end < r.end
		r = genTestRecord()
		r.StartAt, _ = time.Parse(timeForm, "2014/01/01 12:50:00")
		r.EndAt, _ = time.Parse(timeForm, "2014/01/01 13:10:00")
		found, err = d.GetOverlaps(r)
		assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case4)")
		assert.EqInt(1, len(found), "TvRecordDriver#GetOverlaps should return an array with an entity (case4)")

		// case 5: r.start < base.start < base.end < r.end
		r = genTestRecord()
		r.StartAt, _ = time.Parse(timeForm, "2014/01/01 12:10:00")
		r.EndAt, _ = time.Parse(timeForm, "2014/01/01 12:50:00")
		found, err = d.GetOverlaps(r)
		assert.Nil(err, "TvRecordDriver#GetOverlaps should not return an error (case4)")
		assert.EqInt(1, len(found), "TvRecordDriver#GetOverlaps should return an array with an entity (case4)")
	})
}

func FixtureFromMap(ctx appengine.Context, arr []map[string]interface{}) error {
	logger := wcg.NewLogger(nil)
	for _, v := range arr {
		if err := loadJsonToDatastore(ctx, nil, v, logger); err != nil {
			return err
		}
	}
	return nil
}

func Run() {
	go ping.Run()
	go recorder.Run()
	go statsd.Run()

	// block the main until receiving the signal
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGTERM, syscall.SIGINT)
	s := <-c
	wcg.NewLogger(nil).Info("Exitting the signal with %s", s)
	// TODO: delegate the notification to each job
}

func TestDeleteRecordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		d := NewRecordCacheDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		// create an empty cache
		var got []tv.TvRecord
		p := app.Api.Path("/records/")
		req := ts.Get(p)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		res.Json(&got)
		assert.EqInt(0, len(got), "GET %s should return 2 TvRecord entities.")
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "GET %s should create RecordCacheTypeTvRecord cache", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should create RecordCacheTypeIEpg cache", p)

		// prepare
		in_window := genTestRecord()
		in_window.StartAt = d.today.Add(1 * time.Hour)
		in_window.EndAt = d.today.Add(2 * time.Hour)
		d.TvRecord.Save(in_window)

		future := genTestRecord()
		future.StartAt = d.today.Add(RECORD_TIME_WINDOW + 1*time.Hour)
		future.EndAt = future.StartAt.Add(1 * time.Hour)
		d.TvRecord.Save(future)

		err := util.WaitFor(func() bool {
			records, _ := d.TvRecord.NewQuery().Count()
			return records == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvRecord entities has been stored within a timeout window.")

		// case 1: future
		p = fmt.Sprintf("%s%s.json", app.Api.Path("/records/"), future.Key())
		req = ts.Delete(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res = req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should not invalidate RecordCacheTypeTvRecord cache (future record)", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (future record)", p)

		// case 2: in recording window
		p = fmt.Sprintf("%s%s.json", app.Api.Path("/records/"), in_window.Key())
		req = ts.Delete(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res = req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should invalidate RecordCacheTypeTvRecord cache (in_window record)", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (in_window record)", p)
	})
}

func TestDelKeywordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		// prepare
		d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		d.Add(&tv.CrawlerConfig{
			Keyword:  "モーニング娘。'15",
			Category: "モーニング娘",
			Scope:    tv.FEED_SCOPE_ALL,
		})
		d.Add(&tv.CrawlerConfig{
			Keyword:  "SPEED",
			Category: "SPEED",
			Scope:    tv.FEED_SCOPE_ALL,
		})

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")

		p := app.Api.Path("/keywords/モーニング娘。'15.json")
		req := ts.Delete(p)
		lib.SetApiTokenForTest(req, lib.Admin)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)

		err = util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 1
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "DELETE %s Confirm CrawlerConfig entities has been deleted via API within a timeout window.", p)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(!mc.Exists(MC_KEY_KEYWORDS), "DELETE %s should invalidate the cache", p)
	})
}

func TestRecordCache_GetRecords_and_Invalidate(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := wcg.NewAssert(t)
		d := NewRecordCacheDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		// prepare
		in_window := genTestRecord()
		in_window.StartAt = d.today.Add(1 * time.Hour)
		in_window.EndAt = d.today.Add(2 * time.Hour)
		d.TvRecord.Save(in_window)

		future := genTestRecord()
		future.StartAt = d.today.Add(RECORD_TIME_WINDOW + 1*time.Hour)
		future.EndAt = future.StartAt.Add(1 * time.Hour)
		d.TvRecord.Save(future)

		past := genTestRecord()
		past.StartAt = d.today.Add(-RECORD_TIME_WINDOW - 1*time.Hour)
		past.EndAt = past.StartAt.Add(30 * time.Minute)
		d.TvRecord.Save(past)

		in_window_iepg := genTestIEpg()
		in_window.StartAt = d.today.Add(4 * time.Hour)
		in_window.EndAt = d.today.Add(5 * time.Hour)
		d.IEpg.Save(in_window_iepg)

		err := util.WaitFor(func() bool {
			records, _ := d.TvRecord.NewQuery().Count()
			iepgs, _ := d.IEpg.NewQuery().Count()
			return records+iepgs == 4
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm TvRecord/IEpg entities has been stored within a timeout window.")

		list, err := d.GetRecords(false)
		assert.Nil(err, "RecordCache#GetRecords should not return an error")
		assert.EqInt(2, len(list), "RecordCache#GetRecords should return 2 records in a time window")

		// cache check
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "RecordCache#GetRecords should create RecordCacheTypeTvRecord on memcache.")
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "RecordCache#GetRecords should create RecordCacheTypeIEpg on memcache.")

		// invalidate the keys
		assert.Ok(!d.Invalidate(future), "RecordCache#Invalidate should not return true if the passed record is out of window (future)")
		assert.Ok(!d.Invalidate(past), "RecordCache#Invalidate should not return true if the passed record is out of window (future)")
		assert.Ok(d.Invalidate(in_window), "RecordCache#Invalidate should return true if the passed record is in window")
		assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "RecordCache#GetRecords should invalidate RecordCacheTypeTvRecord on memcache.")

		assert.Ok(d.Invalidate(in_window_iepg), "RecordCache#Invalidate should return true if the passed record is in window (iepg)")
		assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "RecordCache#GetRecords should invalidate RecordCacheTypeIEpg on memcache.")
	})
}

func TestListKeywordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)

		// prepare
		d := NewCrawlerConfigDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))
		d.Add(&tv.CrawlerConfig{
			Keyword:  "キーワード1",
			Category: "カテゴリー1",
			Scope:    1,
		})
		d.Add(&tv.CrawlerConfig{
			Keyword:  "キーワード2",
			Category: "カテゴリー2",
			Scope:    1,
		})

		err := util.WaitFor(func() bool {
			c, _ := d.NewQuery().Count()
			return c == 2
		}, util.DefaultWaitForTimeout)
		assert.Nil(err, "Confirm CrawlerConfig entities has been stored within a timeout window.")

		var got []*tv.CrawlerConfig
		p := app.Api.Path("/keywords/") + "?force=true"
		req := ts.Get(p)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		res.Json(&got)
		assert.EqStr("キーワード2", got[0].Keyword, "GET %s should return the list ordered by creation time.", p)
		assert.EqStr("キーワード1", got[1].Keyword, "GET %s should return the list ordered by creation time.", p)

		// Confirm cache invalidation
		mc := memcache.NewDriver(ts.Context, wcg.NewLogger(nil))
		assert.Ok(mc.Exists(MC_KEY_KEYWORDS), "GET %s should create the cache", p)
	})
}

func TestCreateRecordApi(t *testing.T) {
	test.RunTestServer(func(ts *test.TestServer) {
		assert := test.NewAssert(t)
		d := NewRecordCacheDriver(TEST_APP_KEY, ts.Context, wcg.NewLogger(nil))

		// create an empty cache
		var got []tv.TvRecord
		p := app.Api.Path("/records/")
		req := ts.Get(p)
		res := req.RouteTo(app.Routes())
		assert.HttpStatus(200, res)
		res.Json(&got)
		assert.EqInt(0, len(got), "GET %s should return 2 TvRecord entities.")
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "GET %s should create RecordCacheTypeTvRecord cache", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should create RecordCacheTypeIEpg cache", p)

		now := time.Now()

		// case 1: future
		req = ts.PostForm(p, url.Values{
			"title":    []string{"Title"},
			"category": []string{"Category"},
			"cid":      []string{"27"},
			"sid":      []string{"hd"},
			"start_at": []string{util.FormatDateTime(now.Add(RECORD_TIME_WINDOW + 24*time.Hour))},
			"end_at":   []string{util.FormatDateTime(now.Add(RECORD_TIME_WINDOW + 25*time.Hour))},
		})
		lib.SetApiTokenForTest(req, lib.Admin)
		res = req.RouteTo(app.Routes())
		assert.HttpStatus(201, res)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should not invalidate RecordCacheTypeTvRecord cache (future record)", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (future record)", p)

		// case 2: in recording window
		req = ts.PostForm(p, url.Values{
			"title":    []string{"Title"},
			"category": []string{"Category"},
			"cid":      []string{"27"},
			"sid":      []string{"hd"},
			"start_at": []string{util.FormatDateTime(now)},
			"end_at":   []string{util.FormatDateTime(now.Add(1 * time.Hour))},
		})
		lib.SetApiTokenForTest(req, lib.Admin)
		res = req.RouteTo(app.Routes())
		assert.HttpStatus(201, res)
		assert.Ok(!d.Cache.Exists(d.GetMcKey(RecordCacheTypeTvRecord)), "POST %s should invalidate RecordCacheTypeTvRecord cache (in window)", p)
		assert.Ok(d.Cache.Exists(d.GetMcKey(RecordCacheTypeIEpg)), "GET %s should not invalidate RecordCacheTypeIEpg cache (in window)", p)
	})
}

func DatastoreFixture(ctx appengine.Context, path string, bindings interface{}) error {
	logger := wcg.NewLogger(nil)
	logger.Debug("[Fixture] load from %s", path)
	data, err := loadFile(path, bindings)
	if err != nil {
		return err
	}
	var arr []map[string]interface{}
	if err = json.Unmarshal(data, &arr); err != nil {
		return fmt.Errorf("Could not load the json file from %q - JSON Parse error: %v", path, err)
	}
	for _, v := range arr {
		if err := loadJsonToDatastore(ctx, nil, v, logger); err != nil {
			return err
		}
	}
	return nil
}