Python createJsonDumpStore.createTestSet函数代码示例

  def testTransferMany(self):
    createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],rootDir=self.testMoveFrom)
    oldStorage = JDS.JsonDumpStorage(self.testMoveFrom, **self.initKwargs[0])
    itemNumber = 0
    xmas = DT.datetime(2001,12,25,12,25)
    hasLinks = {}
    for id in createJDS.jsonFileData.keys():
      hasLinks[id] = True
      if 0 == itemNumber %2 :
        oldStorage.markAsSeen(id)
        self.__hasNoLinkOrFail(oldStorage,id)
        hasLinks[id] = False

    opts = ((False,True),(True,False),(False,False)) #copyLinks, makeNewLinks
    targets = (self.testMoveTo, self.testMoveToAlt,self.testDir)
    assert len(opts) == len(targets), "set of opts must be one-to-one with set of targets, or fail"
    for i in range(len(opts)):
      aDate = None
      if opts[i][1]: aDate = xmas
      storage = JDS.JsonDumpStorage(targets[i], **self.initKwargs[0])
      storage.transferMany(createJDS.jsonFileData.keys(),oldStorage,copyLinksBoolean=opts[i][0],makeNewDateLinksBoolean=opts[i][1],aDate=aDate)
      for id in createJDS.jsonFileData.keys():
        try:
          storage.getJson(id)
        except Exception,x:
          print '(%s): %s'%(type(x),x)
          assert False, 'Expected to find a transferred json file for %s' % id
        if opts[i][1] or hasLinks[id]:
          self.__hasLinkOrFail(storage,id)
          if opts[i][1]:
            self.__hasDatePathOrFail(storage,xmas)
        if not opts[i][1] and not hasLinks[id]:
          self.__hasNoLinkOrFail(storage,id)

  def testRemove(self):
    createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir)
    storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2])
    counter = 0
    for uuid in createJDS.jsonFileData.keys():
      if 0 == counter % 3:
        # test that we don't throw for missing links
        storage.markAsSeen(uuid)
      if 1 == counter % 3:
        # test that we don't throw for one missing file
        if 0 == counter % 2:
          os.unlink(storage.getDump(uuid))
        else:
          os.unlink(storage.getJson(uuid))
      if 2 == counter % 3:
        # test that we don't throw for both missing files, but with links
        os.unlink(storage.getJson(uuid))
        os.unlink(storage.getDump(uuid))
      storage.remove(uuid)
      counter += 1
    allfiles = []
    alllinks = []
    for dir, dirs, files in os.walk(self.testDir):
      for file in files:
        allfiles.append(file)
        if os.path.islink(os.path.join(dir,file)):
          alllinks.append(file)
      for d in dirs:
        if os.path.islink(os.path.join(dir,d)):
          alllinks.append(d)

    assert [] == allfiles, 'Expect that all removed files are gone, but found %s' % allfiles
    assert [] == alllinks, 'Expect that all links are gone, but found %s' % alllinks
    assert_raises(JDS.NoSuchUuidFound, storage.remove, "bogusdatax3yymmdd")

 def testDestructiveDateWalkNotNow(self):
   createJDS.createTestSet(self.currenttimes,self.initKwargs[1],self.testDir)
   storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[1])
   seenids = []
   for id in storage.destructiveDateWalk():
     seenids.append(id)
   assert [] == seenids

 def testMove(self):
   createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[3],self.testDir)
   storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[3])
   os.mkdir(self.testMoveTo)
   for uuid in createJDS.jsonFileData.keys():
     storage.move(uuid,os.path.join('.','TEST-MOVETO'))
   allfiles = []
   alllinks = []
   for dir, dirs, files in os.walk(self.testDir):
     for file in files:
       allfiles.append(file)
       if os.path.islink(os.path.join(dir,file)):
         alllinks.append(file)
     for d in dirs:
       if os.path.islink(os.path.join(dir,d)):
         alllinks.append(d)
   assert [] == allfiles, 'Expect that all moved files are gone, but found %s' % allfiles
   assert [] == alllinks, 'Expcet that all links are gone, but found %s' % alllinks
   allfiles = []
   alllinks = []
   expectedFiles = [x+storage.jsonSuffix for x in createJDS.jsonFileData.keys() ]
   expectedFiles.extend([x+storage.dumpSuffix for x in createJDS.jsonFileData.keys() ])
   for dir, dirs, files in os.walk(os.path.join('.','TEST-MOVETO')):
     for file in files:
       allfiles.append(file)
       assert file in expectedFiles, 'Expect that each moved file will be expected but found %s' % file
       if os.path.islink(os.path.join(dir,file)): alllinks.append(file)
     for d in dirs:
       if os.path.islink(os.path.join(dir,d)): alllinks.append(d)
   assert [] == alllinks, 'Expect no links in the move-to directory, but found %s' % alllinks
   for file in expectedFiles:
     assert file in allfiles, 'Expect that every file will be moved but did not find %s' % file

 def testCleanUpCompletedAndFailedJobs_WithSaves(self):
   """
   testCleanUpCompletedAndFailedJobs_WithSaves(self):
   The default config asks for successful and failed jobs to be saved
   """
   global me
   cursor = self.connection.cursor()
   dbtestutil.fillProcessorTable(cursor,4)
   m = monitor.Monitor(me.config)
   createJDS.createTestSet(createJDS.jsonFileData,jsonKwargs={'logger':me.logger},rootDir=me.config.storageRoot)
   runInOtherProcess(m.standardJobAllocationLoop, stopCondition=(lambda : self.jobsAllocated() == 14))
   started = dbtestutil.datetimeNow(cursor)
   self.connection.commit()
   completed = started + dt.timedelta(microseconds=100)
   idTimesAndSuccessSeq = [
     [started,completed,True,1],
     [started,completed,True,3],
     [started,completed,True,5],
     [started,completed,True,11],
     [started,None,False,2],
     [started,None,False,4],
     [started,None,False,8],
     [started,None,False,12],
     ]
   dbCon,dbCur = m.getDatabaseConnectionPair()
   try:
     jobdata = self.setJobSuccess(dbCur,idTimesAndSuccessSeq)
     m.cleanUpCompletedAndFailedJobs()
   finally:
     m.databaseConnectionPool.cleanup()
   successSave = set()
   failSave = set()
   expectSuccessSave = set()
   expectFailSave = set()
   remainBehind = set()
   for dir, dirs, files in os.walk(me.config.storageRoot):
     remainBehind.update(os.path.splitext(x)[0] for x in files)
   for d in idTimesAndSuccessSeq:
     if d[2]:
       expectSuccessSave.add(d[3])
     else:
       expectFailSave.add(d[3])
   for dir,dirs,files in os.walk(me.config.saveSuccessfulMinidumpsTo):
     successSave.update((os.path.splitext(x)[0] for x in files))
   for dir,dirs,files in os.walk(me.config.saveFailedMinidumpsTo):
     failSave.update((os.path.splitext(x)[0] for x in files))
   for x in jobdata:
     if None == x[2]:
       assert not x[1] in failSave and not x[1] in successSave, "if we didn't set success state for %s, then it wasn't copied"%(x[1])
       assert x[1] in remainBehind, "if we didn't set success state for %s, then it should remain behind"%(x[1])
       assert not x[0] in expectFailSave and not x[0] in expectSuccessSave, "database should match expectatations for id=%s"%(x[0])
     elif True == x[2]:
       assert  not x[1] in failSave and x[1] in successSave, "if we set success for %s, it is copied to %s"%(x[1],me.config.saveSussessfulMinidumpsTo)
       assert not x[0] in expectFailSave and x[0] in expectSuccessSave, "database should match expectatations for id=%s"%(x[0])
       assert not x[1] in remainBehind, "if we did set success state for %s, then it should not remain behind"%(x[1])
     elif False == x[2]:
       assert  x[1] in failSave and not x[1] in successSave, "if we set failure for %s, it is copied to %s"%(x[1],me.config.saveFailedMinidumpsTo)
       assert  x[0] in expectFailSave and not x[0] in expectSuccessSave, "database should match expectatations for id=%s"%(x[0])
       assert not x[1] in remainBehind, "if we did set success state for %s, then it should not remain behind"%(x[1])

 def testRemoveBadUuidFromJsonDumpStorage(self):
   """
   testRemoveBadUuidFromJsonDumpStorage(self):
   This just wraps JsonDumpStorage. Assure we aren't futzing up the wrap (fail with non-exist uuid)
   """
   global me
   createJDS.createTestSet(createJDS.jsonFileData,jsonKwargs={'logger':me.logger},rootDir=me.config.storageRoot)
   mon = monitor.Monitor(me.config)
   badUuid = '0bad0bad-0bad-6666-9999-0bad20001025'
   assert_raises(monitor.UuidNotFoundException,mon.removeUuidFromJsonDumpStorage,badUuid)

 def testDestructiveDateWalk(self):
   createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],self.testDir)
   storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[0])
   uuids = createJDS.jsonFileData.keys()
   seenids = []
   for id in storage.destructiveDateWalk():
     assert id in uuids, 'Expect that %s is among the uuids we stored' % uuid
     seenids.append(id)
   for id in uuids:
     assert id in seenids, 'Expect that we found every uuid we stored (%s) from %s' % (id,seenids)
   assert not os.listdir(storage.dateBranch), 'Expect that destructive walk will remove all date links, and their dirs'

 def testRemoveAlsoNames(self):
   """testJsonDumpStorage:TestJsonDumpStorage.testRemoveAlsoNames(self)
   Try to remove them all, and check that they are indeed all gone.
   """
   createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir)
   kwargs = self.initKwargs[2]
   kwargs['cleanIndexDirectories'] = 'True'
   storage = JDS.JsonDumpStorage(self.testDir,**kwargs)
   for uuid,data in createJDS.jsonFileData.items():
     storage.remove(uuid)
   assert not os.listdir(storage.root), 'Expected them all to go, but %s'%(os.listdir(storage.root))

 def testGetDump(self):
   createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[1],self.testDir)
   storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[1])
   for uuid,data in createJDS.jsonFileData.items():
     expected = os.sep.join((storage.nameBranch,data[2],uuid+storage.dumpSuffix))
     got =  storage.getDump(uuid)
     assert expected == got, 'Expected dump file %s, got %s' % (expected,got)
   try:
     storage.getDump(createJDS.jsonBadUuid)
     assert False, 'Should throw IOError from attempt to getDump(non-existent-uuid)'
   except OSError,e:
     assert True

  def testRemoveOlderThan(self):
    createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],self.testDir)
    storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[0])
    cutoff = DT.datetime(2008,12,26,05,0)
    youngkeys = [x for x,d in createJDS.jsonFileData.items() if DT.datetime(*[int(i) for i in d[0].split('-')]) >= cutoff]
    oldkeys = [x for x,d in createJDS.jsonFileData.items() if DT.datetime(*[int(i) for i in d[0].split('-')]) < cutoff]

    for k in youngkeys:
      assert k in createJDS.jsonFileData.keys(),"Expected %s in %s"%(k,createJDS.jsonFileData.keys())
    for k in oldkeys:
      assert k in createJDS.jsonFileData.keys()
    for k in createJDS.jsonFileData.keys():
      assert (k in youngkeys or k in oldkeys)
    storage.removeOlderThan(cutoff)
    seenuuid = {}
    seendirs = []
    for dir,dirs,files in os.walk(storage.nameBranch):
      for f in files:
        if os.path.islink(os.path.join(dir,f)):
          uuid = os.path.splitext(f)[0]
          seenuuid[uuid] = True
          assert uuid in youngkeys, 'File: Expect that each remaining link has a young uuid, got %s' % uuid
          assert not uuid in oldkeys, 'File Expect no remaining link has old uuid, got %s' % uuid
      for d in dirs:
        if os.path.islink(os.path.join(dir,d)):
          uuid = os.path.splitext(d)[0]
          seenuuid[uuid] = True
          assert uuid in youngkeys, 'Dir: Expect that each remaining link has a young uuid, got %s' % uuid
          assert not uuid in oldkeys, 'Dir: Expect no remaining link has old uuid, got %s' % uuid
    for id in oldkeys:
      assert not id in seenuuid,'Expect that no old key is found, but %s' % id
    for id in youngkeys:
      assert id in seenuuid, 'Expect that every new key is found, but %s' % id

    seenuuid = {}
    seendirs = []
    for dir, dirs, files in os.walk(storage.dateBranch):
      for f in files:
        uuid = os.path.splitext(f)[0]
        seenuuid[uuid] = True
        assert uuid in youngkeys, 'Expect that each remaining file has a young uuid, got %s' % uuid
        assert not uuid in oldkeys, 'Expect no remaining file has old uuid, got %s' % uuid
      for d in dirs:
        uuid = os.path.splitext(d)[0]
        if '-' in uuid:
          seenuuid[uuid] = True
          assert uuid in youngkeys, 'Expect that each remaining file has a young uuid, got %s' % uuid
          assert not uuid in oldkeys, 'Expect no remaining file has old uuid, got %s' % uuid
    for id in oldkeys:
      assert not id in seenuuid,'Expect that no old key is found but %s' % id
    for id in youngkeys:
      assert id in seenuuid, 'Expect that every new key is found, but %s' % id
      assert os.path.isdir(os.path.join(storage.dateBranch,createJDS.jsonFileData[id][3]))

 def testRemoveWithBadlyFormattedDateLink(self):
   createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[2],self.testDir)
   storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[2])
   uuid = createJDS.jsonFileData.keys()[0]
   head, json_unused = os.path.split(storage.getJson(uuid))
   target = os.readlink(os.path.join(head,uuid))
   idx = target.index('/date/')
   target = "%s%s%s" %(target[:idx+6],target[idx+7:idx+10],target[idx+10:])
   os.unlink(os.path.join(head,uuid))
   os.symlink(target,os.path.join(head,uuid))
   #print "LINK:%s"%(os.readlink(os.path.join(head,uuid)))
   # assure that we don't throw for a badly formatted path
   storage.remove(uuid)

 def testRemoveGoodUuidFromJsonDumpStorage(self):
   """
   testRemoveGoodUuidFromJsonDumpStorage(self):
   This really just wraps JsonDumpStorage call. Assure we aren't futzing up the wrap (succeed with existing uuids)
   """
   global me
   createJDS.createTestSet(createJDS.jsonFileData,jsonKwargs={'logger':me.logger},rootDir=me.config.storageRoot)
   createJDS.createTestSet(createJDS.jsonMoreData,jsonKwargs={'logger':me.logger},rootDir=me.config.deferredStorageRoot)
   mon = monitor.Monitor(me.config)
   goodUuid = '0b781b88-ecbe-4cc4-dead-6bbb20081225';
   # this should work the first time...
   mon.removeUuidFromJsonDumpStorage(goodUuid)
   # ... and then fail the second time
   assert_raises(monitor.UuidNotFoundException,mon.removeUuidFromJsonDumpStorage, goodUuid)

 def testDestructiveDateWalk(self):
   createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[0],self.testDir)
   storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[0])
   uuids = createJDS.jsonFileData.keys()
   #uuidsSet = set(uuids)
   seenids = set()
   for id in storage.destructiveDateWalk():
     assert id in uuids, 'Expect that %s is among the uuids we stored\n%s' % (id,uuids)
     seenids.add(id)
   for id in uuids:
     assert id in seenids, 'Expect that we found every uuid we stored (%s) from %s' % (id,seenids)
   daily = os.listdir(storage.root)
   for d in daily:
     assert not storage.dateName in os.listdir(os.path.join(storage.root,d)), 'Expected all date subdirs to be gone, but %s'%d

 def testGetDump(self):
   createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[1],self.testDir)
   storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[1])
   for uuid,data in createJDS.jsonFileData.items():
     dateparts = data[0].split('-')
     daily = "%4d%02d%02d"%tuple([int(x) for x in dateparts[:3]])
     expected = os.sep.join((storage.root,daily,storage.indexName,data[2],uuid+storage.dumpSuffix))
     got =  storage.getDump(uuid)
     assert expected == got, 'Expected dump file %s, got %s' % (expected,got)
   try:
     storage.getDump(createJDS.jsonBadUuid)
     assert False, 'Should throw IOError from attempt to getDumpAsFile(non-existent-uuid)'
   except OSError,e:
     assert True

 def markAsSeen(self):
   createJDS.createTestSet(createJDS.jsonFileData,self.initKwargs[3],self.testDir)
   storage = JDS.JsonDumpStorage(self.testDir,**self.initKwargs[3])
   for uuid,data in createJDS.jsonFileData.items():
     assert os.path.islink(os.sep.join((storage.dateBranch,data[3],uuid))), 'Expect a link from date to name for %s' % uuid
     assert os.path.islink(os.sep.join((storage.nameBranch,data[2],uuid))), 'Expect link from name to timed for %s' % uuid
     assert not os.path.islink(os.sep.join((storage.dateBranch,data[3],uuid))), 'Expect no link from date to name for %s' % uuid
     assert not os.path.islink(os.sep.join((storage.nameBranch,data[2],uuid))), 'Expect no link from name to date for %s' % uuid
   try:
     storage.markAsSeen(createJDS.jsonBadUuid)
     assert False, 'Expect to throw IOError from attempt to openAndMarkAsSeen(non-existent-uuid)'
   except IOError:
     assert True, 'Got expected error from attempt to openAndMarkAsSeen(non-existent-uuid)'
   except Exception, e:
     assert False, 'Got unexpected error %s from attempt to openAndMarkAsSeen(non-existent-uuid' % e