Golang types.Dat类代码示例

func Dedup(d *types.Dat, deduper Deduper) (*types.Dat, error) {
	dc := new(types.Dat)
	dc.CopyHeader(d)

	for _, g := range d.Games {
		gc := new(types.Game)
		gc.CopyHeader(g)
		for _, r := range g.Roms {
			if !r.Valid() {
				continue
			}
			seen, err := deduper.Seen(r)
			if err != nil {
				return nil, err
			}
			if !seen {
				gc.Roms = append(gc.Roms, r)
				err = deduper.Declare(r)
				if err != nil {
					return nil, err
				}
			}
		}
		if len(gc.Roms) > 0 {
			dc.Games = append(dc.Games, gc)
		}
	}

	if len(dc.Games) > 0 {
		return dc, nil
	}
	return nil, nil
}

func (rs *RombaService) dir2dat(cmd *commander.Command, args []string) error {
	outpath := cmd.Flag.Lookup("out").Value.Get().(string)

	if err := os.MkdirAll(outpath, 0777); err != nil {
		return err
	}

	srcpath := cmd.Flag.Lookup("source").Value.Get().(string)
	srcInfo, err := os.Stat(srcpath)
	if err != nil {
		return err
	}

	if !srcInfo.IsDir() {
		return fmt.Errorf("%s is not a directory", srcpath)
	}

	dat := new(types.Dat)
	dat.Name = cmd.Flag.Lookup("name").Value.Get().(string)
	dat.Description = cmd.Flag.Lookup("description").Value.Get().(string)

	err = archive.Dir2Dat(dat, srcpath, outpath)
	if err != nil {
		return err
	}

	fmt.Fprintf(cmd.Stdout, "dir2dat successfully completed a DAT in %s for directory %s", outpath, srcpath)
	return nil
}

func ParseXml(r io.Reader, path string) (*types.Dat, []byte, error) {
	br := bufio.NewReader(r)

	hr := hashingReader{
		ir: br,
		h:  sha1.New(),
	}

	lr := lineCountingReader{
		ir: hr,
	}

	d := new(types.Dat)
	decoder := xml.NewDecoder(lr)

	err := decoder.Decode(d)
	if err != nil {
		return nil, nil, fmt.Errorf("xml parsing error %d: %v", lr.line, err)
	}

	for _, g := range d.Games {
		for _, rom := range g.Roms {
			fixHashes(rom)
		}
		for _, rom := range g.Disks {
			fixHashes(rom)
		}
		for _, rom := range g.Parts {
			fixHashes(rom)
		}
		for _, rom := range g.Regions {
			fixHashes(rom)
		}
	}

	for _, g := range d.Software {
		for _, rom := range g.Roms {
			fixHashes(rom)
		}
		for _, rom := range g.Disks {
			fixHashes(rom)
		}
		for _, rom := range g.Parts {
			fixHashes(rom)
		}
		for _, rom := range g.Regions {
			fixHashes(rom)
		}
	}

	d.Normalize()
	d.Path = path
	return d, hr.h.Sum(nil), nil
}

func Dir2Dat(dat *types.Dat, srcpath, outpath string) error {
	glog.Infof("composing DAT from source %s into output dir %s", srcpath, outpath)

	fis, err := ioutil.ReadDir(srcpath)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		if fi.IsDir() {
			game, err := populateGame(srcpath, fi)
			if err != nil {
				return err
			}

			dat.Games = append(dat.Games, game)
		}
	}

	outfilename := filepath.Join(outpath, dat.Name+datSuffix)
	outf, err := os.Create(outfilename)
	if err != nil {
		return err
	}
	defer outf.Close()

	outbuf := bufio.NewWriter(outf)
	defer outbuf.Flush()

	return types.ComposeDat(dat, outbuf)
}

func (depot *Depot) BuildDat(dat *types.Dat, outpath string) (bool, error) {
	datPath := filepath.Join(outpath, dat.Name)

	err := os.Mkdir(datPath, 0777)
	if err != nil {
		return false, err
	}

	var fixDat *types.Dat

	for _, game := range dat.Games {
		fixGame, err := depot.buildGame(game, datPath)
		if err != nil {
			return false, err
		}
		if fixGame != nil {
			if fixDat == nil {
				fixDat = new(types.Dat)
				fixDat.Name = dat.Name
				fixDat.Description = dat.Description
				fixDat.Path = dat.Path
			}
			fixDat.Games = append(fixDat.Games, fixGame)
		}
	}

	if fixDat != nil {
		fixDatPath := filepath.Join(outpath, fixPrefix+dat.Name+datSuffix)

		fixFile, err := os.Create(fixDatPath)
		if err != nil {
			return false, err
		}
		defer fixFile.Close()

		fixWriter := bufio.NewWriter(fixFile)
		defer fixWriter.Flush()

		err = types.ComposeDat(fixDat, fixWriter)
		if err != nil {
			return false, err
		}
	}

	return fixDat == nil, nil
}

func writeDiffDat(diffDat *types.Dat, outPath string) error {
	diffDat.Path = outPath

	diffFile, err := os.Create(outPath)
	if err != nil {
		return err
	}
	defer diffFile.Close()

	diffWriter := bufio.NewWriter(diffFile)
	defer diffWriter.Flush()

	return types.ComposeCompliantDat(diffDat, diffWriter)
}

func (kvb *kvBatch) IndexRom(rom *types.Rom) error {
	//glog.Infof("indexing rom %s", rom.Name)

	dats, err := kvb.db.DatsForRom(rom)
	if err != nil {
		return err
	}

	if len(dats) > 0 {
		if rom.Crc != nil && rom.Sha1 != nil {
			//glog.Infof("declaring crc %s -> sha1 %s ampping", hex.EncodeToString(rom.Crc), hex.EncodeToString(rom.Sha1))
			err = kvb.crcsha1Batch.Append(rom.Crc, rom.Sha1)
			if err != nil {
				return err
			}
			kvb.size += int64(sha1.Size)
		}
		if rom.Md5 != nil && rom.Sha1 != nil {
			//glog.Infof("declaring md5 %s -> sha1 %s ampping", hex.EncodeToString(rom.Md5), hex.EncodeToString(rom.Sha1))
			err = kvb.md5sha1Batch.Append(rom.Md5, rom.Sha1)
			if err != nil {
				return err
			}
			kvb.size += int64(sha1.Size)
		}
		return nil
	}

	if rom.Sha1 == nil {
		glog.Warningf("indexing rom %s with missing SHA1", rom.Name)
	}

	dat := new(types.Dat)
	dat.Artificial = true
	dat.Generation = kvb.db.generation
	dat.Name = fmt.Sprintf("Artificial Dat for %s", rom.Name)
	dat.Path = rom.Path
	game := new(types.Game)
	game.Roms = []*types.Rom{rom}
	dat.Games = []*types.Game{game}

	var buf bytes.Buffer

	gobEncoder := gob.NewEncoder(&buf)
	err = gobEncoder.Encode(dat)
	if err != nil {
		return err
	}

	hh := sha1.New()
	_, err = io.Copy(hh, &buf)
	if err != nil {
		return err
	}

	return kvb.IndexDat(dat, hh.Sum(nil))
}

func (kvb *kvBatch) IndexDat(dat *types.Dat, sha1Bytes []byte) error {
	glog.V(4).Infof("indexing dat %s", dat.Name)

	if sha1Bytes == nil {
		return fmt.Errorf("sha1 is nil for %s", dat.Path)
	}

	dat.Generation = kvb.db.generation

	var buf bytes.Buffer

	gobEncoder := gob.NewEncoder(&buf)
	err := gobEncoder.Encode(dat)
	if err != nil {
		return err
	}

	exists, err := kvb.db.datsDB.Exists(sha1Bytes)
	if err != nil {
		return fmt.Errorf("failed to lookup sha1 indexing dats: %v", err)
	}

	kvb.datsBatch.Set(sha1Bytes, buf.Bytes())
	kvb.size += int64(sha1.Size + buf.Len())

	if !exists {
		for _, g := range dat.Games {
			glog.V(4).Infof("indexing game %s", g.Name)
			for _, r := range g.Roms {
				if r.Sha1 != nil {
					err = kvb.sha1Batch.Set(r.Sha1Sha1Key(sha1Bytes), oneValue)
					if err != nil {
						return err
					}
					kvb.size += int64(sha1.Size)
				}

				if r.Md5 != nil {
					err = kvb.md5Batch.Set(r.Md5WithSizeAndSha1Key(sha1Bytes), oneValue)
					if err != nil {
						return err
					}
					kvb.size += int64(sha1.Size)

					if r.Sha1 != nil {
						glog.V(4).Infof("declaring md5 %s -> sha1 %s mapping", hex.EncodeToString(r.Md5), hex.EncodeToString(r.Sha1))
						err = kvb.md5sha1Batch.Set(r.Md5WithSizeAndSha1Key(nil), oneValue)
						if err != nil {
							return err
						}
						kvb.size += int64(sha1.Size)
					}
				}

				if r.Crc != nil {
					err = kvb.crcBatch.Set(r.CrcWithSizeAndSha1Key(sha1Bytes), oneValue)
					if err != nil {
						return err
					}
					kvb.size += int64(sha1.Size)

					if r.Sha1 != nil {
						glog.V(4).Infof("declaring crc %s -> sha1 %s mapping", hex.EncodeToString(r.Crc), hex.EncodeToString(r.Sha1))
						err = kvb.crcsha1Batch.Set(r.CrcWithSizeAndSha1Key(nil), oneValue)
						if err != nil {
							return err
						}
						kvb.size += int64(sha1.Size)
					}
				}
			}
		}
	}
	return nil
}

func ParseXml(r io.Reader, path string) (*types.Dat, []byte, error) {
	br := bufio.NewReader(r)

	hr := hashingReader{
		ir: br,
		h:  sha1.New(),
	}

	lr := lineCountingReader{
		ir: hr,
	}

	d := new(types.Dat)
	decoder := xml.NewDecoder(lr)

	err := decoder.Decode(d)
	if err != nil {
		derrStr := fmt.Sprintf("error in file %s on line %d: %v", path, lr.line, err)
		derr := XMLParseError.NewWith(derrStr, setErrorFilePath(path), setErrorLineNumber(lr.line))
		return nil, nil, derr
	}

	for _, g := range d.Games {
		for _, rom := range g.Roms {
			fixHashes(rom)
		}
		for _, rom := range g.Parts {
			fixHashes(rom)
		}
		for _, rom := range g.Regions {
			fixHashes(rom)
		}
	}

	for _, g := range d.Software {
		for _, rom := range g.Roms {
			fixHashes(rom)
		}
		for _, rom := range g.Parts {
			fixHashes(rom)
		}
		for _, rom := range g.Regions {
			fixHashes(rom)
		}
	}

	for _, g := range d.Machines {
		for _, rom := range g.Roms {
			fixHashes(rom)
		}
		for _, rom := range g.Parts {
			fixHashes(rom)
		}
		for _, rom := range g.Regions {
			fixHashes(rom)
		}
	}

	d.Normalize()
	d.Path = path
	return d, hr.h.Sum(nil), nil
}

func (depot *Depot) BuildDat(dat *types.Dat, outpath string, numSubworkers int, deduper dedup.Deduper) (bool, error) {

	datPath := filepath.Join(outpath, dat.Name)

	err := os.Mkdir(datPath, 0777)
	if err != nil {
		return false, err
	}

	fixDat := new(types.Dat)
	fixDat.FixDat = true
	fixDat.Name = "fix_" + dat.Name
	fixDat.Description = dat.Description
	fixDat.Path = dat.Path
	fixDat.UnzipGames = dat.UnzipGames

	wc := make(chan *types.Game)
	erc := make(chan error)
	closeC := make(chan bool)
	mutex := new(sync.Mutex)

	for i := 0; i < numSubworkers; i++ {
		gb := new(gameBuilder)
		gb.depot = depot
		gb.wc = wc
		gb.erc = erc
		gb.mutex = mutex
		gb.datPath = datPath
		gb.fixDat = fixDat
		gb.index = i
		gb.deduper = deduper
		gb.closeC = closeC

		go gb.work()
	}

	var minionErr error

endLoop:
	for _, game := range dat.Games {
		select {
		case wc <- game:
		case err := <-erc:
			minionErr = err
			break endLoop
		}
	}
	close(wc)

	finishedSubworkers := 0

endLoop2:
	for {
		glog.V(4).Infof("builder master: finished so far %d", finishedSubworkers)

		select {
		case <-closeC:
			glog.V(4).Infof("builder master: finished another subworker")
			finishedSubworkers++
			if finishedSubworkers == numSubworkers {
				break endLoop2
			}
		case err := <-erc:
			glog.V(4).Infof("builder master: minion error")
			minionErr = err
		}
	}

	if minionErr != nil {
		return false, minionErr
	}

	if len(fixDat.Games) > 0 {
		fixDatPath := filepath.Join(outpath, fixPrefix+dat.Name+datSuffix)

		fixFile, err := os.Create(fixDatPath)
		if err != nil {
			return false, err
		}
		defer fixFile.Close()

		fixWriter := bufio.NewWriter(fixFile)
		defer fixWriter.Flush()

		err = types.ComposeCompliantDat(fixDat, fixWriter)
		if err != nil {
			return false, err
		}
	}

	return len(fixDat.Games) > 0, nil
}

func (kvb *kvBatch) IndexDat(dat *types.Dat, sha1Bytes []byte) error {
	glog.Infof("indexing dat %s", dat.Name)

	if sha1Bytes == nil {
		return fmt.Errorf("sha1 is nil for %s", dat.Path)
	}

	dat.Generation = kvb.db.generation

	var buf bytes.Buffer

	gobEncoder := gob.NewEncoder(&buf)
	err := gobEncoder.Encode(dat)
	if err != nil {
		return err
	}

	var exists bool

	if dat.Artificial {
		exists = false
	} else {
		existsSha1, err := kvb.db.datsDB.Exists(sha1Bytes)
		if err != nil {
			return fmt.Errorf("failed to lookup sha1 indexing dats: %v", err)
		}
		exists = existsSha1
	}

	kvb.datsBatch.Set(sha1Bytes, buf.Bytes())

	kvb.size += int64(sha1.Size + buf.Len())

	if !exists {
		for _, g := range dat.Games {
			for _, r := range g.Roms {
				if r.Sha1 != nil {
					err = kvb.sha1Batch.Append(r.Sha1, sha1Bytes)
					if err != nil {
						return err
					}
					kvb.size += int64(sha1.Size)
				}

				if r.Md5 != nil {
					err = kvb.md5Batch.Append(r.Md5, sha1Bytes)
					if err != nil {
						return err
					}
					kvb.size += int64(sha1.Size)

					if r.Sha1 != nil {
						//glog.Infof("declaring md5 %s -> sha1 %s ampping", hex.EncodeToString(r.Md5), hex.EncodeToString(r.Sha1))
						err = kvb.md5sha1Batch.Append(r.Md5, r.Sha1)
						if err != nil {
							return err
						}
						kvb.size += int64(sha1.Size)
					}
				}

				if r.Crc != nil {
					err = kvb.crcBatch.Append(r.Crc, sha1Bytes)
					if err != nil {
						return err
					}
					kvb.size += int64(sha1.Size)

					if r.Sha1 != nil {
						//glog.Infof("declaring crc %s -> sha1 %s ampping", hex.EncodeToString(r.Crc), hex.EncodeToString(r.Sha1))
						err = kvb.crcsha1Batch.Append(r.Crc, r.Sha1)
						if err != nil {
							return err
						}
						kvb.size += int64(sha1.Size)
					}
				}
			}
		}
	}
	return nil
}