C# Service.BatchedGetLinks方法代码示例

 public static void Main(string[] args)
 {
     if (args.Length != 4) {
       Console.Error.WriteLine("Usage: SHS.PageRank <leader> <store> <d> <iters>");
     } else {
       var sw = Stopwatch.StartNew();
       var store = new Service(args[0]).OpenStore(Guid.Parse(args[1]));
       double d = double.Parse(args[2]);
       int numIters = int.Parse(args[3]);
       long n = store.NumUrls();
       using (var wr = new BinaryWriter(new BufferedStream(new FileStream("pr-scores-" + 0 + ".bin", FileMode.Create, FileAccess.Write)))) {
     for (long i = 0; i < n; i++) wr.Write(1.0 / n);
       }
       var scores = store.AllocateUidState<double>();
       var uidBatch = new Batch<long>(50000);
       for (int k = 0; k < numIters; k++) {
     scores.SetAll(x => d / n);
     using (var rd = new BinaryReader(new BufferedStream(new FileStream("pr-scores-" + k + ".bin", FileMode.Open, FileAccess.Read)))) {
       foreach (long u in store.Uids()) {
         uidBatch.Add(u);
         if (uidBatch.Full || store.IsLastUid(u)) {
           var linkBatch = store.BatchedGetLinks(uidBatch, Dir.Fwd);
           var uniqLinks = new UidMap(linkBatch);
           var scoreArr = scores.GetMany(uniqLinks);
           foreach (var links in linkBatch) {
             double f = (1.0 - d) * rd.ReadDouble() / links.Length;
             foreach (var link in links) {
               scoreArr[uniqLinks[link]] += f;
             }
           }
           scores.SetMany(uniqLinks, scoreArr);
           uidBatch.Reset();
         }
       }
     }
     using (var wr = new BinaryWriter(new BufferedStream(new FileStream("pr-scores-" + (k + 1) + ".bin", FileMode.Create, FileAccess.Write)))) {
       foreach (var us in scores.GetAll()) wr.Write(us.val);
     }
     File.Delete("pr-scores-" + k + ".bin");
     Console.WriteLine("Iteration {0} complete", k);
       }
       Console.WriteLine("Done. {0} iterations took {1} seconds.", numIters, 0.001 * sw.ElapsedMilliseconds);
     }
 }

    public static void Main(string[] args)
    {
        if (args.Length != 4) {
          Console.Error.WriteLine("Usage: SHS.PageRankFT <leader> <store> <d> <iters>");
        } else {
          var sw = Stopwatch.StartNew();
          var store = new Service(args[0]).OpenStore(Guid.Parse(args[1]));

          Action<Action> Checkpointed = delegate(Action checkpointedBlock) {
        while (true) {
          try {
            checkpointedBlock();
            store.Checkpoint();
            break;
          } catch (ServerFailure) {
            Console.Error.WriteLine("Restarting from checkpoint");
            // go again
          }
        }
          };

          double d = double.Parse(args[2]);
          int numIters = int.Parse(args[3]);
          long n = store.NumUrls();

          UidState<double> oldScores = null, newScores = null;

          Checkpointed(delegate() {
        newScores = store.AllocateUidState<double>();
        oldScores = store.AllocateUidState<double>();
        oldScores.SetAll(uid => 1.0 / n);
          });

          for (int k = 0; k < numIters; k++) {
        Checkpointed(delegate() {
          var uidBatch = new Batch<long>(50000);
          newScores.SetAll(x => d / n);
          foreach (long u in store.Uids()) {
            uidBatch.Add(u);
            if (uidBatch.Full || store.IsLastUid(u)) {
              var linkBatch = store.BatchedGetLinks(uidBatch, Dir.Fwd);
              var newMap = new UidMap(linkBatch);
              var oldSc = oldScores.GetMany(uidBatch);
              var newSc = newScores.GetMany(newMap);
              for (int i = 0; i < uidBatch.Count; i++) {
                var links = linkBatch[i];
                double f = (1.0 - d) * oldSc[i] / links.Length;
                foreach (var link in links) {
                  newSc[newMap[link]] += f;
                }
              }
              newScores.SetMany(newMap, newSc);
              uidBatch.Reset();
            }
          }
        });
        var tmp = newScores; newScores = oldScores; oldScores = tmp;
        Console.WriteLine("Done with iteration {0}", k);
          }
          using (var wr = new BinaryWriter(new BufferedStream(new FileStream("pr-scores.bin", FileMode.Create, FileAccess.Write)))) {
        foreach (var us in newScores.GetAll()) wr.Write(us.val);
          }
          Console.WriteLine("Done. {0} iterations took {1} seconds.", numIters, 0.001 * sw.ElapsedMilliseconds);
        }
    }

 public static void Main(string[] args)
 {
     if (args.Length != 2) {
       Console.Error.WriteLine("Usage: SHS.SCC2 <leader> <store>");
     } else {
       var sw = Stopwatch.StartNew();
       var shs = new Service(args[0]).OpenStore(Guid.Parse(args[1]));
       var map = shs.AllocateUidState<int>();  // Mapping from UID to local ID
       int numVerts = 0;  // Number of core vertices
       var batch = new Batch<long>(500000);
       foreach (long u in shs.Uids()) {
     batch.Add(u);
     if (batch.Full || shs.IsLastUid(u)) {
       int[] fwdDegs = shs.BatchedGetDegree(batch, Dir.Fwd);
       int[] bwdDegs = shs.BatchedGetDegree(batch, Dir.Bwd);
       var mapChunk = new int[batch.Count];
       for (int i = 0; i < batch.Count; i++) {
         mapChunk[i] = fwdDegs[i] == 0 || bwdDegs[i] == 0 ? -1 : numVerts++;
       }
       map.SetMany(batch, mapChunk);
       batch.Reset();
     }
       }
       uint numEdges = 0;
       foreach (var up in map.GetAll()) {
     if (up.val != -1) batch.Add(up.uid);
     if (batch.Full || shs.IsLastUid(up.uid)) {
       long[][] nbors = shs.BatchedGetLinks(batch, Dir.Fwd);
       int[] mappedNbors = map.GetMany(Flatten(nbors));
       int q = 0;
       for (int i = 0; i < nbors.Length; i++) {
         for (int j = 0; j < nbors[i].Length; j++) {
           if (mappedNbors[q++] != -1) numEdges++;
         }
       }
       batch.Reset();
     }
       }
       uint[] pastLast = new uint[numVerts]; // one past last link of that page
       var links = new int[numEdges];
       int p = 0;
       uint r = 0;
       foreach (var up in map.GetAll()) {
     if (up.val != -1) batch.Add(up.uid);
     if (batch.Full || shs.IsLastUid(up.uid)) {
       long[][] nbors = shs.BatchedGetLinks(batch, Dir.Fwd);
       int[] mappedNbors = map.GetMany(Flatten(nbors));
       int q = 0;
       for (int i = 0; i < nbors.Length; i++) {
         for (int j = 0; j < nbors[i].Length; j++) {
           int id = mappedNbors[q++];
           if (id != -1) links[r++] = id;
         }
         pastLast[p++] = r;
       }
       batch.Reset();
     }
       }
       var bv = new BitVector(numVerts);  // All false at creation
       int[] stk = new int[numVerts];
       int stkPtr = stk.Length;
       for (int u = 0; u < numVerts; u++) {
     if (!bv[u]) {
       bv[u] = true;
       Frame frame = new Frame(null, u, pastLast);
       while (frame != null) {
         while (frame.ctr < pastLast[frame.id]) {
           int v = links[frame.ctr++];
           if (!bv[v]) {
             bv[v] = true;
             frame = new Frame(frame, v, pastLast);
           }
         }
         stk[--stkPtr] = frame.id;
         frame = frame.parent;
       }
     }
       }
       p = 0;
       r = 0;
       foreach (var up in map.GetAll()) {
     if (up.val != -1) batch.Add(up.uid);
     if (batch.Full || shs.IsLastUid(up.uid)) {
       long[][] nbors = shs.BatchedGetLinks(batch, Dir.Bwd);
       int[] mappedNbors = map.GetMany(Flatten(nbors));
       int q = 0;
       for (int i = 0; i < nbors.Length; i++) {
         for (int j = 0; j < nbors[i].Length; j++) {
           int id = mappedNbors[q++];
           if (id != -1) links[r++] = id;
         }
         pastLast[p++] = r;
       }
       batch.Reset();
     }
       }
       var pam = new long[numVerts];
       p = 0;
       foreach (var up in map.GetAll()) {
     if (up.val != -1) pam[p++] = up.uid;
//.........这里部分代码省略.........

    static void Main(string[] args)
    {
        bool rfl = false;  // Can LinkDB contain reflexive edges?
        bool dup = false;  // Can LinkDB contain duplicate edges?
        if (args.Length == 4 && string.CompareOrdinal(args[0], "refdup") == 0) {
          rfl = true;
          dup = true;
        } else if (args.Length == 4 && string.CompareOrdinal(args[0], "refnodup") == 0) {
          rfl = true;
          dup = false;
        } else if (args.Length == 4 && string.CompareOrdinal(args[0], "dupnoref") == 0) {
          rfl = false;
          dup = true;
        } else if (args.Length == 4 && string.CompareOrdinal(args[0], "norefdup") == 0) {
          rfl = false;
          dup = false;
        } else {
          Console.Error.WriteLine("Usage: SHS.RegressionTest0 [refdup|refnodup|dupnoref|norefdup] <rawlinks.bin> <servers.txt> <store>");
          System.Environment.Exit(1);
        }
        var sw = System.Diagnostics.Stopwatch.StartNew();
        var smt = new ThriftySetMemberTest();
        var shs = new Service(args[2]).OpenStore(Guid.Parse(args[3]));
        long pageCnt = 0;
        foreach (var fileName in File.ReadAllLines(args[1])) {
          using (var rd = new BinaryReader(new GZipStream(new BufferedStream(new FileStream(fileName, FileMode.Open, FileAccess.Read)), CompressionMode.Decompress))) {
        while (true) {
          try {
            string pageUrl = rd.ReadString();
            long pageUid = shs.UrlToUid(pageUrl);
            string pageUidUrl = shs.UidToUrl(pageUid);
            if (string.CompareOrdinal(pageUrl, pageUidUrl) != 0) {
              Console.Error.WriteLine();
              Console.Error.WriteLine("Regression test failed for page {0} (self-test)", pageCnt);
              Console.Error.WriteLine("Original URL: {0}", pageUrl);
              Console.Error.WriteLine("UrlToUid UID: {0,16:X}", pageUid);
              Console.Error.WriteLine("UidToUrl URL: {0}", pageUidUrl);
              System.Environment.Exit(1);
            }
            int numLinks = rd.ReadInt32();
            var linkUrls = new string[numLinks];
            for (int i = 0; i < numLinks; i++) {
              linkUrls[i] = rd.ReadString();
            }
            if (!smt.ContainsOrAdd(pageUrl)) {
              // Ignore self-links
              var nonReflectiveL = new List<string>();
              for (int i = 0; i < numLinks; i++) {
                if (string.CompareOrdinal(pageUrl, linkUrls[i]) != 0) nonReflectiveL.Add(linkUrls[i]);
              }
              var nonReflectiveA = nonReflectiveL.ToArray();
              var linkUids = shs.BatchedUrlToUid(nonReflectiveA);
              var bwdLinkUids = shs.BatchedGetLinks(linkUids, Dir.Bwd);

              // Backward test first
              for (int i = 0; i < bwdLinkUids.Length; i++) {
                bool found = false;
                for (int j = 0; j < bwdLinkUids[i].Length && !found; j++) {
                  if (pageUid == bwdLinkUids[i][j]) found = true;
                }
                if (!found) {
                  var bwdLinkUrls = shs.BatchedUidToUrl(bwdLinkUids[i]);
                  Console.Error.WriteLine();
                  Console.Error.WriteLine("Regression test failed after {0} seconds for page {1} (bwd test)",
                                          0.001 * sw.ElapsedMilliseconds, pageCnt);
                  Console.Error.WriteLine("pageUrl={0}", pageUrl);
                  Console.Error.WriteLine("linkUrl[{0}]={1}", i, linkUrls[i]);
                  Console.Error.WriteLine("bwdLinkUrls.Length={0}", bwdLinkUrls.Length);
                  for (int j = 0; j < bwdLinkUrls.Length; j++) {
                    Console.Error.WriteLine("bwdLinkUrl[{0}]={1}", j, bwdLinkUrls[j]);
                  }
                  System.Environment.Exit(1);
                }
              }

              // Forward test second
              var fwdLinkUrls = shs.BatchedUidToUrl(shs.GetLinks(pageUid, Dir.Fwd));
              Array.Sort<string>(linkUrls, string.CompareOrdinal);
              Array.Sort<string>(fwdLinkUrls, string.CompareOrdinal);
              int a = 0;
              int b = 0;
              while (a < numLinks && b < fwdLinkUrls.Length) {
                // Skip any reflexive and/or duplicate links, if desired
                while ((!rfl && a < numLinks && string.CompareOrdinal(pageUrl, linkUrls[a]) == 0)
                       || (!dup && a < numLinks - 1 && string.CompareOrdinal(linkUrls[a], linkUrls[a + 1]) == 0)) {
                  a++;
                }
                if (a >= numLinks) {
                  break;
                }
                if (string.CompareOrdinal(linkUrls[a], fwdLinkUrls[b]) != 0) {
                  Console.Error.WriteLine();
                  Console.Error.WriteLine("Regression test failed for page {0} (fwd test1)",
                          pageCnt);
                  Console.Error.WriteLine("raw links file has {0} links:", numLinks);
                  for (int i = 0; i < numLinks; i++) {
                    Console.Error.WriteLine("  link[{0}]={1}", i, linkUrls[i]);
                  }
                  Console.Error.WriteLine("fwd db has {0} links:", fwdLinkUrls.Length);
                  for (int i = 0; i < fwdLinkUrls.Length; i++) {
//.........这里部分代码省略.........

 public static void Main(string[] args)
 {
     var shs = new Service(args[0]).OpenStore(Guid.Parse(args[1]));
     using (var rd = new BinaryReader(new BufferedStream(new FileStream(args[2], FileMode.Open, FileAccess.Read)))) {
       int bs = int.Parse(args[3]);
       int fs = int.Parse(args[4]);
       while (true) {
     try {
       int queryId = rd.ReadInt32();
       int numUrls = rd.ReadInt32();
       var urls = new string[numUrls];
       for (int i = 0; i < numUrls; i++) urls[i] = rd.ReadString();
       var uids = shs.BatchedUrlToUid(urls);
       var tbl = new UidMap(uids);
       var bwdUids = shs.BatchedSampleLinks(tbl, Dir.Bwd, bs, true);
       var fwdUids = shs.BatchedSampleLinks(tbl, Dir.Fwd, fs, true);
       foreach (long[] x in bwdUids) tbl.Add(x);
       foreach (long[] x in fwdUids) tbl.Add(x);
       long[] srcUids = tbl;
       var dstUids = shs.BatchedGetLinks(srcUids, Dir.Fwd);
       int n = dstUids.Length;
       var srcId = new List<int>[n];
       var dstId = new List<int>[n];
       for (int i = 0; i < n; i++) {
         srcId[i] = new List<int>();
         dstId[i] = new List<int>();
       }
       for (int i = 0; i < n; i++) {
         int sid = tbl[srcUids[i]];
         for (int j = 0; j < dstUids[i].Length; j++) {
           int did = tbl[dstUids[i][j]];
           if (did != -1) {
             srcId[sid].Add(did);
             dstId[did].Add(sid);
           }
         }
       }
       int numAuts = 0;
       for (int i = 0; i < n; i++) {
         if (dstId[i].Count > 0) numAuts++;
       }
       double initAut = 1.0 / numAuts;
       var aut = new double[n];
       var tmp = new double[n];
       for (int i = 0; i < n; i++) {
         aut[i] = dstId[i].Count > 0 ? initAut : 0.0;
       }
       for (int k = 0; k < 100; k++) {
         for (int u = 0; u < n; u++) {
           foreach (var id in dstId[u]) {
             tmp[id] += (aut[u] / dstId[u].Count);
           }
           aut[u] = 0.0;
         }
         for (int u = 0; u < n; u++) {
           foreach (var id in srcId[u]) {
             aut[id] += (tmp[u] / srcId[u].Count);
           }
           tmp[u] = 0.0;
         }
       }
       var scores = new double[urls.Length];
       for (int i = 0; i < scores.Length; i++) {
         scores[i] = uids[i] == -1 ? 0.0 : aut[tbl[uids[i]]];
       }
       double bestScore = double.MinValue;
       string bestUrl = null;
       for (int i = 0; i < urls.Length; i++) {
         if (scores[i] > bestScore) {
           bestScore = scores[i];
           bestUrl = urls[i];
         }
       }
       System.Console.Error.WriteLine("{0} {1}", queryId, bestUrl);
     } catch (EndOfStreamException) {
       break;
     }
       }
     }
 }