Python Client.text方法代码示例

# 需要导入模块: from snakebite.client import Client [as 别名]
# 或者: from snakebite.client.Client import text [as 别名]
class HDFSTextLoader(Unit, TriviallyDistributable):
    def __init__(self, workflow, **kwargs):
        super(HDFSTextLoader, self).__init__(workflow, **kwargs)
        self.file_name = kwargs["file"]
        self.chunk_lines_number = kwargs.get("chunk", 1000)
        client_kwargs = dict(kwargs)
        del client_kwargs["file"]
        if "chunk" in kwargs:
            del client_kwargs["chunk"]
        self.hdfs_client = Client(**client_kwargs)
        self.output = [""] * self.chunk_lines_number
        self.finished = Bool()

    def initialize(self):
        self.debug("Opened %s", self.hdfs_client.stat([self.file_name]))
        self._generator = self.hdfs_client.text([self.file_name])

    def run(self):
        assert not self.finished
        try:
            for i in range(self.chunk_lines_number):
                self.output[i] = next(self._generator)
        except StopIteration:
            self.finished <<= True

# 需要导入模块: from snakebite.client import Client [as 别名]
# 或者: from snakebite.client.Client import text [as 别名]
# Create kafka client
print "Create kafka client to: %s" % args.kafka
kafka = KafkaClient(args.kafka + ':9092')
producer = SimpleProducer(kafka)

# Read testing data from hdfs
hdfsServer = args.hdfs
hdfsPort = int(os.environ.get('HDFS_NAME_PORT', 8020))
hdfsHost = "hdfs://" + hdfsServer + ":" + str(hdfsPort)

topic = args.topic

from snakebite.client import Client
print "Reading input from HDFS: server=%s, port=%d" % (hdfsServer, hdfsPort)
client = Client(hdfsServer, hdfsPort)
data_file = client.text(["/user/" + os.getenv('LOGNAME') + "/data/X_test.txt"]).next()
label_file = client.text(["/user/" + os.getenv('LOGNAME') + "/data/y_test.txt"]).next()

samples = data_file.splitlines()
labels = label_file.splitlines()
test_data = zip(samples, labels)
random.shuffle(test_data) # Shuffle it

import random
import time
import itertools

def getActivityName(a):
    a = int(a)
    if a in range(1,7):
        return str(a)

# 需要导入模块: from snakebite.client import Client [as 别名]
# 或者: from snakebite.client.Client import text [as 别名]
from snakebite.client import Client

client = Client('localhost', 9000)
for l in client.text(['/input/input.txt']):
   print l

# 需要导入模块: from snakebite.client import Client [as 别名]
# 或者: from snakebite.client.Client import text [as 别名]
parser = argparse.ArgumentParser()
parser.add_argument("--hdfs", help="HDFS FS name", default = 'localhost')
parser.add_argument("--model", help="Name of model file", default = 'belt.model')
args = parser.parse_args()


hdfsServer = args.hdfs
hdfsPort = int(os.environ.get('HDFS_NAME_PORT', 8020))
hdfsHost = "hdfs://" + hdfsServer + ":" + str(hdfsPort)
modelSavePath = "/user/" + os.getenv('LOGNAME') + "/data/model/" + args.model + "/"
print "hdfs=%s, savePath=%s, hdfsHost=%s" % (hdfsServer, modelSavePath, hdfsHost)

hdfs_client = Client(hdfsServer, hdfsPort)

X_train_file = hdfs_client.text(["/user/" + os.getenv('LOGNAME') + "/data/X_train.txt"]).next()
y_train_file = hdfs_client.text(["/user/" + os.getenv('LOGNAME') + "/data/y_train.txt"]).next()

X_train = np.genfromtxt(str.splitlines(X_train_file))
y_train = np.genfromtxt(str.splitlines(y_train_file))

clf = LogisticRegression()
clf = clf.fit(X_train, y_train)

files = joblib.dump(clf, "belt.model")

subprocess.check_call(['hdfs', 'dfs', '-rm', '-r', '-f', modelSavePath], shell=False)
subprocess.check_call(['hdfs', 'dfs', '-mkdir', '-p', modelSavePath], shell=False)

for f in files:
    subprocess.check_call(['hdfs', 'dfs', '-put', os.getcwd() + '/' + f, modelSavePath + f], shell=False)

# 需要导入模块: from snakebite.client import Client [as 别名]
# 或者: from snakebite.client.Client import text [as 别名]
import os
from snakebite.client import Client

# provide the Internet Process Communcation Port
INTERNET_PROCESS_CIOMMUNICATION_PORT = "..."

# provide the Name Node of Hadoop
NAME_NODE = "..."

# and get the client of HDFS
CLIENT_HDFS = Client(NAME_NODE, INTERNET_PROCESS_CIOMMUNICATION_PORT)


def read_hdfs_file(file_path_and_name)
    """Reads an hdfs file
    :param meta_info_file: the path and the file to read
    """

    # 1. gets the hdfs file object
    for file_contents in CLIENT_HDFS.text([hdfs_file_name]):
        file_unicode = file_contents.decode('unicode-escape')
        file_obj = StringIO(file_unicode)

    # 2. read and operate on top:
    file_lines = get_hdfs_file_obj(meta_info_file).readlines()
    for line in file_lines:
        # ...
        # do operations on the file